Commit 75d60496 authored by Grzegorz Bizon's avatar Grzegorz Bizon

Merge branch 'per-project-pipeline-iid-ee' into 'master'

Port to EE: Add per-project pipeline id (Use `AtomicInternalId`)

See merge request gitlab-org/gitlab-ee!5745
parents db0f552a d14d6040
......@@ -7,6 +7,7 @@ module Ci
include Presentable
include Gitlab::OptimisticLocking
include Gitlab::Utils::StrongMemoize
include AtomicInternalId
prepend ::EE::Ci::Pipeline
......@@ -15,6 +16,10 @@ module Ci
belongs_to :auto_canceled_by, class_name: 'Ci::Pipeline'
belongs_to :pipeline_schedule, class_name: 'Ci::PipelineSchedule'
has_internal_id :iid, scope: :project, presence: false, init: ->(s) do
s&.project&.pipelines&.maximum(:iid) || s&.project&.pipelines&.count
end
has_one :source_pipeline, class_name: Ci::Sources::Pipeline
has_many :sourced_pipelines, class_name: Ci::Sources::Pipeline, foreign_key: :source_pipeline_id
......@@ -549,6 +554,7 @@ module Ci
def predefined_variables
Gitlab::Ci::Variables::Collection.new
.append(key: 'CI_PIPELINE_IID', value: iid.to_s)
.append(key: 'CI_CONFIG_PATH', value: ci_yaml_file_path)
.append(key: 'CI_PIPELINE_SOURCE', value: source.to_s)
.append(key: 'CI_COMMIT_MESSAGE', value: git_commit_message)
......
......@@ -25,9 +25,13 @@ module AtomicInternalId
extend ActiveSupport::Concern
module ClassMethods
def has_internal_id(column, scope:, init:) # rubocop:disable Naming/PredicateName
before_validation(on: :create) do
def has_internal_id(column, scope:, init:, presence: true) # rubocop:disable Naming/PredicateName
before_validation :"ensure_#{scope}_#{column}!", on: :create
validates column, presence: presence
define_method("ensure_#{scope}_#{column}!") do
scope_value = association(scope).reader
if read_attribute(column).blank? && scope_value
scope_attrs = { scope_value.class.table_name.singularize.to_sym => scope_value }
usage = self.class.table_name.to_sym
......@@ -35,13 +39,9 @@ module AtomicInternalId
new_iid = InternalId.generate_next(self, scope_attrs, usage, init)
write_attribute(column, new_iid)
end
end
validates column, presence: true, numericality: true
read_attribute(column)
end
end
end
def to_param
iid.to_s
end
end
module IidRoutes
##
# This automagically enforces all related routes to use `iid` instead of `id`
# If you want to use `iid` for some routes and `id` for other routes, this module should not to be included,
# instead you should define `iid` or `id` explictly at each route generators. e.g. pipeline_path(project.id, pipeline.iid)
def to_param
iid.to_s
end
end
class Deployment < ActiveRecord::Base
include AtomicInternalId
include IidRoutes
belongs_to :project, required: true
belongs_to :environment, required: true
......
......@@ -14,7 +14,7 @@ class InternalId < ActiveRecord::Base
belongs_to :project
belongs_to :namespace
enum usage: { issues: 0, merge_requests: 1, deployments: 2, milestones: 3, epics: 4 }
enum usage: { issues: 0, merge_requests: 1, deployments: 2, milestones: 3, epics: 4, ci_pipelines: 5 }
validates :usage, presence: true
......
......@@ -5,6 +5,7 @@ class Issue < ActiveRecord::Base
prepend EE::RelativePositioning
include AtomicInternalId
include IidRoutes
include Issuable
include Noteable
include Referable
......
class MergeRequest < ActiveRecord::Base
include AtomicInternalId
include IidRoutes
include Issuable
include Noteable
include Referable
......
......@@ -9,6 +9,7 @@ class Milestone < ActiveRecord::Base
include CacheMarkdownField
include AtomicInternalId
include IidRoutes
include Sortable
include Referable
include StripAttribute
......
---
title: Add per-project pipeline id
merge_request: 18558
author:
type: added
class AddPipelineIidToCiPipelines < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
def up
add_column :ci_pipelines, :iid, :integer
end
def down
remove_column :ci_pipelines, :iid, :integer
end
end
class AddIndexConstraintsToPipelineIid < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
disable_ddl_transaction!
def up
add_concurrent_index :ci_pipelines, [:project_id, :iid], unique: true, where: 'iid IS NOT NULL'
end
def down
remove_concurrent_index :ci_pipelines, [:project_id, :iid]
end
end
......@@ -547,10 +547,12 @@ ActiveRecord::Schema.define(version: 20180529093006) do
t.integer "config_source"
t.boolean "protected"
t.integer "failure_reason"
t.integer "iid"
end
add_index "ci_pipelines", ["auto_canceled_by_id"], name: "index_ci_pipelines_on_auto_canceled_by_id", using: :btree
add_index "ci_pipelines", ["pipeline_schedule_id"], name: "index_ci_pipelines_on_pipeline_schedule_id", using: :btree
add_index "ci_pipelines", ["project_id", "iid"], name: "index_ci_pipelines_on_project_id_and_iid", unique: true, where: "(iid IS NOT NULL)", using: :btree
add_index "ci_pipelines", ["project_id", "ref", "status", "id"], name: "index_ci_pipelines_on_project_id_and_ref_and_status_and_id", using: :btree
add_index "ci_pipelines", ["project_id", "sha"], name: "index_ci_pipelines_on_project_id_and_sha", using: :btree
add_index "ci_pipelines", ["project_id"], name: "index_ci_pipelines_on_project_id", using: :btree
......
......@@ -74,6 +74,7 @@ future GitLab releases.**
| **CI_RUNNER_REVISION** | all | 10.6 | GitLab Runner revision that is executing the current job |
| **CI_RUNNER_EXECUTABLE_ARCH** | all | 10.6 | The OS/architecture of the GitLab Runner executable (note that this is not necessarily the same as the environment of the executor) |
| **CI_PIPELINE_ID** | 8.10 | 0.5 | The unique id of the current pipeline that GitLab CI uses internally |
| **CI_PIPELINE_IID** | 11.0 | all | The unique id of the current pipeline scoped to project |
| **CI_PIPELINE_TRIGGERED** | all | all | The flag to indicate that job was [triggered] |
| **CI_PIPELINE_SOURCE** | 10.0 | all | Indicates how the pipeline was triggered. Possible options are: `push`, `web`, `trigger`, `schedule`, `api`, and `pipeline`. For pipelines created before GitLab 9.5, this will show as `unknown` |
| **CI_PROJECT_DIR** | all | all | The full path where the repository is cloned and where the job is run |
......@@ -373,6 +374,8 @@ Running on runner-8a2f473d-project-1796893-concurrent-0 via runner-8a2f473d-mach
++ CI_PROJECT_URL=https://example.com/gitlab-examples/ci-debug-trace
++ export CI_PIPELINE_ID=52666
++ CI_PIPELINE_ID=52666
++ export CI_PIPELINE_IID=123
++ CI_PIPELINE_IID=123
++ export CI_RUNNER_ID=1337
++ CI_RUNNER_ID=1337
++ export CI_RUNNER_DESCRIPTION=shared-runners-manager-1.example.com
......@@ -460,6 +463,7 @@ export CI_JOB_MANUAL="true"
export CI_JOB_TRIGGERED="true"
export CI_JOB_TOKEN="abcde-1234ABCD5678ef"
export CI_PIPELINE_ID="1000"
export CI_PIPELINE_IID="10"
export CI_PROJECT_ID="34"
export CI_PROJECT_DIR="/builds/gitlab-org/gitlab-ce"
export CI_PROJECT_NAME="gitlab-ce"
......
......@@ -4,6 +4,7 @@ module EE
prepended do
include AtomicInternalId
include IidRoutes
include ::Issuable
include Noteable
include Referable
......
......@@ -23,6 +23,7 @@ describe Epic do
it_behaves_like 'AtomicInternalId' do
let(:internal_id_attribute) { :iid }
let(:instance) { build(:epic) }
let(:scope) { :group }
let(:scope_attrs) { { namespace: instance.group } }
let(:usage) { :epics }
end
......
......@@ -8,6 +8,9 @@ module Gitlab
PopulateError = Class.new(StandardError)
def perform!
# Allocate next IID. This operation must be outside of transactions of pipeline creations.
pipeline.ensure_project_iid!
##
# Populate pipeline with block argument of CreatePipelineService#execute.
#
......
......@@ -42,6 +42,10 @@ describe Gitlab::Ci::Pipeline::Chain::Populate do
it 'correctly assigns user' do
expect(pipeline.builds).to all(have_attributes(user: user))
end
it 'has pipeline iid' do
expect(pipeline.iid).to be > 0
end
end
context 'when pipeline is empty' do
......@@ -68,6 +72,10 @@ describe Gitlab::Ci::Pipeline::Chain::Populate do
expect(pipeline.errors.to_a)
.to include 'No stages / jobs for this pipeline.'
end
it 'wastes pipeline iid' do
expect(InternalId.ci_pipelines.where(project_id: project.id).last.last_value).to be > 0
end
end
context 'when pipeline has validation errors' do
......@@ -87,6 +95,10 @@ describe Gitlab::Ci::Pipeline::Chain::Populate do
expect(pipeline.errors.to_a)
.to include 'Failed to build the pipeline!'
end
it 'wastes pipeline iid' do
expect(InternalId.ci_pipelines.where(project_id: project.id).last.last_value).to be > 0
end
end
context 'when there is a seed blocks present' do
......@@ -111,6 +123,12 @@ describe Gitlab::Ci::Pipeline::Chain::Populate do
expect(pipeline.variables.first.key).to eq 'VAR'
expect(pipeline.variables.first.value).to eq '123'
end
it 'has pipeline iid' do
step.perform!
expect(pipeline.iid).to be > 0
end
end
context 'when seeds block tries to persist some resources' do
......@@ -121,6 +139,12 @@ describe Gitlab::Ci::Pipeline::Chain::Populate do
it 'raises exception' do
expect { step.perform! }.to raise_error(ActiveRecord::RecordNotSaved)
end
it 'wastes pipeline iid' do
expect { step.perform! }.to raise_error
expect(InternalId.ci_pipelines.where(project_id: project.id).last.last_value).to be > 0
end
end
end
......@@ -132,22 +156,39 @@ describe Gitlab::Ci::Pipeline::Chain::Populate do
end
end
context 'when using only/except build policies' do
let(:config) do
{ rspec: { script: 'rspec', stage: 'test', only: ['master'] },
prod: { script: 'cap prod', stage: 'deploy', only: ['tags'] } }
end
context 'when variables policy is specified' do
shared_examples_for 'a correct pipeline' do
it 'populates pipeline according to used policies' do
step.perform!
let(:pipeline) do
build(:ci_pipeline, ref: 'master', project: project, config: config)
expect(pipeline.stages.size).to eq 1
expect(pipeline.stages.first.builds.size).to eq 1
expect(pipeline.stages.first.builds.first.name).to eq 'rspec'
end
end
it 'populates pipeline according to used policies' do
step.perform!
context 'when using only/except build policies' do
let(:config) do
{ rspec: { script: 'rspec', stage: 'test', only: ['master'] },
prod: { script: 'cap prod', stage: 'deploy', only: ['tags'] } }
end
let(:pipeline) do
build(:ci_pipeline, ref: 'master', project: project, config: config)
end
expect(pipeline.stages.size).to eq 1
expect(pipeline.stages.first.builds.size).to eq 1
expect(pipeline.stages.first.builds.first.name).to eq 'rspec'
it_behaves_like 'a correct pipeline'
context 'when variables expression is specified' do
context 'when pipeline iid is the subject' do
let(:config) do
{ rspec: { script: 'rspec', only: { variables: ["$CI_PIPELINE_IID == '1'"] } },
prod: { script: 'cap prod', only: { variables: ["$CI_PIPELINE_IID == '1000'"] } } }
end
it_behaves_like 'a correct pipeline'
end
end
end
end
end
......@@ -245,6 +245,7 @@ Ci::Pipeline:
- config_source
- failure_reason
- protected
- iid
Ci::Stage:
- id
- name
......
......@@ -1579,6 +1579,7 @@ describe Ci::Build do
{ key: 'CI_PROJECT_NAMESPACE', value: project.namespace.full_path, public: true },
{ key: 'CI_PROJECT_URL', value: project.web_url, public: true },
{ key: 'CI_PROJECT_VISIBILITY', value: 'private', public: true },
{ key: 'CI_PIPELINE_IID', value: pipeline.iid.to_s, public: true },
{ key: 'CI_CONFIG_PATH', value: pipeline.ci_yaml_file_path, public: true },
{ key: 'CI_PIPELINE_SOURCE', value: pipeline.source, public: true },
{ key: 'CI_COMMIT_MESSAGE', value: pipeline.git_commit_message, public: true },
......
......@@ -39,6 +39,16 @@ describe Ci::Pipeline, :mailer do
end
end
describe 'modules' do
it_behaves_like 'AtomicInternalId', validate_presence: false do
let(:internal_id_attribute) { :iid }
let(:instance) { build(:ci_pipeline) }
let(:scope) { :project }
let(:scope_attrs) { { project: instance.project } }
let(:usage) { :ci_pipelines }
end
end
describe '#source' do
context 'when creating new pipeline' do
let(:pipeline) do
......@@ -199,7 +209,8 @@ describe Ci::Pipeline, :mailer do
it 'includes all predefined variables in a valid order' do
keys = subject.map { |variable| variable[:key] }
expect(keys).to eq %w[CI_CONFIG_PATH
expect(keys).to eq %w[CI_PIPELINE_IID
CI_CONFIG_PATH
CI_PIPELINE_SOURCE
CI_COMMIT_MESSAGE
CI_COMMIT_TITLE
......
......@@ -20,6 +20,7 @@ describe Deployment do
it_behaves_like 'AtomicInternalId' do
let(:internal_id_attribute) { :iid }
let(:instance) { build(:deployment) }
let(:scope) { :project }
let(:scope_attrs) { { project: instance.project } }
let(:usage) { :deployments }
end
......
......@@ -17,6 +17,7 @@ describe Issue do
it_behaves_like 'AtomicInternalId' do
let(:internal_id_attribute) { :iid }
let(:instance) { build(:issue) }
let(:scope) { :project }
let(:scope_attrs) { { project: instance.project } }
let(:usage) { :issues }
end
......
......@@ -84,6 +84,7 @@ describe MergeRequest do
it_behaves_like 'AtomicInternalId' do
let(:internal_id_attribute) { :iid }
let(:instance) { build(:merge_request) }
let(:scope) { :target_project }
let(:scope_attrs) { { project: instance.target_project } }
let(:usage) { :merge_requests }
end
......
......@@ -6,6 +6,7 @@ describe Milestone do
it_behaves_like 'AtomicInternalId' do
let(:internal_id_attribute) { :iid }
let(:instance) { build(:milestone, project: build(:project), group: nil) }
let(:scope) { :project }
let(:scope_attrs) { { project: instance.project } }
let(:usage) { :milestones }
end
......@@ -15,6 +16,7 @@ describe Milestone do
it_behaves_like 'AtomicInternalId' do
let(:internal_id_attribute) { :iid }
let(:instance) { build(:milestone, project: nil, group: build(:group)) }
let(:scope) { :group }
let(:scope_attrs) { { namespace: instance.group } }
let(:usage) { :milestones }
end
......
require 'spec_helper'
shared_examples_for 'AtomicInternalId' do
shared_examples_for 'AtomicInternalId' do |validate_presence: true|
describe '.has_internal_id' do
describe 'Module inclusion' do
subject { described_class }
......@@ -9,14 +9,31 @@ shared_examples_for 'AtomicInternalId' do
end
describe 'Validation' do
subject { instance }
before do
allow(InternalId).to receive(:generate_next).and_return(nil)
allow_any_instance_of(described_class).to receive(:"ensure_#{scope}_#{internal_id_attribute}!")
instance.valid?
end
it { is_expected.to validate_presence_of(internal_id_attribute) }
it { is_expected.to validate_numericality_of(internal_id_attribute) }
context 'when presence validation is required' do
before do
skip unless validate_presence
end
it 'validates presence' do
expect(instance.errors[internal_id_attribute]).to include("can't be blank")
end
end
context 'when presence validation is not required' do
before do
skip if validate_presence
end
it 'does not validate presence' do
expect(instance.errors[internal_id_attribute]).to be_empty
end
end
end
describe 'Creating an instance' do
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment