Commit 0bd8eed8 authored by Kamil Trzciński's avatar Kamil Trzciński

Merge branch 'mc/feature/trigger-pipelines-project-subscriptions' into 'master'

Implement subscription triggering mechanism

Closes #9045

See merge request gitlab-org/gitlab!20063
parents f33f8847 7a933848
---
title: Add ability to trigger pipelines when project is rebuilt.
merge_request: 20063
author:
type: added
# frozen_string_literal: true
class AddProjectSubscriptionsToPlanLimits < ActiveRecord::Migration[6.0]
DOWNTIME = false
def change
add_column(:plan_limits, :ci_project_subscriptions, :integer, default: 0, null: false)
end
end
# frozen_string_literal: true
class InsertProjectSubscriptionsPlanLimits < ActiveRecord::Migration[6.0]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
def up
return if Rails.env.test?
if Gitlab.com?
create_or_update_plan_limit('ci_project_subscriptions', 'free', 2)
create_or_update_plan_limit('ci_project_subscriptions', 'bronze', 2)
create_or_update_plan_limit('ci_project_subscriptions', 'silver', 2)
create_or_update_plan_limit('ci_project_subscriptions', 'gold', 2)
else
create_or_update_plan_limit('ci_project_subscriptions', 'default', 2)
end
end
def down
return if Rails.env.test?
if Gitlab.com?
create_or_update_plan_limit('ci_project_subscriptions', 'free', 0)
create_or_update_plan_limit('ci_project_subscriptions', 'bronze', 0)
create_or_update_plan_limit('ci_project_subscriptions', 'silver', 0)
create_or_update_plan_limit('ci_project_subscriptions', 'gold', 0)
else
create_or_update_plan_limit('ci_project_subscriptions', 'default', 0)
end
end
end
# frozen_string_literal: true
class CreateCiSourcesProjects < ActiveRecord::Migration[6.0]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
def change
create_table :ci_sources_projects do |t|
t.bigint :pipeline_id, null: false
t.bigint :source_project_id, null: false
t.index [:source_project_id, :pipeline_id], unique: true
t.index :pipeline_id
end
end
end
# frozen_string_literal: true
class AddCiSourcesProjectPipelineForeignKey < ActiveRecord::Migration[6.0]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
def change
with_lock_retries do
add_foreign_key :ci_sources_projects, :ci_pipelines, column: :pipeline_id, on_delete: :cascade # rubocop:disable Migration/AddConcurrentForeignKey
end
end
end
# frozen_string_literal: true
class AddCiSourcesProjectSourceProjectForeignKey < ActiveRecord::Migration[6.0]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
def change
with_lock_retries do
add_foreign_key :ci_sources_projects, :projects, column: :source_project_id, on_delete: :cascade # rubocop:disable Migration/AddConcurrentForeignKey
end
end
end
......@@ -966,6 +966,13 @@ ActiveRecord::Schema.define(version: 2020_03_04_160823) do
t.index ["source_project_id"], name: "index_ci_sources_pipelines_on_source_project_id"
end
create_table "ci_sources_projects", force: :cascade do |t|
t.bigint "pipeline_id", null: false
t.bigint "source_project_id", null: false
t.index ["pipeline_id"], name: "index_ci_sources_projects_on_pipeline_id"
t.index ["source_project_id", "pipeline_id"], name: "index_ci_sources_projects_on_source_project_id_and_pipeline_id", unique: true
end
create_table "ci_stages", id: :serial, force: :cascade do |t|
t.integer "project_id"
t.integer "pipeline_id"
......@@ -3131,6 +3138,7 @@ ActiveRecord::Schema.define(version: 2020_03_04_160823) do
t.integer "ci_active_jobs", default: 0, null: false
t.integer "project_hooks", default: 0, null: false
t.integer "group_hooks", default: 0, null: false
t.integer "ci_project_subscriptions", default: 0, null: false
t.index ["plan_id"], name: "index_plan_limits_on_plan_id", unique: true
end
......@@ -4735,6 +4743,8 @@ ActiveRecord::Schema.define(version: 2020_03_04_160823) do
add_foreign_key "ci_sources_pipelines", "ci_pipelines", column: "source_pipeline_id", name: "fk_d4e29af7d7", on_delete: :cascade
add_foreign_key "ci_sources_pipelines", "projects", column: "source_project_id", name: "fk_acd9737679", on_delete: :cascade
add_foreign_key "ci_sources_pipelines", "projects", name: "fk_1e53c97c0a", on_delete: :cascade
add_foreign_key "ci_sources_projects", "ci_pipelines", column: "pipeline_id", on_delete: :cascade
add_foreign_key "ci_sources_projects", "projects", column: "source_project_id", on_delete: :cascade
add_foreign_key "ci_stages", "ci_pipelines", column: "pipeline_id", name: "fk_fb57e6cc56", on_delete: :cascade
add_foreign_key "ci_stages", "projects", name: "fk_2360681d1d", on_delete: :cascade
add_foreign_key "ci_subscriptions_projects", "projects", column: "downstream_project_id", on_delete: :cascade
......
......@@ -87,6 +87,28 @@ Plan.default.limits.update!(ci_active_jobs: 500)
NOTE: **Note:** Set the limit to `0` to disable it.
### Number of CI/CD subscriptions to a project
> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/9045) in GitLab 12.9.
The total number of subscriptions can be limited per project. This limit is
checked each time a new subscription is created.
If a new subscription would cause the total number of subscription to exceed the
limit, the subscription will be considered invalid.
- On GitLab.com different [limits are defined per plan](../user/gitlab_com/index.md#gitlab-cicd) and they affect all projects under that plan.
- On [GitLab Starter](https://about.gitlab.com/pricing/#self-managed) tier or higher self-hosted installations, this limit is defined for the `default` plan that affects all projects.
To set this limit on a self-hosted installation, run the following in the
[GitLab Rails console](https://docs.gitlab.com/omnibus/maintenance/#starting-a-rails-console-session):
```ruby
Plan.default.limits.update!(ci_project_subscriptions: 500)
```
NOTE: **Note:** Set the limit to `0` to disable it.
## Environment data on Deploy Boards
[Deploy Boards](../user/project/deploy_boards.md) load information from Kubernetes about
......
......@@ -227,3 +227,19 @@ Some features are not implemented yet. For example, support for environments.
- `only` and `except`
- `when` (only with `on_success`, `on_failure`, and `always` values)
- `extends`
## Trigger a pipeline when an upstream project is rebuilt
> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/9045) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.8.
You can trigger a pipeline in your project whenever a pipeline finishes for a new
tag in a different project:
1. Go to the project's **Settings > CI / CD** page, and expand the **Pipeline subscriptions** section.
1. Enter the path to the project you want to subscribe to.
1. Click subscribe.
Any pipelines that complete successfully for new tags in the subscribed project
will now trigger a pipeline on the current project's default branch. The maximum
number of upstream pipeline subscriptions is 2, for both the upstream and
downstream projects.
......@@ -39,6 +39,12 @@ limit values. It's recommended to create separate migration script files.
create_or_update_plan_limit('project_hooks', 'gold', 100)
```
NOTE: **Note:** Some plans exist only on GitLab.com. You can check if the
migration is running on GitLab.com with `Gitlab.com?`.
NOTE: **Note:** The test environment doesn't have any plans. You can check if a
migration is running in a test environment with `Rails.env.test?`
### Plan limits validation
#### Get current limit
......@@ -93,3 +99,20 @@ it_behaves_like 'includes Limitable concern' do
subject { build(:project_hook, project: create(:project)) }
end
```
### Subscription Plans
Self-hosted:
- `default` - Everyone
Hosted:
- `free` - Everyone
- `bronze`- Namespaces with a Bronze subscription
- `silver` - Namespaces with a Silver subscription
- `gold` - Namespaces with a Gold subscription
NOTE: **Note:** Hosted plans exist only on GitLab.com.
NOTE: **Note:** The test environment doesn't have any plans.
......@@ -4,17 +4,17 @@ class Projects::SubscriptionsController < Projects::ApplicationController
include ::Gitlab::Utils::StrongMemoize
before_action :authorize_admin_project!
before_action :authorize_read_upstream_project!, only: [:create]
before_action :feature_ci_project_subscriptions!
before_action :authorize_upstream_project!, only: [:create]
def create
subscription = project.upstream_project_subscriptions.create(upstream_project: upstream_project)
flash[:notice] = if subscription.persisted?
_('Subscription successfully created.')
else
_('This project path either does not exist or is private.')
end
if subscription.persisted?
flash[:notice] = _('Subscription successfully created.')
else
flash[:alert] = subscription.errors.full_messages
end
redirect_to project_settings_ci_cd_path(project)
end
......@@ -41,11 +41,14 @@ class Projects::SubscriptionsController < Projects::ApplicationController
project.upstream_project_subscriptions.find(params[:id])
end
def authorize_read_upstream_project!
render_404 unless can?(current_user, :read_project, upstream_project)
end
def feature_ci_project_subscriptions!
render_404 unless project.feature_available?(:ci_project_subscriptions)
end
def authorize_upstream_project!
return if can?(current_user, :developer_access, upstream_project)
flash[:warning] = _('This project path either does not exist or you do not have access.')
redirect_to project_settings_ci_cd_path(project)
end
end
# frozen_string_literal: true
module Ci
module Sources
class Project < ApplicationRecord
self.table_name = "ci_sources_projects"
belongs_to :pipeline, class_name: "Ci::Pipeline", optional: false
belongs_to :source_project, class_name: "Project", foreign_key: :source_project_id, optional: false
validates :pipeline_id, uniqueness: { scope: :source_project_id }
end
end
end
......@@ -3,8 +3,13 @@
module Ci
module Subscriptions
class Project < ApplicationRecord
include ::Limitable
self.table_name = "ci_subscriptions_projects"
self.limit_name = 'ci_project_subscriptions'
self.limit_scope = :upstream_project
belongs_to :downstream_project, class_name: '::Project', optional: false
belongs_to :upstream_project, class_name: '::Project', optional: false
......
......@@ -25,6 +25,8 @@ module EE
has_many :downstream_bridges, class_name: '::Ci::Bridge', foreign_key: :upstream_pipeline_id
has_many :security_scans, class_name: 'Security::Scan', through: :builds
has_one :source_project, class_name: 'Ci::Sources::Project', foreign_key: :pipeline_id
# Legacy way to fetch security reports based on job name. This has been replaced by the reports feature.
scope :with_legacy_security_reports, -> do
joins(:artifacts).where(ci_builds: { name: %w[sast dependency_scanning sast:container container_scanning dast] })
......@@ -65,9 +67,22 @@ module EE
::Ci::PipelineBridgeStatusWorker.perform_async(pipeline.id)
end
end
after_transition any => ::Ci::Pipeline.completed_statuses do |pipeline|
next unless pipeline.triggers_subscriptions?
pipeline.run_after_commit do
::Ci::TriggerDownstreamSubscriptionsWorker.perform_async(pipeline.id)
end
end
end
end
def triggers_subscriptions?
# Currently we trigger subscriptions only for tags.
tag? && project_has_subscriptions?
end
def retryable?
!merge_train_pipeline? && super
end
......@@ -145,6 +160,12 @@ module EE
private
def project_has_subscriptions?
return false unless ::Feature.enabled?(:ci_project_subscriptions, project)
project.downstream_projects.any?
end
def merge_train_ref?
::MergeRequest.merge_train_ref?(ref)
end
......
......@@ -95,6 +95,8 @@ module EE
has_many :downstream_project_subscriptions, class_name: 'Ci::Subscriptions::Project', foreign_key: :upstream_project_id, inverse_of: :upstream_project
has_many :downstream_projects, class_name: 'Project', through: :downstream_project_subscriptions, source: :downstream_project
has_many :sourced_pipelines, class_name: 'Ci::Sources::Project', foreign_key: :source_project_id
scope :with_shared_runners_limit_enabled, -> { with_shared_runners.non_public_only }
scope :mirror, -> { where(mirror: true) }
......
# frozen_string_literal: true
module Ci
class TriggerDownstreamSubscriptionService < ::BaseService
def execute(pipeline)
pipeline.project.downstream_projects.each do |downstream_project|
::Ci::CreatePipelineService.new(downstream_project, pipeline.user, ref: downstream_project.default_branch)
.execute(:pipeline) do |downstream_pipeline|
downstream_pipeline.build_source_project(source_project: pipeline.project)
end
end
end
end
end
......@@ -9,10 +9,10 @@
%p
= _("Set up pipeline subscriptions for this project.")
%p
- default_branch_docs = link_to(_("default branch"), help_page_path('user/project/repository/branches/index.md', anchor: 'default-branch'))
= _("Subscriptions allow successfully completed pipelines on the %{default_branch_docs} of the subscribed project to trigger a new pipeline on the default branch of this project.").html_safe % { default_branch_docs: default_branch_docs }
- default_branch_docs = link_to(_("default branch"), help_page_path('user/project/repository/branches', anchor: 'default-branch'))
= _("A subscription will trigger a new pipeline on the default branch of this project when a pipeline successfully completes for a new tag on the %{default_branch_docs} of the subscribed project.").html_safe % { default_branch_docs: default_branch_docs }
%p
= _("There is a limit of 100 subscriptions from or to a project.")
= _("There is a limit of %{ci_project_subscriptions_limit} subscriptions from or to a project.").html_safe % { ci_project_subscriptions_limit: @project.actual_limits.ci_project_subscriptions }
.settings-content
= render 'projects/settings/subscriptions/index'
......@@ -402,6 +402,13 @@
:resource_boundary: :unknown
:weight: 1
:idempotent:
- :name: pipeline_default:ci_trigger_downstream_subscriptions
:feature_category: :continuous_integration
:has_external_dependencies:
:urgency: :default
:resource_boundary: :cpu
:weight: 3
:idempotent:
- :name: security_scans:store_security_reports
:feature_category: :static_application_security_testing
:has_external_dependencies:
......
# frozen_string_literal: true
module Ci
class TriggerDownstreamSubscriptionsWorker # rubocop:disable Scalability/IdempotentWorker
include ::ApplicationWorker
include ::PipelineQueue
worker_resource_boundary :cpu
def perform(pipeline_id)
::Ci::Pipeline.find_by_id(pipeline_id).try do |pipeline|
::Ci::TriggerDownstreamSubscriptionService
.new(pipeline.project, pipeline.user)
.execute(pipeline)
end
end
end
end
......@@ -16,10 +16,14 @@ describe Projects::SubscriptionsController do
let(:upstream_project) { create(:project, :public) }
before do
plan_limits = create(:plan_limits, :default_plan)
plan_limits.update(ci_project_subscriptions: 2)
end
context 'when user is authorized' do
before do
project.add_maintainer(user)
upstream_project.add_developer(user)
end
context 'when feature is available' do
......@@ -27,29 +31,77 @@ describe Projects::SubscriptionsController do
stub_licensed_features(ci_project_subscriptions: true)
end
context 'when project is public' do
it 'creates a new subscription' do
expect { post_create }.to change { project.upstream_project_subscriptions.count }.from(0).to(1)
context 'when user is developer in upstream project' do
before do
upstream_project.add_developer(user)
end
it 'sets the flash' do
post_create
context 'when project is public' do
context 'when subscription count is below the limit' do
it 'creates a new subscription' do
expect { post_create }.to change { project.upstream_project_subscriptions.count }.from(0).to(1)
end
expect(response).to set_flash[:notice].to('Subscription successfully created.')
end
it 'sets the flash' do
post_create
it 'redirects to ci_cd settings' do
post_create
expect(response).to set_flash[:notice].to('Subscription successfully created.')
end
expect(response).to redirect_to project_settings_ci_cd_path(project)
it 'redirects to ci_cd settings' do
post_create
expect(response).to redirect_to project_settings_ci_cd_path(project)
end
end
context 'when subscription count is above the limit' do
before do
create_list(:ci_subscriptions_project, 2, upstream_project: upstream_project)
end
it 'does not create a new subscription' do
expect { post_create }.not_to change { project.upstream_project_subscriptions.count }.from(0)
end
it 'sets the flash' do
post_create
expect(response).to set_flash[:alert].to(['Maximum number of ci project subscriptions (2) exceeded'])
end
it 'redirects to ci_cd settings' do
post_create
expect(response).to redirect_to project_settings_ci_cd_path(project)
end
end
end
end
context 'when project is not public' do
before do
upstream_project.update(visibility_level: Gitlab::VisibilityLevel::PRIVATE)
context 'when project is not public' do
before do
upstream_project.update(visibility_level: Gitlab::VisibilityLevel::PRIVATE)
end
it 'does not create a new subscription' do
expect { post_create }.not_to change { project.upstream_project_subscriptions.count }.from(0)
end
it 'sets the flash' do
post_create
expect(response).to set_flash[:alert].to(['Upstream project needs to be public'])
end
it 'redirects to ci_cd settings' do
post_create
expect(response).to redirect_to project_settings_ci_cd_path(project)
end
end
end
context 'when user is not developer in upstream project' do
it 'does not create a new subscription' do
expect { post_create }.not_to change { project.upstream_project_subscriptions.count }.from(0)
end
......@@ -57,7 +109,7 @@ describe Projects::SubscriptionsController do
it 'sets the flash' do
post_create
expect(response).to set_flash[:notice].to('This project path either does not exist or is private.')
expect(response).to set_flash[:warning].to('This project path either does not exist or you do not have access.')
end
it 'redirects to ci_cd settings' do
......
# frozen_string_literal: true
FactoryBot.define do
factory :ci_sources_project, class: 'Ci::Sources::Project' do
pipeline factory: :ci_pipeline
source_project factory: :project
end
end
......@@ -385,6 +385,44 @@ describe Ci::Pipeline do
pipeline.cancel!
end
end
context 'when pipeline project has downstream subscriptions' do
let(:pipeline) { create(:ci_empty_pipeline, project: create(:project, :public)) }
before do
pipeline.project.downstream_projects << create(:project)
end
context 'when pipeline runs on a tag' do
before do
pipeline.update(tag: true)
end
context 'when feature is not available' do
before do
stub_feature_flags(ci_project_subscriptions: false)
end
it 'does not schedule the trigger downstream subscriptions worker' do
expect(::Ci::TriggerDownstreamSubscriptionsWorker).not_to receive(:perform_async)
pipeline.succeed!
end
end
context 'when feature is available' do
before do
stub_feature_flags(ci_project_subscriptions: true)
end
it 'schedules the trigger downstream subscriptions worker' do
expect(::Ci::TriggerDownstreamSubscriptionsWorker).to receive(:perform_async)
pipeline.succeed!
end
end
end
end
end
describe '#latest_merge_request_pipeline?' do
......
# frozen_string_literal: true
require 'spec_helper'
describe Ci::Sources::Project do
describe 'Relations' do
it { is_expected.to belong_to(:pipeline).required }
it { is_expected.to belong_to(:source_project).required }
end
describe 'Validations' do
let!(:project_source) { create(:ci_sources_project) }
it { is_expected.to validate_uniqueness_of(:pipeline_id).scoped_to(:source_project_id) }
end
end
......@@ -3,18 +3,24 @@
require 'spec_helper'
describe Ci::Subscriptions::Project do
let!(:subscription) { create(:ci_subscriptions_project) }
let(:upstream_project) { create(:project, :public) }
describe 'Relations' do
it { is_expected.to belong_to(:downstream_project).required }
it { is_expected.to belong_to(:upstream_project).required }
end
it_behaves_like 'includes Limitable concern' do
subject { build(:ci_subscriptions_project, upstream_project: upstream_project) }
end
describe 'Validations' do
let!(:subscription) { create(:ci_subscriptions_project, upstream_project: upstream_project) }
it { is_expected.to validate_uniqueness_of(:upstream_project_id).scoped_to(:downstream_project_id) }
it 'validates that upstream project is public' do
subscription.upstream_project.update(visibility_level: Gitlab::VisibilityLevel::PRIVATE)
upstream_project.update(visibility_level: Gitlab::VisibilityLevel::PRIVATE)
expect(subscription).not_to be_valid
end
......
# frozen_string_literal: true
require 'spec_helper'
describe Ci::TriggerDownstreamSubscriptionService do
describe '#execute' do
subject(:execute) { described_class.new(pipeline.project, pipeline.user).execute(pipeline) }
let(:upstream_project) { create(:project, :public) }
let(:pipeline) { create(:ci_pipeline, project: upstream_project, user: create(:user)) }
before do
stub_ci_pipeline_yaml_file(YAML.dump(job_name: { script: 'echo 1' }))
end
context 'when pipeline project has downstream projects' do
before do
downstream_project = create(:project, :repository, upstream_projects: [upstream_project])
downstream_project.add_developer(pipeline.user)
end
it 'creates a pipeline' do
expect { execute }.to change { ::Ci::Pipeline.count }.from(1).to(2)
end
it 'associates the downstream pipeline with the upstream project' do
expect { execute }.to change { pipeline.project.sourced_pipelines.count }.from(0).to(1)
end
end
context 'when pipeline project does not have downstream projects' do
it 'does not call the create pipeline service' do
expect(::Ci::CreatePipelineService).not_to receive(:new)
execute
end
end
end
end
......@@ -7,7 +7,7 @@ RSpec.shared_examples 'includes Limitable concern' do
it { is_expected.to be_a(Limitable) }
context 'without plan limits configured' do
it 'can create new group hooks' do
it 'can create new models' do
expect { subject.save }.to change { described_class.count }
end
end
......
# frozen_string_literal: true
require 'spec_helper'
describe Ci::TriggerDownstreamSubscriptionsWorker do
describe '#perform' do
subject(:perform) { described_class.new.perform(pipeline_id) }
context 'when pipeline exists' do
let(:pipeline_id) { create(:ci_pipeline, user: create(:user)).id }
it 'calls the trigger downstream pipeline service' do
expect(::Ci::TriggerDownstreamSubscriptionService).to receive_message_chain(:new, :execute)
perform
end
end
context 'when pipeline does not exist' do
let(:pipeline_id) { 1234 }
it 'does nothing' do
expect(::Ci::TriggerDownstreamSubscriptionService).not_to receive(:new)
perform
end
end
end
end
......@@ -861,6 +861,9 @@ msgstr ""
msgid "A secure token that identifies an external storage request."
msgstr ""
msgid "A subscription will trigger a new pipeline on the default branch of this project when a pipeline successfully completes for a new tag on the %{default_branch_docs} of the subscribed project."
msgstr ""
msgid "A user with write access to the source branch selected this option"
msgstr ""
......@@ -18897,9 +18900,6 @@ msgstr ""
msgid "Subscriptions"
msgstr ""
msgid "Subscriptions allow successfully completed pipelines on the %{default_branch_docs} of the subscribed project to trigger a new pipeline on the default branch of this project."
msgstr ""
msgid "Subtracted"
msgstr ""
......@@ -19738,7 +19738,7 @@ msgstr ""
msgid "There are no unstaged changes"
msgstr ""
msgid "There is a limit of 100 subscriptions from or to a project."
msgid "There is a limit of %{ci_project_subscriptions_limit} subscriptions from or to a project."
msgstr ""
msgid "There is already a repository with that name on disk"
......@@ -20203,7 +20203,7 @@ msgstr ""
msgid "This project is archived and cannot be commented on."
msgstr ""
msgid "This project path either does not exist or is private."
msgid "This project path either does not exist or you do not have access."
msgstr ""
msgid "This project will be removed on %{date}"
......
......@@ -197,6 +197,7 @@ ci_pipelines:
- source_bridge
- source_job
- sourced_pipelines
- source_project
- triggered_by_pipeline
- triggered_pipelines
- child_pipelines
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment