Make CreatePipelineService to run in dry-run mode

This allows a pipeline creation to fully run while skipping
the persistence steps. In the end it returns a non-persisted
pipeline with all its errors and warnings.

This feature is useful to allow CI Lint to use the actual
pipeline creation and display all errors rather than using
only YamlProcessor.

app/controllers/projects/ci/lints_controller.rb

@@ -8,17 +8,30 @@ class Projects::Ci::LintsController < Projects::ApplicationController
 
   def create
     @content = params[:content]
-    result   = Gitlab::Ci::YamlProcessor.new_with_validation_errors(@content, yaml_processor_options)
-
-    @status = result.valid?
-    @errors = result.errors
-    @warnings = result.warnings
-
-    if result.valid?
-      @config_processor = result.config
-      @stages = @config_processor.stages
-      @builds = @config_processor.builds
-      @jobs = @config_processor.jobs
+    @dry_run = params[:dry_run]
+
+    if @dry_run && Gitlab::Ci::Features.lint_creates_pipeline_with_dry_run?(@project)
+      pipeline = Ci::CreatePipelineService
+        .new(@project, current_user, ref: @project.default_branch)
+        .execute(:push, dry_run: true, content: @content)
+
+      @status = pipeline.error_messages.empty?
+      @stages = pipeline.stages
+      @errors = pipeline.error_messages.map(&:content)
+      @warnings = pipeline.warning_messages.map(&:content)
+    else
+      result = Gitlab::Ci::YamlProcessor.new_with_validation_errors(@content, yaml_processor_options)
+
+      @status = result.valid?
+      @errors = result.errors
+      @warnings = result.warnings
+
+      if result.valid?
+        @config_processor = result.config
+        @stages = @config_processor.stages
+        @builds = @config_processor.builds
+        @jobs = @config_processor.jobs
+      end
     end
 
     render :show

app/views/projects/ci/lints/_create.html.haml

@@ -11,33 +11,54 @@
           %th= _("Parameter")
           %th= _("Value")
       %tbody
-        - @stages.each do |stage|
-          - @builds.select { |build| build[:stage] == stage }.each do |build|
-            - job = @jobs[build[:name].to_sym]
-            %tr
-              %td #{stage.capitalize} Job - #{build[:name]}
-              %td
-                %pre= job[:before_script].to_a.join('\n')
-                %pre= job[:script].to_a.join('\n')
-                %pre= job[:after_script].to_a.join('\n')
+        - if @dry_run
+          - @stages.each do |stage|
+            - stage.statuses.each do |job|
+              %tr
+                %td #{stage.name.capitalize} Job - #{job.name}
+                %td
+                  %pre= job.options[:before_script].to_a.join('\n')
+                  %pre= job.options[:script].to_a.join('\n')
+                  %pre= job.options[:after_script].to_a.join('\n')
+                  %br
+                  %b= _("Tag list:")
+                  = job.tag_list.to_a.join(", ") if job.is_a?(Ci::Build)
+                  %br
+                  %b= _("Environment:")
+                  = job.options.dig(:environment, :name)
+                  %br
+                  %b= _("When:")
+                  = job.when
+                  - if job.allow_failure
+                    %b= _("Allowed to fail")
 
-                %br
-                %b= _("Tag list:")
-                = build[:tag_list].to_a.join(", ")
-                %br
-                %b= _("Only policy:")
-                = job[:only].to_a.join(", ")
-                %br
-                %b= _("Except policy:")
-                = job[:except].to_a.join(", ")
-                %br
-                %b= _("Environment:")
-                = build[:environment]
-                %br
-                %b= _("When:")
-                = build[:when]
-                - if build[:allow_failure]
-                  %b= _("Allowed to fail")
+        - else
+          - @stages.each do |stage|
+            - @builds.select { |build| build[:stage] == stage }.each do |build|
+              - job = @jobs[build[:name].to_sym]
+              %tr
+                %td #{stage.capitalize} Job - #{build[:name]}
+                %td
+                  %pre= job[:before_script].to_a.join('\n')
+                  %pre= job[:script].to_a.join('\n')
+                  %pre= job[:after_script].to_a.join('\n')
+                  %br
+                  %b= _("Tag list:")
+                  = build[:tag_list].to_a.join(", ")
+                  %br
+                  %b= _("Only policy:")
+                  = job[:only].to_a.join(", ")
+                  %br
+                  %b= _("Except policy:")
+                  = job[:except].to_a.join(", ")
+                  %br
+                  %b= _("Environment:")
+                  = build[:environment]
+                  %br
+                  %b= _("When:")
+                  = build[:when]
+                  - if build[:allow_failure]
+                    %b= _("Allowed to fail")
 
 - else
   .bs-callout.bs-callout-danger

app/views/projects/ci/lints/show.html.haml

@@ -3,7 +3,7 @@
 - content_for :library_javascripts do
   = page_specific_javascript_tag('lib/ace.js')
 
-%h2.pt-3.pb-3= _("Check your .gitlab-ci.yml")
+%h2.pt-3.pb-3= _("Validate your GitLab CI configuration")
 
 .project-ci-linter
   = form_tag project_ci_lint_path(@project), method: :post do
@@ -17,7 +17,11 @@
       .col-sm-12
         .float-left.gl-mt-3
           = submit_tag(_('Validate'), class: 'btn btn-success submit-yml')
-        .float-right.gl-mt-3
+          - if Gitlab::Ci::Features.lint_creates_pipeline_with_dry_run?(@project)
+            = check_box_tag(:dry_run, 'true', params[:dry_run])
+            = label_tag(:dry_run, _('Simulate a pipeline created for the default branch'))
+            = link_to icon('question-circle'), help_page_path('ci/lint', anchor: 'pipeline-simulation'), target: '_blank', rel: 'noopener noreferrer'
+        .float-right.prepend-top-10
           = button_tag(_('Clear'), type: 'button', class: 'btn btn-default clear-yml')
 
   .row.prepend-top-20

changelogs/unreleased/use-dry-run-on-ci-lint.yml

+---
+title: Allow user to simulate pipeline creation via CI Lint and go beyond syntax checks
+merge_request: 37828
+author:
+type: added

doc/ci/lint.md

+# CI Lint
+
+If you want to test the validity of your GitLab CI/CD configuration before committing
+the changes, you can use the CI Lint tool. This tool checks for syntax and logical
+errors by default, and can simulate pipeline creation to try to find more complicated
+issues as well.
+
+To access the CI Lint tool, navigate to **CI/CD > Pipelines** or **CI/CD > Jobs**
+in your project and click **CI lint**.
+
+## Validate basic logic and syntax
+
+By default, the CI lint checks the syntax of your CI YAML configuration and also runs
+some basic logical validations.
+
+To use the CI lint, paste a complete CI configuration (`.gitlab-ci.yml` for example)
+into the text box and click **Validate**:
+
+![CI Lint](img/ci_lint.png)
+
+## Pipeline simulation
+
+> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/229794) in GitLab 13.3.
+
+Not all pipeline configuration issues can be found by the [basic CI lint validation](#validate-basic-logic-and-syntax).
+You can simulate the creation of a pipeline for deeper validation that can discover
+more complicated issues.
+
+To validate the configuration by running a pipeline simulation:
+
+1. Paste the GitLab CI configuration to verify into the text box.
+1. Click the **Simulate pipeline creation for the default branch** checkbox.
+1. Click **Validate**.
+
+![Dry run](img/ci_lint_dry_run.png)
+
+### Pipeline simulation limitations
+
+Simulations run as `git push` events against the default branch. You must have
+[permissions](../user/permissions.md#project-members-permissions) to create pipelines
+on this branch to validate with a simulation.

doc/ci/quick_start/README.md

@@ -114,9 +114,7 @@ Jobs are used to create jobs, which are then picked by
 What is important is that each job is run independently from each other.
 
 If you want to check whether the `.gitlab-ci.yml` of your project is valid, there is a
-Lint tool under the page `/-/ci/lint` of your project namespace. You can also find
-a "CI Lint" button to go to this page under **CI/CD ➔ Pipelines** and
-**Pipelines ➔ Jobs** in your project.
+[CI Lint tool](../lint.md) available in every project.
 
 For more information and a complete `.gitlab-ci.yml` syntax, please read
 [the reference documentation on `.gitlab-ci.yml`](../yaml/README.md).

lib/gitlab/ci/features.rb

@@ -76,6 +76,10 @@ module Gitlab
         ::Feature.enabled?(:ci_job_entry_matches_all_keys)
       end
 
+      def self.lint_creates_pipeline_with_dry_run?(project)
+        ::Feature.enabled?(:ci_lint_creates_pipeline_with_dry_run, project)
+      end
+
       def self.reset_ci_minutes_for_all_namespaces?
         ::Feature.enabled?(:reset_ci_minutes_for_all_namespaces, default_enabled: false)
       end

lib/gitlab/ci/pipeline/metrics.rb

@@ -45,6 +45,15 @@ module Gitlab
             Gitlab::Metrics.counter(name, comment)
           end
         end
+
+        def pipelines_created_counter
+          strong_memoize(:pipelines_created_count) do
+            name = :pipelines_created_total
+            comment = 'Counter of pipelines created'
+
+            Gitlab::Metrics.counter(name, comment)
+          end
+        end
       end
     end
   end

locale/gitlab.pot

@@ -4550,9 +4550,6 @@ msgstr ""
 msgid "Check the %{docs_link_start}documentation%{docs_link_end}."
 msgstr ""
 
-msgid "Check your .gitlab-ci.yml"
-msgstr ""
-
 msgid "Check your Docker images for known vulnerabilities."
 msgstr ""
 
@@ -22459,6 +22456,9 @@ msgstr ""
 msgid "Similar issues"
 msgstr ""
 
+msgid "Simulate a pipeline created for the default branch"
+msgstr ""
+
 msgid "Single or combined queries"
 msgstr ""
 
@@ -26754,6 +26754,9 @@ msgstr ""
 msgid "Validate"
 msgstr ""
 
+msgid "Validate your GitLab CI configuration"
+msgstr ""
+
 msgid "Validate your GitLab CI configuration file"
 msgstr ""
 

spec/controllers/projects/ci/lints_controller_spec.rb

@@ -45,6 +45,9 @@ RSpec.describe Projects::Ci::LintsController do
   end
 
   describe 'POST #create' do
+    subject { post :create, params: params }
+
+    let(:params) { { namespace_id: project.namespace, project_id: project, content: content } }
     let(:remote_file_path) { 'https://gitlab.com/gitlab-org/gitlab-foss/blob/1234/.gitlab-ci-1.yml' }
 
     let(:remote_file_content) do
@@ -72,18 +75,62 @@ RSpec.describe Projects::Ci::LintsController do
       before do
         stub_full_request(remote_file_path).to_return(body: remote_file_content)
         project.add_developer(user)
+      end
 
-        post :create, params: { namespace_id: project.namespace, project_id: project, content: content }
+      shared_examples 'returns a successful validation' do
+        it 'returns successfully' do
+          subject
+          expect(response).to be_successful
+        end
+
+        it 'render show page' do
+          subject
+          expect(response).to render_template :show
+        end
+
+        it 'retrieves project' do
+          subject
+          expect(assigns(:project)).to eq(project)
+        end
       end
 
-      it { expect(response).to be_successful }
+      context 'using legacy validation (YamlProcessor)' do
+        it_behaves_like 'returns a successful validation'
 
-      it 'render show page' do
-        expect(response).to render_template :show
+        it 'runs validations through YamlProcessor' do
+          expect(Gitlab::Ci::YamlProcessor).to receive(:new_with_validation_errors).and_call_original
+
+          subject
+        end
       end
 
-      it 'retrieves project' do
-        expect(assigns(:project)).to eq(project)
+      context 'using dry_run mode' do
+        subject { post :create, params: params.merge(dry_run: 'true') }
+
+        it_behaves_like 'returns a successful validation'
+
+        it 'runs validations through Ci::CreatePipelineService' do
+          expect(Ci::CreatePipelineService)
+            .to receive(:new)
+            .with(project, user, ref: 'master')
+            .and_call_original
+
+          subject
+        end
+
+        context 'when dry_run feature flag is disabled' do
+          before do
+            stub_feature_flags(ci_lint_creates_pipeline_with_dry_run: false)
+          end
+
+          it_behaves_like 'returns a successful validation'
+
+          it 'runs validations through YamlProcessor' do
+            expect(Gitlab::Ci::YamlProcessor).to receive(:new_with_validation_errors).and_call_original
+
+            subject
+          end
+        end
       end
     end
 
@@ -98,13 +145,23 @@ RSpec.describe Projects::Ci::LintsController do
 
       before do
         project.add_developer(user)
-
-        post :create, params: { namespace_id: project.namespace, project_id: project, content: content }
       end
 
       it 'assigns errors' do
+        subject
+
         expect(assigns[:errors]).to eq(['root config contains unknown keys: rubocop'])
       end
+
+      context 'with dry_run mode' do
+        subject { post :create, params: params.merge(dry_run: 'true') }
+
+        it 'assigns errors' do
+          subject
+
+          expect(assigns[:errors]).to eq(['root config contains unknown keys: rubocop'])
+        end
+      end
     end
 
     context 'without enough privileges' do

spec/features/projects/ci/lint_spec.rb

@@ -21,32 +21,48 @@ RSpec.describe 'CI Lint', :js do
   end
 
   describe 'YAML parsing' do
-    before do
-      click_on 'Validate'
-    end
+    shared_examples 'validates the YAML' do
+      before do
+        click_on 'Validate'
+      end
 
-    context 'YAML is correct' do
-      let(:yaml_content) do
-        File.read(Rails.root.join('spec/support/gitlab_stubs/gitlab_ci.yml'))
+      context 'YAML is correct' do
+        let(:yaml_content) do
+          File.read(Rails.root.join('spec/support/gitlab_stubs/gitlab_ci.yml'))
+        end
+
+        it 'parses Yaml and displays the jobs' do
+          expect(page).to have_content('Status: syntax is correct')
+
+          within "table" do
+            aggregate_failures do
+              expect(page).to have_content('Job - rspec')
+              expect(page).to have_content('Job - spinach')
+              expect(page).to have_content('Deploy Job - staging')
+              expect(page).to have_content('Deploy Job - production')
+            end
+          end
+        end
       end
 
-      it 'parses Yaml' do
-        within "table" do
-          expect(page).to have_content('Job - rspec')
-          expect(page).to have_content('Job - spinach')
-          expect(page).to have_content('Deploy Job - staging')
-          expect(page).to have_content('Deploy Job - production')
+      context 'YAML is incorrect' do
+        let(:yaml_content) { 'value: cannot have :' }
+
+        it 'displays information about an error' do
+          expect(page).to have_content('Status: syntax is incorrect')
+          expect(page).to have_selector('.ace_content', text: yaml_content)
         end
       end
     end
 
-    context 'YAML is incorrect' do
-      let(:yaml_content) { 'value: cannot have :' }
+    it_behaves_like 'validates the YAML'
 
-      it 'displays information about an error' do
-        expect(page).to have_content('Status: syntax is incorrect')
-        expect(page).to have_selector('.ace_content', text: yaml_content)
+    context 'when Dry Run is checked' do
+      before do
+        check 'Simulate a pipeline created for the default branch'
       end
+
+      it_behaves_like 'validates the YAML'
     end
 
     describe 'YAML revalidate' do