Merge branch 'mc/feature/project-ci-lint-api' into 'master'

Add project scope CI lint API endpoint

See merge request gitlab-org/gitlab!42998

app/models/ci/pipeline.rb

@@ -27,6 +27,13 @@ module Ci
     sha_attribute :source_sha
     sha_attribute :target_sha
 
+    # Ci::CreatePipelineService returns Ci::Pipeline so this is the only place
+    # where we can pass additional information from the service. This accessor
+    # is used for storing the processed CI YAML contents for linting purposes.
+    # There is an open issue to address this:
+    # https://gitlab.com/gitlab-org/gitlab/-/issues/259010
+    attr_accessor :merged_yaml
+
     belongs_to :project, inverse_of: :all_pipelines
     belongs_to :user
     belongs_to :auto_canceled_by, class_name: 'Ci::Pipeline'

changelogs/unreleased/mc-feature-project-ci-lint-api.yml

+---
+title: Add project scoped CI lint API endpoint.
+merge_request: 42998
+author:
+type: added

doc/api/lint.md

@@ -94,3 +94,51 @@ Example response:
   "merged_config": "---\n:another_test:\n  :stage: test\n  :script: echo 2\n:test:\n  :stage: test\n  :script: echo 1\n"
 }
 ```
+
+## Validate a project's CI configuration
+
+> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/231352) in GitLab 13.5.
+
+Checks if a project's latest (`HEAD` of the project's default branch)
+`.gitlab-ci.yml` configuration is valid. This endpoint uses all namespace
+specific data available, including variables, local includes, and so on.
+
+```plaintext
+GET /projects/:id/ci/lint
+```
+
+| Attribute  | Type    | Required | Description |
+| ---------- | ------- | -------- | -------- |
+| `dry_run`  | boolean | no       | Run pipeline creation simulation, or only do static check. |
+
+Example request:
+
+```shell
+curl "https://gitlab.example.com/api/v4/projects/:id/ci/lint"
+```
+
+Example responses:
+
+- Valid config:
+
+```json
+{
+  "valid": true,
+  "merged_yaml": "---\n:test_job:\n  :script: echo 1\n",
+  "errors": [],
+  "warnings": []
+}
+```
+
+- Invalid config:
+
+```json
+{
+  "valid": false,
+  "merged_yaml": "---\n:test_job:\n  :script: echo 1\n",
+  "errors": [
+    "jobs config should contain at least one visible job"
+  ],
+  "warnings": []
+}
+```

lib/api/entities/ci/lint/result.rb

+# frozen_string_literal: true
+
+module API
+  module Entities
+    module Ci
+      module Lint
+        class Result < Grape::Entity
+          expose :valid?, as: :valid
+          expose :errors
+          expose :warnings
+          expose :merged_yaml
+        end
+      end
+    end
+  end
+end

lib/api/lint.rb

@@ -25,5 +25,24 @@ module API
         end
       end
     end
+
+    resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
+      desc 'Validation of .gitlab-ci.yml content' do
+        detail 'This feature was introduced in GitLab 13.5.'
+      end
+      params do
+        optional :dry_run, type: Boolean, default: false, desc: 'Run pipeline creation simulation, or only do static check.'
+      end
+      get ':id/ci/lint' do
+        authorize! :download_code, user_project
+
+        content = user_project.repository.gitlab_ci_yml_for(user_project.commit.id, user_project.ci_config_path_or_default)
+        result = Gitlab::Ci::Lint
+          .new(project: user_project, current_user: current_user)
+          .validate(content, dry_run: params[:dry_run])
+
+        present result, with: Entities::Ci::Lint::Result, current_user: current_user
+      end
+    end
   end
 end

lib/gitlab/ci/lint.rb

@@ -4,10 +4,11 @@ module Gitlab
   module Ci
     class Lint
       class Result
-        attr_reader :jobs, :errors, :warnings
+        attr_reader :jobs, :merged_yaml, :errors, :warnings
 
-        def initialize(jobs:, errors:, warnings:)
+        def initialize(jobs:, merged_yaml:, errors:, warnings:)
           @jobs = jobs
+          @merged_yaml = merged_yaml
           @errors = errors
           @warnings = warnings
         end
@@ -39,6 +40,7 @@ module Gitlab
 
         Result.new(
           jobs: dry_run_convert_to_jobs(pipeline.stages),
+          merged_yaml: pipeline.merged_yaml,
           errors: pipeline.error_messages.map(&:content),
           warnings: pipeline.warning_messages(limit: ::Gitlab::Ci::Warnings::MAX_LIMIT).map(&:content)
         )
@@ -54,6 +56,7 @@ module Gitlab
 
         Result.new(
           jobs: static_validation_convert_to_jobs(result),
+          merged_yaml: result.merged_yaml,
           errors: result.errors,
           warnings: result.warnings.take(::Gitlab::Ci::Warnings::MAX_LIMIT) # rubocop: disable CodeReuse/ActiveRecord
         )

lib/gitlab/ci/pipeline/chain/config/process.rb

@@ -28,6 +28,8 @@ module Gitlab
                 error(result.errors.first, config_error: true)
               end
 
+              @pipeline.merged_yaml = result.merged_yaml
+
             rescue => ex
               Gitlab::ErrorTracking.track_exception(ex,
                 project_id: project.id,

spec/lib/gitlab/ci/lint_spec.rb

@@ -3,7 +3,7 @@
 require 'spec_helper'
 
 RSpec.describe Gitlab::Ci::Lint do
-  let_it_be(:project) { create(:project, :repository) }
+  let(:project) { create(:project, :repository) }
   let_it_be(:user) { create(:user) }
 
   let(:lint) { described_class.new(project: project, current_user: user) }
@@ -61,6 +61,43 @@ RSpec.describe Gitlab::Ci::Lint do
       end
     end
 
+    shared_examples 'sets merged yaml' do
+      let(:content) do
+        <<~YAML
+        :include:
+          :local: another-gitlab-ci.yml
+        :test_job:
+          :stage: test
+          :script: echo
+        YAML
+      end
+
+      let(:included_content) do
+        <<~YAML
+        :another_job:
+          :script: echo
+        YAML
+      end
+
+      before do
+        project.repository.create_file(
+          project.creator,
+          'another-gitlab-ci.yml',
+          included_content,
+          message: 'Automatically created another-gitlab-ci.yml',
+          branch_name: 'master'
+        )
+      end
+
+      it 'sets merged_config' do
+        root_config = YAML.safe_load(content, [Symbol])
+        included_config = YAML.safe_load(included_content, [Symbol])
+        expected_config = included_config.merge(root_config).except(:include)
+
+        expect(subject.merged_yaml).to eq(expected_config.to_yaml)
+      end
+    end
+
     shared_examples 'content with errors and warnings' do
       context 'when content has errors' do
         let(:content) do
@@ -173,6 +210,8 @@ RSpec.describe Gitlab::Ci::Lint do
           end
         end
 
+        it_behaves_like 'sets merged yaml'
+
         include_context 'advanced validations' do
           it 'does not catch advanced logical errors' do
             expect(subject).to be_valid
@@ -203,6 +242,8 @@ RSpec.describe Gitlab::Ci::Lint do
           end
         end
 
+        it_behaves_like 'sets merged yaml'
+
         include_context 'advanced validations' do
           it 'runs advanced logical validations' do
             expect(subject).not_to be_valid

spec/requests/api/lint_spec.rb

@@ -75,4 +75,115 @@ RSpec.describe API::Lint do
       end
     end
   end
+
+  describe 'GET /projects/:id/ci/lint' do
+    subject(:ci_lint) { get api("/projects/#{project.id}/ci/lint", api_user), params: { dry_run: dry_run } }
+
+    let_it_be(:api_user) { create(:user) }
+    let(:project) { create(:project, :repository) }
+    let(:dry_run) { nil }
+
+    RSpec.shared_examples 'valid config' do
+      it 'passes validation' do
+        ci_lint
+
+        included_config = YAML.safe_load(included_content, [Symbol])
+        root_config = YAML.safe_load(yaml_content, [Symbol])
+        expected_yaml = included_config.merge(root_config).except(:include).to_yaml
+
+        expect(response).to have_gitlab_http_status(:ok)
+        expect(json_response).to be_an Hash
+        expect(json_response['merged_yaml']).to eq(expected_yaml)
+        expect(json_response['valid']).to eq(true)
+        expect(json_response['errors']).to eq([])
+      end
+    end
+
+    RSpec.shared_examples 'invalid config' do
+      it 'responds with errors about invalid configuration' do
+        ci_lint
+
+        expect(response).to have_gitlab_http_status(:ok)
+        expect(json_response['merged_yaml']).to eq(yaml_content)
+        expect(json_response['valid']).to eq(false)
+        expect(json_response['errors']).to eq(['jobs config should contain at least one visible job'])
+      end
+    end
+
+    context 'when unauthenticated' do
+      it 'returns authentication error' do
+        ci_lint
+
+        expect(response).to have_gitlab_http_status(:not_found)
+      end
+    end
+
+    context 'when authenticated as project member' do
+      before do
+        project.add_developer(api_user)
+      end
+
+      context 'with valid .gitlab-ci.yml content' do
+        let(:yaml_content) do
+          { include: { local: 'another-gitlab-ci.yml' }, test: { stage: 'test', script: 'echo 1' } }.to_yaml
+        end
+
+        let(:included_content) do
+          { another_test: { stage: 'test', script: 'echo 1' } }.to_yaml
+        end
+
+        before do
+          project.repository.create_file(
+            project.creator,
+            '.gitlab-ci.yml',
+            yaml_content,
+            message: 'Automatically created .gitlab-ci.yml',
+            branch_name: 'master'
+          )
+
+          project.repository.create_file(
+            project.creator,
+            'another-gitlab-ci.yml',
+            included_content,
+            message: 'Automatically created another-gitlab-ci.yml',
+            branch_name: 'master'
+          )
+        end
+
+        context 'when running as dry run' do
+          let(:dry_run) { true }
+
+          it_behaves_like 'valid config'
+        end
+
+        context 'when running static validation' do
+          let(:dry_run) { false }
+
+          it_behaves_like 'valid config'
+        end
+      end
+
+      context 'with invalid .gitlab-ci.yml content' do
+        let(:yaml_content) do
+          { image: 'ruby:2.7', services: ['postgres'] }.to_yaml
+        end
+
+        before do
+          stub_ci_pipeline_yaml_file(yaml_content)
+        end
+
+        context 'when running as dry run' do
+          let(:dry_run) { true }
+
+          it_behaves_like 'invalid config'
+        end
+
+        context 'when running static validation' do
+          let(:dry_run) { false }
+
+          it_behaves_like 'invalid config'
+        end
+      end
+    end
+  end
 end