Merge branch '31197-webhook-payload-to-pipelines' into 'master'

Implement passing trigger payload into pipeline variable [RUN ALL RSPEC] [RUN AS-IF-FOSS]

See merge request gitlab-org/gitlab!53837

app/services/ci/pipeline_trigger_service.rb

@@ -17,6 +17,9 @@ module Ci
 
     private
 
+    PAYLOAD_VARIABLE_KEY = 'TRIGGER_PAYLOAD'
+    PAYLOAD_VARIABLE_HIDDEN_PARAMS = %i(token).freeze
+
     def create_pipeline_from_trigger(trigger)
       # this check is to not leak the presence of the project if user cannot read it
       return unless trigger.project == project
@@ -70,9 +73,23 @@ module Ci
     end
 
     def variables
+      if ::Feature.enabled?(:ci_trigger_payload_into_pipeline, project, default_enabled: :yaml)
+        param_variables + [payload_variable]
+      else
+        param_variables
+      end
+    end
+
+    def param_variables
       params[:variables].to_h.map do |key, value|
         { key: key, value: value }
       end
     end
+
+    def payload_variable
+      { key: PAYLOAD_VARIABLE_KEY,
+        value: params.except(*PAYLOAD_VARIABLE_HIDDEN_PARAMS).to_json,
+        variable_type: :file }
+    end
   end
 end

config/feature_flags/development/ci_trigger_payload_into_pipeline.yml

+---
+name: ci_trigger_payload_into_pipeline
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/53837
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/321027
+milestone: '13.9'
+type: development
+group: group::pipeline authoring
+default_enabled: false

doc/ci/triggers/README.md

@@ -185,6 +185,41 @@ You should pass `ref` as part of the URL, to take precedence over `ref` from
 the webhook body that designates the branch ref that fired the trigger in the
 source repository. Be sure to URL-encode `ref` if it contains slashes.
 
+### Using webhook payload in the triggered pipeline
+
+> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/31197) in GitLab 13.9.
+> - It's [deployed behind a feature flag](../../user/feature_flags.md), disabled by default.
+> - It's disabled on GitLab.com.
+> - It's not recommended for production use.
+> - To use it in GitLab self-managed instances, ask a GitLab administrator to [enable it](#enable-or-disable-the-trigger_payload-variable). **(FREE SELF)**
+
+WARNING:
+This feature might not be available to you. Check the **version history** note above for details.
+
+If you trigger a pipeline by using a webhook, you can access the webhook payload with
+the `TRIGGER_PAYLOAD` [predefined CI/CD variable](../variables/predefined_variables.md).
+The payload is exposed as a [file-type variable](../variables/README.md#custom-cicd-variables-of-type-file),
+so you can access the data with `cat $TRIGGER_PAYLOAD` or a similar command.
+
+#### Enable or disable the `TRIGGER_PAYLOAD` variable
+
+The `TRIGGER_PAYLOAD` CI/CD variable is under development and not ready for production use. It is
+deployed behind a feature flag that is **disabled by default**.
+[GitLab administrators with access to the GitLab Rails console](../../administration/feature_flags.md)
+can enable it.
+
+To enable it:
+
+```ruby
+Feature.enable(:ci_trigger_payload_into_pipeline)
+```
+
+To disable it:
+
+```ruby
+Feature.disable(:ci_trigger_payload_into_pipeline)
+```
+
 ## Making use of trigger variables
 
 You can pass any number of arbitrary variables in the trigger API call and they

doc/ci/variables/predefined_variables.md

@@ -145,3 +145,4 @@ Kubernetes-specific variables are detailed in the
 | `GITLAB_USER_ID`                              | 8.12   | all    | The ID of the user who started the job. |
 | `GITLAB_USER_LOGIN`                           | 10.0   | all    | The login username of the user who started the job. |
 | `GITLAB_USER_NAME`                            | 10.0   | all    | The real name of the user who started the job. |
+| `TRIGGER_PAYLOAD`                             | 13.9   | all    | This variable is available when a pipeline is [triggered with a webhook](../triggers/README.md#using-webhook-payload-in-the-triggered-pipeline) |

ee/spec/requests/api/triggers_spec.rb

@@ -96,7 +96,7 @@ RSpec.describe API::Triggers do
               expect(response).to have_gitlab_http_status(:created)
               expect(Ci::Pipeline.last.source).to eq('pipeline')
               expect(Ci::Pipeline.last.triggered_by_pipeline).not_to be_nil
-              expect(Ci::Pipeline.last.variables.map { |v| { v.key => v.value } }.last).to eq(params[:variables])
+              expect(Ci::Pipeline.last.variables.find { |v| v.key == 'KEY' }.value).to eq('VALUE')
             end
           end
         end

spec/requests/api/triggers_spec.rb

@@ -54,6 +54,15 @@ RSpec.describe API::Triggers do
         expect(pipeline.builds.size).to eq(5)
       end
 
+      it 'stores payload as a variable' do
+        post api("/projects/#{project.id}/trigger/pipeline"), params: options.merge(ref: 'master')
+
+        expect(response).to have_gitlab_http_status(:created)
+        expect(pipeline.variables.find { |v| v.key == 'TRIGGER_PAYLOAD' }.value).to eq(
+          "{\"ref\":\"master\",\"id\":\"#{project.id}\",\"variables\":{}}"
+        )
+      end
+
       it 'returns bad request with no pipeline created if there\'s no commit for that ref' do
         post api("/projects/#{project.id}/trigger/pipeline"), params: options.merge(ref: 'other-branch')
 
@@ -84,7 +93,7 @@ RSpec.describe API::Triggers do
           post api("/projects/#{project.id}/trigger/pipeline"), params: options.merge(variables: variables, ref: 'master')
 
           expect(response).to have_gitlab_http_status(:created)
-          expect(pipeline.variables.map { |v| { v.key => v.value } }.last).to eq(variables)
+          expect(pipeline.variables.find { |v| v.key == 'TRIGGER_KEY' }.value).to eq('TRIGGER_VALUE')
         end
       end
     end

spec/services/ci/pipeline_trigger_service_spec.rb

@@ -45,6 +45,27 @@ RSpec.describe Ci::PipelineTriggerService do
             expect(result[:status]).to eq(:success)
           end
 
+          it 'stores the payload as a variable' do
+            expect { result }.to change { Ci::PipelineVariable.count }.by(1)
+
+            var = result[:pipeline].variables.first
+
+            expect(var.key).to eq('TRIGGER_PAYLOAD')
+            expect(var.value).to eq('{"ref":"master","variables":null}')
+            expect(var.variable_type).to eq('file')
+          end
+
+          context 'when FF ci_trigger_payload_into_pipeline is disabled' do
+            before do
+              stub_feature_flags(ci_trigger_payload_into_pipeline: false)
+            end
+
+            it 'does not store the payload as a variable' do
+              expect { result }.not_to change { Ci::PipelineVariable.count }
+              expect(result[:pipeline].variables).to be_empty
+            end
+          end
+
           context 'when commit message has [ci skip]' do
             before do
               allow_next(Ci::Pipeline).to receive(:git_commit_message) { '[ci skip]' }
@@ -60,8 +81,8 @@ RSpec.describe Ci::PipelineTriggerService do
             let(:params) { { token: trigger.token, ref: 'master', variables: variables } }
             let(:variables) { { 'AAA' => 'AAA123' } }
 
-            it 'has a variable' do
-              expect { result }.to change { Ci::PipelineVariable.count }.by(1)
+            it 'has variables' do
+              expect { result }.to change { Ci::PipelineVariable.count }.by(2)
                                .and change { Ci::TriggerRequest.count }.by(1)
               expect(result[:pipeline].variables.map { |v| { v.key => v.value } }.first).to eq(variables)
               expect(result[:pipeline].trigger_requests.last.variables).to be_nil
@@ -155,8 +176,8 @@ RSpec.describe Ci::PipelineTriggerService do
             let(:params) { { token: job.token, ref: 'master', variables: variables } }
             let(:variables) { { 'AAA' => 'AAA123' } }
 
-            it 'has a variable' do
-              expect { result }.to change { Ci::PipelineVariable.count }.by(1)
+            it 'has variables' do
+              expect { result }.to change { Ci::PipelineVariable.count }.by(2)
                                .and change { Ci::Sources::Pipeline.count }.by(1)
               expect(result[:pipeline].variables.map { |v| { v.key => v.value } }.first).to eq(variables)
               expect(job.sourced_pipelines.last.pipeline_id).to eq(result[:pipeline].id)