Allow to retrieve all jobs for a given pipeline

`:id/pipelines/:pipeline_id/jobs` endpoint is extended with a new parameter `include_retried`.
This parameter controls whether to include retried jobs into a response or not.
This change is required in order to fix the following issue: https://gitlab.com/gitlab-org/gitlab/-/issues/272627

app/finders/ci/jobs_finder.rb

@@ -45,7 +45,8 @@ module Ci
       return unless pipeline
       raise Gitlab::Access::AccessDeniedError unless can?(current_user, :read_build, pipeline)
 
-      jobs_by_type(pipeline, type).latest
+      jobs_scope = jobs_by_type(pipeline, type)
+      params[:include_retried] ? jobs_scope : jobs_scope.latest
     end
 
     def filter_by_scope(builds)

changelogs/unreleased/return-all-jobs.yml

+---
+title: Allow to retrieve all jobs for a given pipeline
+merge_request: 48589
+author: Alexander Kutelev
+type: fixed

doc/api/jobs.md

@@ -154,11 +154,12 @@ Get a list of jobs for a pipeline.
 GET /projects/:id/pipelines/:pipeline_id/jobs
 ```
 
-| Attribute     | Type                           | Required | Description                                                                                                                                                                                                    |
-|---------------|--------------------------------|----------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| `id`          | integer/string                 | yes      | ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user.                                                                                               |
-| `pipeline_id` | integer                        | yes      | ID of a pipeline.                                                                                                                                                                                          |
-| `scope`       | string **or** array of strings | no       | Scope of jobs to show. Either one of or an array of the following: `created`, `pending`, `running`, `failed`, `success`, `canceled`, `skipped`, or `manual`. All jobs are returned if `scope` is not provided. |
+| Attribute         | Type                           | Required | Description                                                                                                                                                                                                    |
+|-------------------|--------------------------------|----------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| `id`              | integer/string                 | yes      | ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user.                                                                                                   |
+| `pipeline_id`     | integer                        | yes      | ID of a pipeline.                                                                                                                                                                                              |
+| `scope`           | string **or** array of strings | no       | Scope of jobs to show. Either one of or an array of the following: `created`, `pending`, `running`, `failed`, `success`, `canceled`, `skipped`, or `manual`. All jobs are returned if `scope` is not provided. |
+| `include_retried` | boolean                        | no       | Include retried jobs in the response. Defaults to `false`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/272627) in GitLab 13.9.                                                                  |
 
 ```shell
 curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/pipelines/6/jobs?scope[]=pending&scope[]=running"
@@ -260,7 +261,6 @@ Example of response
       "status": "pending"
     },
     "ref": "master",
-    "artifacts": [],
     "runner": null,
     "stage": "test",
     "status": "failed",
@@ -290,6 +290,12 @@ Example of response
 In GitLab 13.3 and later, this endpoint [returns data for any pipeline](pipelines.md#single-pipeline-requests)
 including [child pipelines](../ci/parent_child_pipelines.md).
 
+In GitLab 13.5 and later, this endpoint does not return retried jobs in the response
+by default.
+
+In GitLab 13.9 and later, this endpoint can include retried jobs in the response
+with `include_retried` set to `true`.
+
 ## List pipeline bridges
 
 Get a list of bridge jobs for a pipeline.

lib/api/ci/pipelines.rb

@@ -121,6 +121,7 @@ module API
         end
         params do
           requires :pipeline_id, type: Integer, desc: 'The pipeline ID'
+          optional :include_retried, type: Boolean, default: false, desc: 'Includes retried jobs'
           use :optional_scope
           use :pagination
         end

spec/finders/ci/jobs_finder_spec.rb

@@ -9,7 +9,7 @@ RSpec.describe Ci::JobsFinder, '#execute' do
   let_it_be(:pipeline) { create(:ci_pipeline, project: project) }
   let_it_be(:job_1) { create(:ci_build) }
   let_it_be(:job_2) { create(:ci_build, :running) }
-  let_it_be(:job_3) { create(:ci_build, :success, pipeline: pipeline) }
+  let_it_be(:job_3) { create(:ci_build, :success, pipeline: pipeline, name: 'build') }
 
   let(:params) { {} }
 
@@ -95,4 +95,35 @@ RSpec.describe Ci::JobsFinder, '#execute' do
       end
     end
   end
+
+  context 'when pipeline is present' do
+    before_all do
+      project.add_maintainer(user)
+      job_3.update!(retried: true)
+    end
+
+    let_it_be(:job_4) { create(:ci_build, :success, pipeline: pipeline, name: 'build') }
+
+    subject { described_class.new(current_user: user, pipeline: pipeline, params: params).execute }
+
+    it 'does not return retried jobs by default' do
+      expect(subject).to match_array([job_4])
+    end
+
+    context 'when include_retried is false' do
+      let(:params) { { include_retried: false } }
+
+      it 'does not return retried jobs' do
+        expect(subject).to match_array([job_4])
+      end
+    end
+
+    context 'when include_retried is true' do
+      let(:params) { { include_retried: true } }
+
+      it 'returns retried jobs' do
+        expect(subject).to match_array([job_3, job_4])
+      end
+    end
+  end
 end

spec/requests/api/ci/pipelines_spec.rb

@@ -312,7 +312,7 @@ RSpec.describe API::Ci::Pipelines do
     let(:query) { {} }
     let(:api_user) { user }
     let_it_be(:job) do
-      create(:ci_build, :success, pipeline: pipeline,
+      create(:ci_build, :success, name: 'build', pipeline: pipeline,
                                   artifacts_expire_at: 1.day.since)
     end
 
@@ -405,6 +405,38 @@ RSpec.describe API::Ci::Pipelines do
           get api("/projects/#{project.id}/pipelines/#{pipeline.id}/jobs", api_user), params: query
         end.not_to exceed_all_query_limit(control_count)
       end
+
+      context 'pipeline has retried jobs' do
+        before_all do
+          job.update!(retried: true)
+        end
+
+        let_it_be(:successor) { create(:ci_build, :success, name: 'build', pipeline: pipeline) }
+
+        it 'does not return retried jobs by default' do
+          expect(json_response).to be_an Array
+          expect(json_response.length).to eq(1)
+        end
+
+        context 'when include_retried is false' do
+          let(:query) { { include_retried: false } }
+
+          it 'does not return retried jobs' do
+            expect(json_response).to be_an Array
+            expect(json_response.length).to eq(1)
+          end
+        end
+
+        context 'when include_retried is true' do
+          let(:query) { { include_retried: true } }
+
+          it 'returns retried jobs' do
+            expect(json_response).to be_an Array
+            expect(json_response.length).to eq(2)
+            expect(json_response[0]['name']).to eq(json_response[1]['name'])
+          end
+        end
+      end
     end
 
     context 'no pipeline is found' do