Get job id with artifact path

scripts/api/get_job_id

@@ -20,6 +20,7 @@ class JobFinder
     @job_query = options.delete(:job_query)
     @pipeline_id = options.delete(:pipeline_id)
     @job_name = options.delete(:job_name)
+    @artifact_path = options.delete(:artifact_path)
 
     # Force the token to be a string so that if api_token is nil, it's set to '', allowing unauthenticated requests (for forks).
     api_token = options.delete(:api_token).to_s
@@ -33,19 +34,31 @@ class JobFinder
   end
 
   def execute
-    find_job_with_filtered_pipelines || find_job_in_pipeline
+    find_job_with_artifact || find_job_with_filtered_pipelines || find_job_in_pipeline
   end
 
   private
 
-  attr_reader :project, :pipeline_query, :job_query, :pipeline_id, :job_name
+  attr_reader :project, :pipeline_query, :job_query, :pipeline_id, :job_name, :artifact_path
+
+  def find_job_with_artifact
+    return if artifact_path.nil?
+
+    Gitlab.pipelines(project, pipeline_query_params).auto_paginate do |pipeline|
+      Gitlab.pipeline_jobs(project, pipeline.id, job_query_params).auto_paginate do |job|
+        return job if found_job_with_artifact?(job) # rubocop:disable Cop/AvoidReturnFromBlocks
+      end
+    end
+
+    raise 'Job not found!'
+  end
 
   def find_job_with_filtered_pipelines
     return if pipeline_query.empty?
 
     Gitlab.pipelines(project, pipeline_query_params).auto_paginate do |pipeline|
       Gitlab.pipeline_jobs(project, pipeline.id, job_query_params).auto_paginate do |job|
-        return job if job.name == job_name # rubocop:disable Cop/AvoidReturnFromBlocks
+        return job if found_job_by_name?(job) # rubocop:disable Cop/AvoidReturnFromBlocks
       end
     end
 
@@ -56,12 +69,22 @@ class JobFinder
     return unless pipeline_id
 
     Gitlab.pipeline_jobs(project, pipeline_id, job_query_params).auto_paginate do |job|
-      return job if job.name == job_name # rubocop:disable Cop/AvoidReturnFromBlocks
+      return job if found_job_by_name?(job) # rubocop:disable Cop/AvoidReturnFromBlocks
     end
 
     raise 'Job not found!'
   end
 
+  def found_job_with_artifact?(job)
+    artifact_url = "https://gitlab.com/api/v4/projects/#{CGI.escape(project)}/jobs/#{job.id}/artifacts/#{artifact_path}"
+    response = HTTParty.head(artifact_url) # rubocop:disable Gitlab/HTTParty
+    response.success?
+  end
+
+  def found_job_by_name?(job)
+    job.name == job_name
+  end
+
   def pipeline_query_params
     @pipeline_query_params ||= { per_page: 100, **pipeline_query }
   end
@@ -95,6 +118,10 @@ if $0 == __FILE__
       options[:job_name] = value
     end
 
+    opts.on("-a", "--artifact-path ARTIFACT_PATH", String, "A valid artifact path") do |value|
+      options[:artifact_path] = value
+    end
+
     opts.on("-t", "--api-token API_TOKEN", String, "A value API token with the `read_api` scope") do |value|
       options[:api_token] = value
     end

scripts/rspec_helpers.sh

@@ -17,12 +17,14 @@ function retrieve_tests_metadata() {
     scripts/api/download_job_artifact --project "${project_path}" --job-id "${test_metadata_job_id}" --artifact-path "${FLAKY_RSPEC_SUITE_REPORT_PATH}" || echo "{}" > "${FLAKY_RSPEC_SUITE_REPORT_PATH}"
   fi
 
-  # FIXME: We will need to find a pipeline where the $RSPEC_PACKED_TESTS_MAPPING_PATH.gz actually exists (Crystalball only runs every two-hours, but the `update-tests-metadata` runs for all `master` pipelines...).
-  # if [[ ! -f "${RSPEC_PACKED_TESTS_MAPPING_PATH}" ]]; then
-  #   (scripts/api/download_job_artifact --project "${project_path}" --job-id "${test_metadata_job_id}" --artifact-path "${RSPEC_PACKED_TESTS_MAPPING_PATH}.gz" && gzip -d "${RSPEC_PACKED_TESTS_MAPPING_PATH}.gz") || echo "{}" > "${RSPEC_PACKED_TESTS_MAPPING_PATH}"
-  # fi
-  #
-  # scripts/unpack-test-mapping "${RSPEC_PACKED_TESTS_MAPPING_PATH}" "${RSPEC_TESTS_MAPPING_PATH}"
+  local test_metadata_with_mapping_job_id
+  test_metadata_with_mapping_job_id=$(scripts/api/get_job_id --project "${project_path}" -q "status=success" -q "ref=master" -q "username=gitlab-bot" -Q "scope=success" --job-name "update-tests-metadata" --artifact-path "${RSPEC_PACKED_TESTS_MAPPING_PATH}.gz")
+
+  if [[ ! -f "${RSPEC_PACKED_TESTS_MAPPING_PATH}" ]]; then
+   (scripts/api/download_job_artifact --project "${project_path}" --job-id "${test_metadata_job_id}" --artifact-path "${RSPEC_PACKED_TESTS_MAPPING_PATH}.gz" && gzip -d "${RSPEC_PACKED_TESTS_MAPPING_PATH}.gz") || echo "{}" > "${RSPEC_PACKED_TESTS_MAPPING_PATH}"
+  fi
+
+  scripts/unpack-test-mapping "${RSPEC_PACKED_TESTS_MAPPING_PATH}" "${RSPEC_TESTS_MAPPING_PATH}"
 }
 
 function update_tests_metadata() {