Merge branch...

Merge branch 'reinstate-335675-consider-storing-retrieving-tests-metadata-in-from-pages-instead-of-artifacts' into 'master'

ci: Retrieve tests metadata from Pages as an opt-in strategy (take 2)

See merge request gitlab-org/gitlab!69073

.gitlab/ci/test-metadata.gitlab-ci.yml

@@ -26,6 +26,7 @@ update-tests-metadata:
     - .test-metadata:rules:update-tests-metadata
   stage: post-test
   dependencies:
+    - retrieve-tests-metadata
     - setup-test-env
     - rspec migration pg12
     - rspec frontend_fixture

scripts/rspec_helpers.sh

 #!/usr/bin/env bash
 
 function retrieve_tests_metadata() {
-  mkdir -p knapsack/ rspec_flaky/ rspec_profiling/
+  mkdir -p $(dirname "$KNAPSACK_RSPEC_SUITE_REPORT_PATH") $(dirname "$FLAKY_RSPEC_SUITE_REPORT_PATH") rspec_profiling/
 
-  # ${CI_DEFAULT_BRANCH} might not be master in other forks but we want to
-  # always target the canonical project here, so the branch must be hardcoded
-  local project_path="gitlab-org/gitlab"
-  local artifact_branch="master"
-  local test_metadata_job_id
+  if [[ -n "${RETRIEVE_TESTS_METADATA_FROM_PAGES}" ]]; then
+    if [[ ! -f "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" ]]; then
+      curl --location -o "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" "https://gitlab-org.gitlab.io/gitlab/${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" || echo "{}" > "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}"
+    fi
 
-  # Ruby
-  test_metadata_job_id=$(scripts/api/get_job_id.rb --endpoint "https://gitlab.com/api/v4" --project "${project_path}" -q "status=success" -q "ref=${artifact_branch}" -q "username=gitlab-bot" -Q "scope=success" --job-name "update-tests-metadata")
+    if [[ ! -f "${FLAKY_RSPEC_SUITE_REPORT_PATH}" ]]; then
+      curl --location -o "${FLAKY_RSPEC_SUITE_REPORT_PATH}" "https://gitlab-org.gitlab.io/gitlab/${FLAKY_RSPEC_SUITE_REPORT_PATH}" || echo "{}" > "${FLAKY_RSPEC_SUITE_REPORT_PATH}"
+    fi
+  else
+    # ${CI_DEFAULT_BRANCH} might not be master in other forks but we want to
+    # always target the canonical project here, so the branch must be hardcoded
+    local project_path="gitlab-org/gitlab"
+    local artifact_branch="master"
+    local test_metadata_job_id
 
-  if [[ ! -f "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" ]]; then
-    scripts/api/download_job_artifact.rb --endpoint "https://gitlab.com/api/v4" --project "${project_path}" --job-id "${test_metadata_job_id}" --artifact-path "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" || echo "{}" > "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}"
-  fi
+    # Ruby
+    test_metadata_job_id=$(scripts/api/get_job_id.rb --endpoint "https://gitlab.com/api/v4" --project "${project_path}" -q "status=success" -q "ref=${artifact_branch}" -q "username=gitlab-bot" -Q "scope=success" --job-name "update-tests-metadata")
 
-  if [[ ! -f "${FLAKY_RSPEC_SUITE_REPORT_PATH}" ]]; then
-    scripts/api/download_job_artifact.rb --endpoint "https://gitlab.com/api/v4" --project "${project_path}" --job-id "${test_metadata_job_id}" --artifact-path "${FLAKY_RSPEC_SUITE_REPORT_PATH}" || echo "{}" > "${FLAKY_RSPEC_SUITE_REPORT_PATH}"
+    if [[ ! -f "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" ]]; then
+      scripts/api/download_job_artifact.rb --endpoint "https://gitlab.com/api/v4" --project "${project_path}" --job-id "${test_metadata_job_id}" --artifact-path "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" || echo "{}" > "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}"
+    fi
+
+    if [[ ! -f "${FLAKY_RSPEC_SUITE_REPORT_PATH}" ]]; then
+      scripts/api/download_job_artifact.rb --endpoint "https://gitlab.com/api/v4" --project "${project_path}" --job-id "${test_metadata_job_id}" --artifact-path "${FLAKY_RSPEC_SUITE_REPORT_PATH}" || echo "{}" > "${FLAKY_RSPEC_SUITE_REPORT_PATH}"
+    fi
   fi
 }
 
@@ -40,18 +50,24 @@ function update_tests_metadata() {
 }
 
 function retrieve_tests_mapping() {
-  mkdir -p crystalball/
+  mkdir -p $(dirname "$RSPEC_PACKED_TESTS_MAPPING_PATH")
 
-  # ${CI_DEFAULT_BRANCH} might not be master in other forks but we want to
-  # always target the canonical project here, so the branch must be hardcoded
-  local project_path="gitlab-org/gitlab"
-  local artifact_branch="master"
-  local test_metadata_with_mapping_job_id
+  if [[ -n "${RETRIEVE_TESTS_METADATA_FROM_PAGES}" ]]; then
+    if [[ ! -f "${RSPEC_PACKED_TESTS_MAPPING_PATH}" ]]; then
+      (curl --location  -o "${RSPEC_PACKED_TESTS_MAPPING_PATH}.gz" "https://gitlab-org.gitlab.io/gitlab/${RSPEC_PACKED_TESTS_MAPPING_PATH}.gz" && gzip -d "${RSPEC_PACKED_TESTS_MAPPING_PATH}.gz") || echo "{}" > "${RSPEC_PACKED_TESTS_MAPPING_PATH}"
+    fi
+  else
+    # ${CI_DEFAULT_BRANCH} might not be master in other forks but we want to
+    # always target the canonical project here, so the branch must be hardcoded
+    local project_path="gitlab-org/gitlab"
+    local artifact_branch="master"
+    local test_metadata_with_mapping_job_id
 
-  test_metadata_with_mapping_job_id=$(scripts/api/get_job_id.rb --endpoint "https://gitlab.com/api/v4" --project "${project_path}" -q "status=success" -q "ref=${artifact_branch}" -q "username=gitlab-bot" -Q "scope=success" --job-name "update-tests-metadata" --artifact-path "${RSPEC_PACKED_TESTS_MAPPING_PATH}.gz")
+    test_metadata_with_mapping_job_id=$(scripts/api/get_job_id.rb --endpoint "https://gitlab.com/api/v4" --project "${project_path}" -q "status=success" -q "ref=${artifact_branch}" -q "username=gitlab-bot" -Q "scope=success" --job-name "update-tests-metadata" --artifact-path "${RSPEC_PACKED_TESTS_MAPPING_PATH}.gz")
 
-  if [[ ! -f "${RSPEC_PACKED_TESTS_MAPPING_PATH}" ]]; then
-   (scripts/api/download_job_artifact.rb --endpoint "https://gitlab.com/api/v4" --project "${project_path}" --job-id "${test_metadata_with_mapping_job_id}" --artifact-path "${RSPEC_PACKED_TESTS_MAPPING_PATH}.gz" && gzip -d "${RSPEC_PACKED_TESTS_MAPPING_PATH}.gz") || echo "{}" > "${RSPEC_PACKED_TESTS_MAPPING_PATH}"
+    if [[ ! -f "${RSPEC_PACKED_TESTS_MAPPING_PATH}" ]]; then
+     (scripts/api/download_job_artifact.rb --endpoint "https://gitlab.com/api/v4" --project "${project_path}" --job-id "${test_metadata_with_mapping_job_id}" --artifact-path "${RSPEC_PACKED_TESTS_MAPPING_PATH}.gz" && gzip -d "${RSPEC_PACKED_TESTS_MAPPING_PATH}.gz") || echo "{}" > "${RSPEC_PACKED_TESTS_MAPPING_PATH}"
+    fi
   fi
 
   scripts/unpack-test-mapping "${RSPEC_PACKED_TESTS_MAPPING_PATH}" "${RSPEC_TESTS_MAPPING_PATH}"