Merge branch 'ee-54610-allow-review-jobs-to-fail' into 'master'

[EE] Retry the review-{deploy,qa} jobs twice, and allow them to fail

Closes gitlab-ce#54610 and gitlab-ce#54717

See merge request gitlab-org/gitlab-ee!8677

.gitlab-ci.yml

@@ -1127,6 +1127,8 @@ no_ee_check:
 # GitLab Review apps
 review-deploy:
   <<: *review-base
+  retry: 2
+  allow_failure: true
   variables:
     GIT_DEPTH: "1"
     HOST_SUFFIX: "${CI_ENVIRONMENT_SLUG}"
@@ -1156,6 +1158,8 @@ review-deploy:
 
 .review-qa-base: &review-qa-base
   <<: *review-docker
+  retry: 2
+  allow_failure: true
   variables:
     <<: *review-docker-variables
     API_TOKEN: "${GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN}"
@@ -1183,10 +1187,8 @@ review-deploy:
 
 review-qa-smoke:
   <<: *review-qa-base
-  # retry: 2
   script:
     - gitlab-qa Test::Instance::Smoke "${QA_IMAGE}" "${CI_ENVIRONMENT_URL}"
-  allow_failure: true
 
 review-qa-all:
   <<: *review-qa-base

db/fixtures/production/001_application_settings.rb

+puts "Creating the default ApplicationSetting record.".color(:green)
+Gitlab::CurrentSettings.current_application_settings

doc/development/testing_guide/review_apps.md

@@ -62,6 +62,41 @@ You can also manually start the `review-qa-all`: it runs the full QA suite.
 
 Note that both jobs first wait for the `review-deploy` job to be finished.
 
+## How to?
+
+### Find my Review App slug?
+
+1. Open the `review-deploy` job.
+1. Look for `Checking for previous deployment of review-*`.
+1. For instance for `Checking for previous deployment of review-qa-raise-e-12chm0`,
+  your Review App slug would be `review-qa-raise-e-12chm0` in this case.
+
+### Run a Rails console?
+
+1. [Filter Workloads by your Review App slug](https://console.cloud.google.com/kubernetes/workload?project=gitlab-review-apps)
+  , e.g. `review-29951-issu-id2qax`.
+1. Find and open the `task-runner` Deployment, e.g. `review-29951-issu-id2qax-task-runner`.
+1. Click on the Pod in the "Managed pods" section, e.g. `review-29951-issu-id2qax-task-runner-d5455cc8-2lsvz`.
+1. Click on the `KUBECTL` dropdown, then `Exec` -> `task-runner`.
+1. Replace `-c task-runner -- ls` with `-- /srv/gitlab/bin/rails c` from the
+  default command or
+  - Run `kubectl exec --namespace review-apps-ce -it review-29951-issu-id2qax-task-runner-d5455cc8-2lsvz -- /srv/gitlab/bin/rails c`
+    and
+  - Replace `review-apps-ce` with `review-apps-ee` if the Review App
+    is running EE, and
+  - Replace `review-29951-issu-id2qax-task-runner-d5455cc8-2lsvz`
+    with your Pod's name.
+
+### Dig into a Pod's logs?
+
+1. [Filter Workloads by your Review App slug](https://console.cloud.google.com/kubernetes/workload?project=gitlab-review-apps)
+  , e.g. `review-1979-1-mul-dnvlhv`.
+1. Find and open the `migrations` Deployment, e.g.
+  `review-1979-1-mul-dnvlhv-migrations.1`.
+1. Click on the Pod in the "Managed pods" section, e.g.
+  `review-1979-1-mul-dnvlhv-migrations.1-nqwtx`.
+1. Click on the `Container logs` link.
+
 ## Frequently Asked Questions
 
 **Isn't it too much to trigger CNG image builds on every test run? This creates

scripts/review_apps/review-apps.sh

@@ -289,8 +289,8 @@ function get_job_id() {
     local url="https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/pipelines/${CI_PIPELINE_ID}/jobs?per_page=100&page=${page}${query_string}"
     echoerr "GET ${url}"
 
-    local job_id=$(curl --silent --show-error --header "PRIVATE-TOKEN: ${API_TOKEN}" "${url}" | jq ".[] | select(.name == \"${job_name}\") | .id")
-    [[ "${job_id}" == "" && "${page}" -lt "$max_page" ]] || break
+    local job_id=$(curl --silent --show-error --header "PRIVATE-TOKEN: ${API_TOKEN}" "${url}" | jq "map(select(.name == \"${job_name}\")) | map(.id) | last")
+    [[ "${job_id}" == "null" && "${page}" -lt "$max_page" ]] || break
 
     ((page++))
   done
@@ -328,17 +328,18 @@ function wait_for_job_to_be_done() {
 
   # In case the job hasn't finished yet. Keep trying until the job times out.
   local interval=30
-  local elapsed=0
+  local elapsed_seconds=0
   while true; do
     local job_status=$(curl --silent --show-error --header "PRIVATE-TOKEN: ${API_TOKEN}" "${url}" | jq ".status" | sed -e s/\"//g)
     [[ "${job_status}" == "pending" || "${job_status}" == "running" ]] || break
 
     printf "."
-    ((elapsed+=$interval))
+    ((elapsed_seconds+=$interval))
     sleep ${interval}
   done
 
-  echoerr "Waited '${job_name}' for ${elapsed} seconds."
+  local elapsed_minutes=$((elapsed_seconds / 60))
+  echoerr "Waited '${job_name}' for ${elapsed_minutes} minutes."
 
   if [[ "${job_status}" == "failed" ]]; then
     echo "The '${job_name}' failed."