Reinstate 60s timeout in Cluster Prometheus

app/models/clusters/applications/prometheus.rb

@@ -106,7 +106,9 @@ module Clusters
         proxy_url = kube_client.proxy_url('service', service_name, service_port, Gitlab::Kubernetes::Helm::NAMESPACE)
 
         # ensures headers containing auth data are appended to original k8s client options
-        options = kube_client.rest_client.options.merge(headers: kube_client.headers)
+        options = kube_client.rest_client.options
+          .merge(prometheus_client_default_options)
+          .merge(headers: kube_client.headers)
         Gitlab::PrometheusClient.new(proxy_url, options)
       rescue Kubeclient::HttpError, Errno::ECONNRESET, Errno::ECONNREFUSED, Errno::ENETUNREACH
         # If users have mistakenly set parameters or removed the depended clusters,

app/models/concerns/prometheus_adapter.rb

@@ -3,6 +3,11 @@
 module PrometheusAdapter
   extend ActiveSupport::Concern
 
+  # We should choose more conservative timeouts, but some queries we run are now busting our
+  # default timeouts, which are stricter. We should make those queries faster instead.
+  # See https://gitlab.com/gitlab-org/gitlab/-/issues/232786
+  DEFAULT_PROMETHEUS_REQUEST_TIMEOUT_SEC = 60.seconds
+
   included do
     include ReactiveCaching
 
@@ -15,6 +20,12 @@ module PrometheusAdapter
       raise NotImplementedError
     end
 
+    def prometheus_client_default_options
+      {
+        timeout: DEFAULT_PROMETHEUS_REQUEST_TIMEOUT_SEC
+      }
+    end
+
     # This is a light-weight check if a prometheus client is properly configured.
     def configured?
       raise NotImplemented

app/models/project_services/prometheus_service.rb

@@ -97,13 +97,9 @@ class PrometheusService < MonitoringService
   def prometheus_client
     return unless should_return_client?
 
-    options = {
-      allow_local_requests: allow_local_api_url?,
-      # We should choose more conservative timeouts, but some queries we run are now busting our
-      # default timeouts, which are stricter. We should make those queries faster instead.
-      # See https://gitlab.com/gitlab-org/gitlab/-/issues/233109
-      timeout: 60
-    }
+    options = prometheus_client_default_options.merge(
+      allow_local_requests: allow_local_api_url?
+    )
 
     if behind_iap?
       # Adds the Authorization header

changelogs/unreleased/232786-reinstate-60s-timeout.yml

+---
+title: Reinstate 60s timeout in Cluster Prometheus
+merge_request: 39595
+author:
+type: other

spec/models/clusters/applications/prometheus_spec.rb

@@ -109,10 +109,13 @@ RSpec.describe Clusters::Applications::Prometheus do
         expect(subject.prometheus_client).to be_instance_of(Gitlab::PrometheusClient)
       end
 
-      it 'copies proxy_url, options and headers from kube client to prometheus_client' do
+      it 'merges proxy_url, options and headers from kube client with prometheus_client options' do
         expect(Gitlab::PrometheusClient)
           .to(receive(:new))
-          .with(a_valid_url, kube_client.rest_client.options.merge(headers: kube_client.headers))
+          .with(a_valid_url, kube_client.rest_client.options.merge({
+            headers: kube_client.headers,
+            timeout: PrometheusAdapter::DEFAULT_PROMETHEUS_REQUEST_TIMEOUT_SEC
+          }))
         subject.prometheus_client
       end