Merge branch 'bjk/fix_metrics' into 'master'

Auto-register Prometheus metrics

Closes #326

See merge request gitlab-org/gitlab-workhorse!660

changelogs/unreleased/bjk-fix_metrics.yml

+---
+title: Auto-register Prometheus metrics
+merge_request: 660
+author:
+type: fixed

go.mod

@@ -13,15 +13,15 @@ require (
 	github.com/disintegration/imaging v1.6.2
 	github.com/getsentry/raven-go v0.2.0
 	github.com/golang/gddo v0.0.0-20190419222130-af0f2af80721
-	github.com/golang/protobuf v1.4.2
+	github.com/golang/protobuf v1.4.3
 	github.com/gomodule/redigo v2.0.0+incompatible
 	github.com/gorilla/websocket v1.4.0
-	github.com/grpc-ecosystem/go-grpc-middleware v1.0.0
+	github.com/grpc-ecosystem/go-grpc-middleware v1.2.2
 	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0
 	github.com/johannesboyne/gofakes3 v0.0.0-20200510090907-02d71f533bec
-	github.com/jpillora/backoff v0.0.0-20170918002102-8eab2debe79d
+	github.com/jpillora/backoff v1.0.0
 	github.com/mitchellh/copystructure v1.0.0
-	github.com/prometheus/client_golang v1.0.0
+	github.com/prometheus/client_golang v1.8.0
 	github.com/rafaeljusto/redigomock v0.0.0-20190202135759-257e089e14a1
 	github.com/sebest/xff v0.0.0-20160910043805-6c115e0ffa35
 	github.com/shabbyrobe/gocovmerge v0.0.0-20190829150210-3e036491d500 // indirect
@@ -32,7 +32,7 @@ require (
 	gitlab.com/gitlab-org/labkit v1.0.0
 	gocloud.dev v0.20.0
 	golang.org/x/lint v0.0.0-20200302205851-738671d3881b
-	golang.org/x/net v0.0.0-20200602114024-627f9648deb9
+	golang.org/x/net v0.0.0-20200625001655-4c5254603344
 	golang.org/x/tools v0.0.0-20200608174601-1b747fd94509
 	google.golang.org/grpc v1.29.1
 	honnef.co/go/tools v0.0.1-2020.1.5

internal/api/api.go

@@ -11,6 +11,7 @@ import (
 	"strings"
 
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 
 	"gitlab.com/gitlab-org/gitaly/proto/go/gitalypb"
 
@@ -34,14 +35,14 @@ type API struct {
 }
 
 var (
-	requestsCounter = prometheus.NewCounterVec(
+	requestsCounter = promauto.NewCounterVec(
 		prometheus.CounterOpts{
 			Name: "gitlab_workhorse_internal_api_requests",
 			Help: "How many internal API requests have been completed by gitlab-workhorse, partitioned by status code and HTTP method.",
 		},
 		[]string{"code", "method"},
 	)
-	bytesTotal = prometheus.NewCounter(
+	bytesTotal = promauto.NewCounter(
 		prometheus.CounterOpts{
 			Name: "gitlab_workhorse_internal_api_failure_response_bytes",
 			Help: "How many bytes have been returned by upstream GitLab in API failure/rejection response bodies.",
@@ -49,11 +50,6 @@ var (
 	)
 )
 
-func init() {
-	prometheus.MustRegister(requestsCounter)
-	prometheus.MustRegister(bytesTotal)
-}
-
 func NewAPI(myURL *url.URL, version string, roundTripper http.RoundTripper) *API {
 	return &API{
 		Client:  &http.Client{Transport: roundTripper},

internal/artifacts/artifacts_upload.go

@@ -12,6 +12,7 @@ import (
 	"syscall"
 
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 	"gitlab.com/gitlab-org/labkit/log"
 
 	"gitlab.com/gitlab-org/gitlab-workhorse/internal/api"
@@ -28,16 +29,12 @@ const (
 	ArtifactFormatDefault = ""
 )
 
-var zipSubcommandsErrorsCounter = prometheus.NewCounterVec(
+var zipSubcommandsErrorsCounter = promauto.NewCounterVec(
 	prometheus.CounterOpts{
 		Name: "gitlab_workhorse_zip_subcommand_errors_total",
 		Help: "Errors comming from subcommands used for processing ZIP archives",
 	}, []string{"error"})
 
-func init() {
-	prometheus.MustRegister(zipSubcommandsErrorsCounter)
-}
-
 type artifactsUploadProcessor struct {
 	opts   *filestore.SaveFileOpts
 	format string

internal/builds/register.go

@@ -7,6 +7,7 @@ import (
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 
 	"gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
 	"gitlab.com/gitlab-org/gitlab-workhorse/internal/redis"
@@ -20,14 +21,14 @@ const (
 )
 
 var (
-	registerHandlerRequests = prometheus.NewCounterVec(
+	registerHandlerRequests = promauto.NewCounterVec(
 		prometheus.CounterOpts{
 			Name: "gitlab_workhorse_builds_register_handler_requests",
 			Help: "Describes how many requests in different states hit a register handler",
 		},
 		[]string{"status"},
 	)
-	registerHandlerOpen = prometheus.NewGaugeVec(
+	registerHandlerOpen = promauto.NewGaugeVec(
 		prometheus.GaugeOpts{
 			Name: "gitlab_workhorse_builds_register_handler_open",
 			Help: "Describes how many requests is currently open in given state",
@@ -53,13 +54,6 @@ type largeBodyError struct{ error }
 
 type WatchKeyHandler func(key, value string, timeout time.Duration) (redis.WatchKeyStatus, error)
 
-func init() {
-	prometheus.MustRegister(
-		registerHandlerRequests,
-		registerHandlerOpen,
-	)
-}
-
 type runnerRequest struct {
 	Token      string `json:"token,omitempty"`
 	LastUpdate string `json:"last_update,omitempty"`

internal/git/archive.go

@@ -18,6 +18,7 @@ import (
 	"github.com/golang/protobuf/proto" //lint:ignore SA1019 https://gitlab.com/gitlab-org/gitlab-workhorse/-/issues/274
 
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 
 	"gitlab.com/gitlab-org/gitaly/proto/go/gitalypb"
 
@@ -39,7 +40,7 @@ type archiveParams struct {
 
 var (
 	SendArchive     = &archive{"git-archive:"}
-	gitArchiveCache = prometheus.NewCounterVec(
+	gitArchiveCache = promauto.NewCounterVec(
 		prometheus.CounterOpts{
 			Name: "gitlab_workhorse_git_archive_cache",
 			Help: "Cache hits and misses for 'git archive' streaming",
@@ -48,10 +49,6 @@ var (
 	)
 )
 
-func init() {
-	prometheus.MustRegister(gitArchiveCache)
-}
-
 func (a *archive) Inject(w http.ResponseWriter, r *http.Request, sendData string) {
 	var params archiveParams
 	if err := a.Unpack(&params, sendData); err != nil {

internal/git/responsewriter.go

@@ -5,6 +5,7 @@ import (
 	"strconv"
 
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 
 	"gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
 )
@@ -15,12 +16,12 @@ const (
 )
 
 var (
-	gitHTTPSessionsActive = prometheus.NewGauge(prometheus.GaugeOpts{
+	gitHTTPSessionsActive = promauto.NewGauge(prometheus.GaugeOpts{
 		Name: "gitlab_workhorse_git_http_sessions_active",
 		Help: "Number of Git HTTP request-response cycles currently being handled by gitlab-workhorse.",
 	})
 
-	gitHTTPRequests = prometheus.NewCounterVec(
+	gitHTTPRequests = promauto.NewCounterVec(
 		prometheus.CounterOpts{
 			Name: "gitlab_workhorse_git_http_requests",
 			Help: "How many Git HTTP requests have been processed by gitlab-workhorse, partitioned by request type and agent.",
@@ -28,7 +29,7 @@ var (
 		[]string{"method", "code", "service", "agent"},
 	)
 
-	gitHTTPBytes = prometheus.NewCounterVec(
+	gitHTTPBytes = promauto.NewCounterVec(
 		prometheus.CounterOpts{
 			Name: "gitlab_workhorse_git_http_bytes",
 			Help: "How many Git HTTP bytes have been sent by gitlab-workhorse, partitioned by request type, agent and direction.",
@@ -37,12 +38,6 @@ var (
 	)
 )
 
-func init() {
-	prometheus.MustRegister(gitHTTPSessionsActive)
-	prometheus.MustRegister(gitHTTPRequests)
-	prometheus.MustRegister(gitHTTPBytes)
-}
-
 type HttpResponseWriter struct {
 	helper.CountingResponseWriter
 }

internal/gitaly/gitaly.go

@@ -10,6 +10,7 @@ import (
 	grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware"
 	grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus"
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 	gitalyauth "gitlab.com/gitlab-org/gitaly/auth"
 	gitalyclient "gitlab.com/gitlab-org/gitaly/client"
 	"gitlab.com/gitlab-org/gitaly/proto/go/gitalypb"
@@ -43,7 +44,7 @@ var (
 		connections: make(map[cacheKey]*grpc.ClientConn),
 	}
 
-	connectionsTotal = prometheus.NewCounterVec(
+	connectionsTotal = promauto.NewCounterVec(
 		prometheus.CounterOpts{
 			Name: "gitlab_workhorse_gitaly_connections_total",
 			Help: "Number of Gitaly connections that have been established",
@@ -52,10 +53,6 @@ var (
 	)
 )
 
-func init() {
-	prometheus.MustRegister(connectionsTotal)
-}
-
 func withOutgoingMetadata(ctx context.Context, features map[string]string) context.Context {
 	md := metadata.New(nil)
 	for k, v := range features {

internal/imageresizer/image_resizer.go

@@ -17,6 +17,7 @@ import (
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 
 	"gitlab.com/gitlab-org/labkit/correlation"
 	"gitlab.com/gitlab-org/labkit/log"
@@ -96,7 +97,7 @@ const (
 )
 
 var (
-	imageResizeConcurrencyLimitExceeds = prometheus.NewCounter(
+	imageResizeConcurrencyLimitExceeds = promauto.NewCounter(
 		prometheus.CounterOpts{
 			Namespace: namespace,
 			Subsystem: subsystem,
@@ -104,7 +105,7 @@ var (
 			Help:      "Amount of image resizing requests that exceeded the maximum allowed scaler processes",
 		},
 	)
-	imageResizeProcesses = prometheus.NewGauge(
+	imageResizeProcesses = promauto.NewGauge(
 		prometheus.GaugeOpts{
 			Namespace: namespace,
 			Subsystem: subsystem,
@@ -112,7 +113,7 @@ var (
 			Help:      "Amount of image scaler processes working now",
 		},
 	)
-	imageResizeMaxProcesses = prometheus.NewGauge(
+	imageResizeMaxProcesses = promauto.NewGauge(
 		prometheus.GaugeOpts{
 			Namespace: namespace,
 			Subsystem: subsystem,
@@ -120,7 +121,7 @@ var (
 			Help:      "The maximum amount of image scaler processes allowed to run concurrently",
 		},
 	)
-	imageResizeRequests = prometheus.NewCounterVec(
+	imageResizeRequests = promauto.NewCounterVec(
 		prometheus.CounterOpts{
 			Namespace: namespace,
 			Subsystem: subsystem,
@@ -129,7 +130,7 @@ var (
 		},
 		[]string{"status"},
 	)
-	imageResizeDurations = prometheus.NewHistogramVec(
+	imageResizeDurations = promauto.NewHistogramVec(
 		prometheus.HistogramOpts{
 			Namespace: namespace,
 			Subsystem: subsystem,
@@ -154,14 +155,6 @@ const (
 	maxMagicLen = 8                   // 8 first bytes is enough to detect PNG or JPEG
 )
 
-func init() {
-	prometheus.MustRegister(imageResizeConcurrencyLimitExceeds)
-	prometheus.MustRegister(imageResizeProcesses)
-	prometheus.MustRegister(imageResizeMaxProcesses)
-	prometheus.MustRegister(imageResizeRequests)
-	prometheus.MustRegister(imageResizeDurations)
-}
-
 func NewResizer(cfg config.Config) *Resizer {
 	imageResizeMaxProcesses.Set(float64(cfg.ImageResizerConfig.MaxScalerProcs))
 

internal/objectstore/prometheus.go

 package objectstore
 
-import "github.com/prometheus/client_golang/prometheus"
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
 
 var (
-	objectStorageUploadRequests = prometheus.NewCounterVec(
+	objectStorageUploadRequests = promauto.NewCounterVec(
 		prometheus.CounterOpts{
 			Name: "gitlab_workhorse_object_storage_upload_requests",
 			Help: "How many object storage requests have been processed",
 		},
 		[]string{"status"},
 	)
-	objectStorageUploadsOpen = prometheus.NewGauge(
+	objectStorageUploadsOpen = promauto.NewGauge(
 		prometheus.GaugeOpts{
 			Name: "gitlab_workhorse_object_storage_upload_open",
 			Help: "Describes many object storage requests are open now",
 		},
 	)
-	objectStorageUploadBytes = prometheus.NewCounter(
+	objectStorageUploadBytes = promauto.NewCounter(
 		prometheus.CounterOpts{
 			Name: "gitlab_workhorse_object_storage_upload_bytes",
 			Help: "How many bytes were sent to object storage",
 		},
 	)
-	objectStorageUploadTime = prometheus.NewHistogram(
+	objectStorageUploadTime = promauto.NewHistogram(
 		prometheus.HistogramOpts{
 			Name:    "gitlab_workhorse_object_storage_upload_time",
 			Help:    "How long it took to upload objects",
@@ -34,10 +37,3 @@ var (
 
 	objectStorageUploadTimeBuckets = []float64{.1, .25, .5, 1, 2.5, 5, 10, 25, 50, 100}
 )
-
-func init() {
-	prometheus.MustRegister(
-		objectStorageUploadRequests,
-		objectStorageUploadsOpen,
-		objectStorageUploadBytes)
-}

internal/objectstore/uploader.go

@@ -90,6 +90,8 @@ func (u *uploader) Consume(outerCtx context.Context, reader io.Reader, deadline
 		}
 	}
 
+	objectStorageUploadBytes.Add(float64(cr.n))
+
 	return cr.n, nil
 }
 

internal/queueing/queue.go

@@ -5,6 +5,7 @@ import (
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 )
 
 type errTooManyRequests struct{ error }
@@ -44,7 +45,7 @@ func newQueueMetrics(name string, timeout time.Duration) *queueMetrics {
 	}
 
 	metrics := &queueMetrics{
-		queueingLimit: prometheus.NewGauge(prometheus.GaugeOpts{
+		queueingLimit: promauto.NewGauge(prometheus.GaugeOpts{
 			Name: "gitlab_workhorse_queueing_limit",
 			Help: "Current limit set for the queueing mechanism",
 			ConstLabels: prometheus.Labels{
@@ -52,7 +53,7 @@ func newQueueMetrics(name string, timeout time.Duration) *queueMetrics {
 			},
 		}),
 
-		queueingQueueLimit: prometheus.NewGauge(prometheus.GaugeOpts{
+		queueingQueueLimit: promauto.NewGauge(prometheus.GaugeOpts{
 			Name: "gitlab_workhorse_queueing_queue_limit",
 			Help: "Current queueLimit set for the queueing mechanism",
 			ConstLabels: prometheus.Labels{
@@ -60,7 +61,7 @@ func newQueueMetrics(name string, timeout time.Duration) *queueMetrics {
 			},
 		}),
 
-		queueingQueueTimeout: prometheus.NewGauge(prometheus.GaugeOpts{
+		queueingQueueTimeout: promauto.NewGauge(prometheus.GaugeOpts{
 			Name: "gitlab_workhorse_queueing_queue_timeout",
 			Help: "Current queueTimeout set for the queueing mechanism",
 			ConstLabels: prometheus.Labels{
@@ -68,7 +69,7 @@ func newQueueMetrics(name string, timeout time.Duration) *queueMetrics {
 			},
 		}),
 
-		queueingBusy: prometheus.NewGauge(prometheus.GaugeOpts{
+		queueingBusy: promauto.NewGauge(prometheus.GaugeOpts{
 			Name: "gitlab_workhorse_queueing_busy",
 			Help: "How many queued requests are now processed",
 			ConstLabels: prometheus.Labels{
@@ -76,7 +77,7 @@ func newQueueMetrics(name string, timeout time.Duration) *queueMetrics {
 			},
 		}),
 
-		queueingWaiting: prometheus.NewGauge(prometheus.GaugeOpts{
+		queueingWaiting: promauto.NewGauge(prometheus.GaugeOpts{
 			Name: "gitlab_workhorse_queueing_waiting",
 			Help: "How many requests are now queued",
 			ConstLabels: prometheus.Labels{
@@ -84,7 +85,7 @@ func newQueueMetrics(name string, timeout time.Duration) *queueMetrics {
 			},
 		}),
 
-		queueingWaitingTime: prometheus.NewHistogram(prometheus.HistogramOpts{
+		queueingWaitingTime: promauto.NewHistogram(prometheus.HistogramOpts{
 			Name: "gitlab_workhorse_queueing_waiting_time",
 			Help: "How many time a request spent in queue",
 			ConstLabels: prometheus.Labels{
@@ -93,7 +94,7 @@ func newQueueMetrics(name string, timeout time.Duration) *queueMetrics {
 			Buckets: waitingTimeBuckets,
 		}),
 
-		queueingErrors: prometheus.NewCounterVec(
+		queueingErrors: promauto.NewCounterVec(
 			prometheus.CounterOpts{
 				Name: "gitlab_workhorse_queueing_errors",
 				Help: "How many times the TooManyRequests or QueueintTimedout errors were returned while queueing, partitioned by error type",
@@ -105,14 +106,6 @@ func newQueueMetrics(name string, timeout time.Duration) *queueMetrics {
 		),
 	}
 
-	prometheus.MustRegister(metrics.queueingLimit)
-	prometheus.MustRegister(metrics.queueingQueueLimit)
-	prometheus.MustRegister(metrics.queueingQueueTimeout)
-	prometheus.MustRegister(metrics.queueingBusy)
-	prometheus.MustRegister(metrics.queueingWaiting)
-	prometheus.MustRegister(metrics.queueingWaitingTime)
-	prometheus.MustRegister(metrics.queueingErrors)
-
 	return metrics
 }
 

internal/redis/keywatcher.go

@@ -9,6 +9,7 @@ import (
 	"github.com/gomodule/redigo/redis"
 	"github.com/jpillora/backoff"
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 	"gitlab.com/gitlab-org/labkit/log"
 
 	"gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
@@ -24,13 +25,13 @@ var (
 		Factor: 2,
 		Jitter: true,
 	}
-	keyWatchers = prometheus.NewGauge(
+	keyWatchers = promauto.NewGauge(
 		prometheus.GaugeOpts{
 			Name: "gitlab_workhorse_keywatcher_keywatchers",
 			Help: "The number of keys that is being watched by gitlab-workhorse",
 		},
 	)
-	totalMessages = prometheus.NewCounter(
+	totalMessages = promauto.NewCounter(
 		prometheus.CounterOpts{
 			Name: "gitlab_workhorse_keywatcher_total_messages",
 			Help: "How many messages gitlab-workhorse has received in total on pubsub.",
@@ -38,13 +39,6 @@ var (
 	)
 )
 
-func init() {
-	prometheus.MustRegister(
-		keyWatchers,
-		totalMessages,
-	)
-}
-
 const (
 	keySubChannel = "workhorse:notifications"
 )

internal/redis/redis.go

@@ -10,6 +10,7 @@ import (
 	"github.com/FZambia/sentinel"
 	"github.com/gomodule/redigo/redis"
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 	"gitlab.com/gitlab-org/labkit/log"
 
 	"gitlab.com/gitlab-org/gitlab-workhorse/internal/config"
@@ -45,14 +46,14 @@ const (
 )
 
 var (
-	totalConnections = prometheus.NewCounter(
+	totalConnections = promauto.NewCounter(
 		prometheus.CounterOpts{
 			Name: "gitlab_workhorse_redis_total_connections",
 			Help: "How many connections gitlab-workhorse has opened in total. Can be used to track Redis connection rate for this process",
 		},
 	)
 
-	errorCounter = prometheus.NewCounterVec(
+	errorCounter = promauto.NewCounterVec(
 		prometheus.CounterOpts{
 			Name: "gitlab_workhorse_redis_errors",
 			Help: "Counts different types of Redis errors encountered by workhorse, by type and destination (redis, sentinel)",
@@ -61,13 +62,6 @@ var (
 	)
 )
 
-func init() {
-	prometheus.MustRegister(
-		totalConnections,
-		errorCounter,
-	)
-}
-
 func sentinelConn(master string, urls []config.TomlURL) *sentinel.Sentinel {
 	if len(urls) == 0 {
 		return nil

internal/senddata/senddata.go

@@ -8,17 +8,18 @@ import (
 	"gitlab.com/gitlab-org/gitlab-workhorse/internal/senddata/contentprocessor"
 
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 )
 
 var (
-	sendDataResponses = prometheus.NewCounterVec(
+	sendDataResponses = promauto.NewCounterVec(
 		prometheus.CounterOpts{
 			Name: "gitlab_workhorse_senddata_responses",
 			Help: "How many HTTP responses have been hijacked by a workhorse senddata injecter",
 		},
 		[]string{"injecter"},
 	)
-	sendDataResponseBytes = prometheus.NewCounterVec(
+	sendDataResponseBytes = promauto.NewCounterVec(
 		prometheus.CounterOpts{
 			Name: "gitlab_workhorse_senddata_response_bytes",
 			Help: "How many bytes have been written by workhorse senddata response injecters",
@@ -27,11 +28,6 @@ var (
 	)
 )
 
-func init() {
-	prometheus.MustRegister(sendDataResponses)
-	prometheus.MustRegister(sendDataResponseBytes)
-}
-
 type sendDataResponseWriter struct {
 	rw        http.ResponseWriter
 	status    int

internal/sendfile/sendfile.go

@@ -14,6 +14,7 @@ import (
 	"regexp"
 
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 
 	"gitlab.com/gitlab-org/labkit/log"
 	"gitlab.com/gitlab-org/labkit/mask"
@@ -23,7 +24,7 @@ import (
 )
 
 var (
-	sendFileRequests = prometheus.NewCounterVec(
+	sendFileRequests = promauto.NewCounterVec(
 		prometheus.CounterOpts{
 			Name: "gitlab_workhorse_sendfile_requests",
 			Help: "How many X-Sendfile requests have been processed by gitlab-workhorse, partitioned by sendfile type.",
@@ -31,7 +32,7 @@ var (
 		[]string{"type"},
 	)
 
-	sendFileBytes = prometheus.NewCounterVec(
+	sendFileBytes = promauto.NewCounterVec(
 		prometheus.CounterOpts{
 			Name: "gitlab_workhorse_sendfile_bytes",
 			Help: "How many X-Sendfile bytes have been sent by gitlab-workhorse, partitioned by sendfile type.",
@@ -49,11 +50,6 @@ type sendFileResponseWriter struct {
 	req      *http.Request
 }
 
-func init() {
-	prometheus.MustRegister(sendFileRequests)
-	prometheus.MustRegister(sendFileBytes)
-}
-
 func SendFile(h http.Handler) http.Handler {
 	return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
 		s := &sendFileResponseWriter{

internal/sendurl/sendurl.go

@@ -8,6 +8,7 @@ import (
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 
 	"gitlab.com/gitlab-org/labkit/correlation"
 	"gitlab.com/gitlab-org/labkit/log"
@@ -68,20 +69,20 @@ var httpClient = &http.Client{
 }
 
 var (
-	sendURLRequests = prometheus.NewCounterVec(
+	sendURLRequests = promauto.NewCounterVec(
 		prometheus.CounterOpts{
 			Name: "gitlab_workhorse_send_url_requests",
 			Help: "How many send URL requests have been processed",
 		},
 		[]string{"status"},
 	)
-	sendURLOpenRequests = prometheus.NewGauge(
+	sendURLOpenRequests = promauto.NewGauge(
 		prometheus.GaugeOpts{
 			Name: "gitlab_workhorse_send_url_open_requests",
 			Help: "Describes how many send URL requests are open now",
 		},
 	)
-	sendURLBytes = prometheus.NewCounter(
+	sendURLBytes = promauto.NewCounter(
 		prometheus.CounterOpts{
 			Name: "gitlab_workhorse_send_url_bytes",
 			Help: "How many bytes were passed with send URL",
@@ -93,13 +94,6 @@ var (
 	sendURLRequestsSucceeded     = sendURLRequests.WithLabelValues("succeeded")
 )
 
-func init() {
-	prometheus.MustRegister(
-		sendURLRequests,
-		sendURLOpenRequests,
-		sendURLBytes)
-}
-
 func (e *entry) Inject(w http.ResponseWriter, r *http.Request, sendData string) {
 	var params entryParams
 

internal/staticpages/error_pages.go

@@ -8,12 +8,13 @@ import (
 	"path/filepath"
 
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 
 	"gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
 )
 
 var (
-	staticErrorResponses = prometheus.NewCounterVec(
+	staticErrorResponses = promauto.NewCounterVec(
 		prometheus.CounterOpts{
 			Name: "gitlab_workhorse_static_error_responses",
 			Help: "How many HTTP responses have been changed to a static error page, by HTTP status code.",
@@ -30,10 +31,6 @@ const (
 	ErrorFormatText
 )
 
-func init() {
-	prometheus.MustRegister(staticErrorResponses)
-}
-
 type errorPageResponseWriter struct {
 	rw       http.ResponseWriter
 	status   int

internal/upload/rewrite.go

@@ -11,6 +11,7 @@ import (
 	"strings"
 
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 	"gitlab.com/gitlab-org/labkit/log"
 
 	"gitlab.com/gitlab-org/gitlab-workhorse/internal/api"
@@ -23,7 +24,7 @@ import (
 var ErrInjectedClientParam = errors.New("injected client parameter")
 
 var (
-	multipartUploadRequests = prometheus.NewCounterVec(
+	multipartUploadRequests = promauto.NewCounterVec(
 		prometheus.CounterOpts{
 
 			Name: "gitlab_workhorse_multipart_upload_requests",
@@ -32,7 +33,7 @@ var (
 		[]string{"type"},
 	)
 
-	multipartFileUploadBytes = prometheus.NewCounterVec(
+	multipartFileUploadBytes = promauto.NewCounterVec(
 		prometheus.CounterOpts{
 			Name: "gitlab_workhorse_multipart_upload_bytes",
 			Help: "How many disk bytes of multipart file parts have been successfully written by gitlab-workhorse. Partitioned by type.",
@@ -40,7 +41,7 @@ var (
 		[]string{"type"},
 	)
 
-	multipartFiles = prometheus.NewCounterVec(
+	multipartFiles = promauto.NewCounterVec(
 		prometheus.CounterOpts{
 			Name: "gitlab_workhorse_multipart_upload_files",
 			Help: "How many multipart file parts have been processed by gitlab-workhorse. Partitioned by type.",
@@ -56,12 +57,6 @@ type rewriter struct {
 	finalizedFields map[string]bool
 }
 
-func init() {
-	prometheus.MustRegister(multipartUploadRequests)
-	prometheus.MustRegister(multipartFileUploadBytes)
-	prometheus.MustRegister(multipartFiles)
-}
-
 func rewriteFormFilesFromMultipart(r *http.Request, writer *multipart.Writer, preauth *api.Response, filter MultipartFormProcessor, opts *filestore.SaveFileOpts) error {
 	// Create multipart reader
 	reader, err := r.MultipartReader()

internal/upstream/metrics.go

@@ -4,6 +4,7 @@ import (
 	"net/http"
 
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 )
 
@@ -40,14 +41,14 @@ func byteSizeBuckets() []float64 {
 }
 
 var (
-	httpInFlightRequests = prometheus.NewGauge(prometheus.GaugeOpts{
+	httpInFlightRequests = promauto.NewGauge(prometheus.GaugeOpts{
 		Namespace: namespace,
 		Subsystem: httpSubsystem,
 		Name:      "in_flight_requests",
 		Help:      "A gauge of requests currently being served by workhorse.",
 	})
 
-	httpRequestsTotal = prometheus.NewCounterVec(
+	httpRequestsTotal = promauto.NewCounterVec(
 		prometheus.CounterOpts{
 			Namespace: namespace,
 			Subsystem: httpSubsystem,
@@ -57,7 +58,7 @@ var (
 		[]string{"code", "method", "route"},
 	)
 
-	httpRequestDurationSeconds = prometheus.NewHistogramVec(
+	httpRequestDurationSeconds = promauto.NewHistogramVec(
 		prometheus.HistogramOpts{
 			Namespace: namespace,
 			Subsystem: httpSubsystem,
@@ -68,7 +69,7 @@ var (
 		[]string{"code", "method", "route"},
 	)
 
-	httpRequestSizeBytes = prometheus.NewHistogramVec(
+	httpRequestSizeBytes = promauto.NewHistogramVec(
 		prometheus.HistogramOpts{
 			Namespace: namespace,
 			Subsystem: httpSubsystem,
@@ -79,7 +80,7 @@ var (
 		[]string{"code", "method", "route"},
 	)
 
-	httpResponseSizeBytes = prometheus.NewHistogramVec(
+	httpResponseSizeBytes = promauto.NewHistogramVec(
 		prometheus.HistogramOpts{
 			Namespace: namespace,
 			Subsystem: httpSubsystem,
@@ -90,7 +91,7 @@ var (
 		[]string{"code", "method", "route"},
 	)
 
-	httpTimeToWriteHeaderSeconds = prometheus.NewHistogramVec(
+	httpTimeToWriteHeaderSeconds = promauto.NewHistogramVec(
 		prometheus.HistogramOpts{
 			Namespace: namespace,
 			Subsystem: httpSubsystem,
@@ -102,14 +103,6 @@ var (
 	)
 )
 
-func init() {
-	prometheus.MustRegister(httpInFlightRequests)
-	prometheus.MustRegister(httpRequestsTotal)
-	prometheus.MustRegister(httpRequestDurationSeconds)
-	prometheus.MustRegister(httpRequestSizeBytes)
-	prometheus.MustRegister(httpTimeToWriteHeaderSeconds)
-}
-
 func instrumentRoute(next http.Handler, method string, regexpStr string) http.Handler {
 	handler := next