Merge branch 'sh-update-workhorse-8.58.0' into 'master'

Update GitLab Workhorse to v8.58.0

See merge request gitlab-org/gitlab!49534

GITLAB_WORKHORSE_VERSION

-8.57.0
+8.58.0

changelogs/unreleased/sh-update-workhorse-8-58-0.yml

+---
+title: Update GitLab Workhorse to v8.58.0
+merge_request: 49534
+author:
+type: changed

workhorse/.gitignore

 testdata/data
 testdata/scratch
 testdata/public
+testdata/alt-public
 /gitlab-workhorse
 /gitlab-resize-image
 /gitlab-zip-cat

workhorse/CHANGELOG

 # Changelog for gitlab-workhorse
 
+## v8.58.0
+
+### Added
+- Support alternate document root directory
+  https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/626
+
+### Fixed
+- Fix uploader not returning 413 when artifact too large
+  https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/663
+- Auto-register Prometheus metrics
+  https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/660
+
+### Other
+- Do not resize when image is less than 8 bytes
+  https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/666
+
 ## v8.57.0
 
 ### Added

workhorse/README.md

@@ -7,6 +7,20 @@ push/pull and Git archive downloads.
 Workhorse itself is not a feature, but there are [several features in
 GitLab](doc/architecture/gitlab_features.md) that would not work efficiently without Workhorse.
 
+## Canonical source
+
+The canonical source for Workhorse is currently
+[gitlab-org/gitlab-workhorse](https://gitlab.com/gitlab-org/gitlab-workhorse).
+As explained in https://gitlab.com/groups/gitlab-org/-/epics/4826, we
+are in the process of moving the canonical source to
+[gitlab-org/gitlab/workhorse](https://gitlab.com/gitlab-org/gitlab/tree/master/workhorse).
+
+Until that transition is complete, changes (Merge Requests) for
+Workhorse should be submitted at
+[gitlab-org/gitlab-workhorse](https://gitlab.com/gitlab-org/gitlab-workhorse).
+Once merged, they will propagate to gitlab-org/gitlab/workhorse via
+the usual Workhorse release process.
+
 ## Documentation
 
 Workhorse documentation is available in the [`doc` folder of this repository](doc/).

workhorse/VERSION

-8.57.0
+8.58.0

workhorse/config.toml.example

+# alt_document_root = '/home/git/public/assets'
+
 [redis]
 URL = "unix:/home/git/gitlab/redis/redis.socket"
 

workhorse/go.mod

@@ -13,15 +13,15 @@ require (
 	github.com/disintegration/imaging v1.6.2
 	github.com/getsentry/raven-go v0.2.0
 	github.com/golang/gddo v0.0.0-20190419222130-af0f2af80721
-	github.com/golang/protobuf v1.4.2
+	github.com/golang/protobuf v1.4.3
 	github.com/gomodule/redigo v2.0.0+incompatible
 	github.com/gorilla/websocket v1.4.0
-	github.com/grpc-ecosystem/go-grpc-middleware v1.0.0
+	github.com/grpc-ecosystem/go-grpc-middleware v1.2.2
 	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0
 	github.com/johannesboyne/gofakes3 v0.0.0-20200510090907-02d71f533bec
-	github.com/jpillora/backoff v0.0.0-20170918002102-8eab2debe79d
+	github.com/jpillora/backoff v1.0.0
 	github.com/mitchellh/copystructure v1.0.0
-	github.com/prometheus/client_golang v1.0.0
+	github.com/prometheus/client_golang v1.8.0
 	github.com/rafaeljusto/redigomock v0.0.0-20190202135759-257e089e14a1
 	github.com/sebest/xff v0.0.0-20160910043805-6c115e0ffa35
 	github.com/shabbyrobe/gocovmerge v0.0.0-20190829150210-3e036491d500 // indirect
@@ -32,7 +32,7 @@ require (
 	gitlab.com/gitlab-org/labkit v1.0.0
 	gocloud.dev v0.20.0
 	golang.org/x/lint v0.0.0-20200302205851-738671d3881b
-	golang.org/x/net v0.0.0-20200602114024-627f9648deb9
+	golang.org/x/net v0.0.0-20200625001655-4c5254603344
 	golang.org/x/tools v0.0.0-20200608174601-1b747fd94509
 	google.golang.org/grpc v1.29.1
 	honnef.co/go/tools v0.0.1-2020.1.5

workhorse/internal/api/api.go

@@ -11,6 +11,7 @@ import (
 	"strings"
 
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 
 	"gitlab.com/gitlab-org/gitaly/proto/go/gitalypb"
 
@@ -34,14 +35,14 @@ type API struct {
 }
 
 var (
-	requestsCounter = prometheus.NewCounterVec(
+	requestsCounter = promauto.NewCounterVec(
 		prometheus.CounterOpts{
 			Name: "gitlab_workhorse_internal_api_requests",
 			Help: "How many internal API requests have been completed by gitlab-workhorse, partitioned by status code and HTTP method.",
 		},
 		[]string{"code", "method"},
 	)
-	bytesTotal = prometheus.NewCounter(
+	bytesTotal = promauto.NewCounter(
 		prometheus.CounterOpts{
 			Name: "gitlab_workhorse_internal_api_failure_response_bytes",
 			Help: "How many bytes have been returned by upstream GitLab in API failure/rejection response bodies.",
@@ -49,11 +50,6 @@ var (
 	)
 )
 
-func init() {
-	prometheus.MustRegister(requestsCounter)
-	prometheus.MustRegister(bytesTotal)
-}
-
 func NewAPI(myURL *url.URL, version string, roundTripper http.RoundTripper) *API {
 	return &API{
 		Client:  &http.Client{Transport: roundTripper},

workhorse/internal/artifacts/artifacts_upload.go

@@ -12,6 +12,7 @@ import (
 	"syscall"
 
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 	"gitlab.com/gitlab-org/labkit/log"
 
 	"gitlab.com/gitlab-org/gitlab-workhorse/internal/api"
@@ -28,16 +29,12 @@ const (
 	ArtifactFormatDefault = ""
 )
 
-var zipSubcommandsErrorsCounter = prometheus.NewCounterVec(
+var zipSubcommandsErrorsCounter = promauto.NewCounterVec(
 	prometheus.CounterOpts{
 		Name: "gitlab_workhorse_zip_subcommand_errors_total",
 		Help: "Errors comming from subcommands used for processing ZIP archives",
 	}, []string{"error"})
 
-func init() {
-	prometheus.MustRegister(zipSubcommandsErrorsCounter)
-}
-
 type artifactsUploadProcessor struct {
 	opts   *filestore.SaveFileOpts
 	format string

workhorse/internal/builds/register.go

@@ -7,6 +7,7 @@ import (
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 
 	"gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
 	"gitlab.com/gitlab-org/gitlab-workhorse/internal/redis"
@@ -20,14 +21,14 @@ const (
 )
 
 var (
-	registerHandlerRequests = prometheus.NewCounterVec(
+	registerHandlerRequests = promauto.NewCounterVec(
 		prometheus.CounterOpts{
 			Name: "gitlab_workhorse_builds_register_handler_requests",
 			Help: "Describes how many requests in different states hit a register handler",
 		},
 		[]string{"status"},
 	)
-	registerHandlerOpen = prometheus.NewGaugeVec(
+	registerHandlerOpen = promauto.NewGaugeVec(
 		prometheus.GaugeOpts{
 			Name: "gitlab_workhorse_builds_register_handler_open",
 			Help: "Describes how many requests is currently open in given state",
@@ -53,13 +54,6 @@ type largeBodyError struct{ error }
 
 type WatchKeyHandler func(key, value string, timeout time.Duration) (redis.WatchKeyStatus, error)
 
-func init() {
-	prometheus.MustRegister(
-		registerHandlerRequests,
-		registerHandlerOpen,
-	)
-}
-
 type runnerRequest struct {
 	Token      string `json:"token,omitempty"`
 	LastUpdate string `json:"last_update,omitempty"`

workhorse/internal/config/config.go

@@ -105,6 +105,7 @@ type Config struct {
 	ObjectStorageCredentials ObjectStorageCredentials `toml:"object_storage"`
 	PropagateCorrelationID   bool                     `toml:"-"`
 	ImageResizerConfig       ImageResizerConfig       `toml:"image_resizer"`
+	AltDocumentRoot          string                   `toml:"alt_document_root"`
 }
 
 var DefaultImageResizerConfig = ImageResizerConfig{

workhorse/internal/config/config_test.go

@@ -21,6 +21,7 @@ func TestLoadEmptyConfig(t *testing.T) {
 	cfg, err := LoadConfig(config)
 	require.NoError(t, err)
 
+	require.Empty(t, cfg.AltDocumentRoot)
 	require.Equal(t, cfg.ImageResizerConfig.MaxFilesize, uint64(250000))
 	require.GreaterOrEqual(t, cfg.ImageResizerConfig.MaxScalerProcs, uint32(2))
 
@@ -97,3 +98,14 @@ max_filesize = 350000
 
 	require.Equal(t, expected, cfg.ImageResizerConfig)
 }
+
+func TestAltDocumentConfig(t *testing.T) {
+	config := `
+alt_document_root = "/path/to/documents"
+`
+
+	cfg, err := LoadConfig(config)
+	require.NoError(t, err)
+
+	require.Equal(t, "/path/to/documents", cfg.AltDocumentRoot)
+}

workhorse/internal/filestore/file_handler.go

@@ -166,7 +166,13 @@ func SaveFileFromReader(ctx context.Context, reader io.Reader, size int64, opts
 			return nil, SizeError(fmt.Errorf("the upload size %d is over maximum of %d bytes", size, opts.MaximumSize))
 		}
 
-		reader = &hardLimitReader{r: reader, n: opts.MaximumSize}
+		hlr := &hardLimitReader{r: reader, n: opts.MaximumSize}
+		reader = hlr
+		defer func() {
+			if hlr.n < 0 {
+				err = ErrEntityTooLarge
+			}
+		}()
 	}
 
 	fh.Size, err = uploadDestination.Consume(ctx, reader, opts.Deadline)

workhorse/internal/filestore/file_handler_test.go

@@ -2,6 +2,7 @@ package filestore_test
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"os"
@@ -428,6 +429,107 @@ func TestSaveMultipartInBodyFailure(t *testing.T) {
 	require.EqualError(t, err, test.MultipartUploadInternalError().Error())
 }
 
+func TestSaveRemoteFileWithLimit(t *testing.T) {
+	testhelper.ConfigureSecret()
+
+	type remote int
+	const (
+		notRemote remote = iota
+		remoteSingle
+		remoteMultipart
+	)
+
+	remoteTypes := []remote{remoteSingle, remoteMultipart}
+
+	tests := []struct {
+		name        string
+		objectSize  int64
+		maxSize     int64
+		expectedErr error
+		testData    string
+	}{
+		{
+			name:       "known size with no limit",
+			testData:   test.ObjectContent,
+			objectSize: test.ObjectSize,
+		},
+		{
+			name:       "unknown size with no limit",
+			testData:   test.ObjectContent,
+			objectSize: -1,
+		},
+		{
+			name:        "unknown object size with limit",
+			testData:    test.ObjectContent,
+			objectSize:  -1,
+			maxSize:     test.ObjectSize - 1,
+			expectedErr: filestore.ErrEntityTooLarge,
+		},
+		{
+			name:        "large object with unknown size with limit",
+			testData:    string(make([]byte, 20000)),
+			objectSize:  -1,
+			maxSize:     19000,
+			expectedErr: filestore.ErrEntityTooLarge,
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			var opts filestore.SaveFileOpts
+
+			for _, remoteType := range remoteTypes {
+				tmpFolder, err := ioutil.TempDir("", "workhorse-test-tmp")
+				require.NoError(t, err)
+				defer os.RemoveAll(tmpFolder)
+
+				osStub, ts := test.StartObjectStore()
+				defer ts.Close()
+
+				switch remoteType {
+				case remoteSingle:
+					objectURL := ts.URL + test.ObjectPath
+
+					opts.RemoteID = "test-file"
+					opts.RemoteURL = objectURL
+					opts.PresignedPut = objectURL + "?Signature=ASignature"
+					opts.PresignedDelete = objectURL + "?Signature=AnotherSignature"
+					opts.Deadline = testDeadline()
+					opts.MaximumSize = tc.maxSize
+				case remoteMultipart:
+					objectURL := ts.URL + test.ObjectPath
+
+					opts.RemoteID = "test-file"
+					opts.RemoteURL = objectURL
+					opts.PresignedDelete = objectURL + "?Signature=AnotherSignature"
+					opts.PartSize = int64(len(tc.testData)/2) + 1
+					opts.PresignedParts = []string{objectURL + "?partNumber=1", objectURL + "?partNumber=2"}
+					opts.PresignedCompleteMultipart = objectURL + "?Signature=CompleteSignature"
+					opts.Deadline = testDeadline()
+					opts.MaximumSize = tc.maxSize
+
+					require.Less(t, int64(len(tc.testData)), int64(len(opts.PresignedParts))*opts.PartSize, "check part size calculation")
+
+					osStub.InitiateMultipartUpload(test.ObjectPath)
+				}
+
+				ctx, cancel := context.WithCancel(context.Background())
+				defer cancel()
+
+				fh, err := filestore.SaveFileFromReader(ctx, strings.NewReader(tc.testData), tc.objectSize, &opts)
+
+				if tc.expectedErr == nil {
+					require.NoError(t, err)
+					require.NotNil(t, fh)
+				} else {
+					require.True(t, errors.Is(err, tc.expectedErr))
+					require.Nil(t, fh)
+				}
+			}
+		})
+	}
+}
+
 func checkFileHandlerWithFields(t *testing.T, fh *filestore.FileHandler, fields map[string]string, prefix string) {
 	key := func(field string) string {
 		if prefix == "" {

workhorse/internal/git/archive.go

@@ -18,6 +18,7 @@ import (
 	"github.com/golang/protobuf/proto" //lint:ignore SA1019 https://gitlab.com/gitlab-org/gitlab-workhorse/-/issues/274
 
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 
 	"gitlab.com/gitlab-org/gitaly/proto/go/gitalypb"
 
@@ -39,7 +40,7 @@ type archiveParams struct {
 
 var (
 	SendArchive     = &archive{"git-archive:"}
-	gitArchiveCache = prometheus.NewCounterVec(
+	gitArchiveCache = promauto.NewCounterVec(
 		prometheus.CounterOpts{
 			Name: "gitlab_workhorse_git_archive_cache",
 			Help: "Cache hits and misses for 'git archive' streaming",
@@ -48,10 +49,6 @@ var (
 	)
 )
 
-func init() {
-	prometheus.MustRegister(gitArchiveCache)
-}
-
 func (a *archive) Inject(w http.ResponseWriter, r *http.Request, sendData string) {
 	var params archiveParams
 	if err := a.Unpack(&params, sendData); err != nil {

workhorse/internal/git/responsewriter.go

@@ -5,6 +5,7 @@ import (
 	"strconv"
 
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 
 	"gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
 )
@@ -15,12 +16,12 @@ const (
 )
 
 var (
-	gitHTTPSessionsActive = prometheus.NewGauge(prometheus.GaugeOpts{
+	gitHTTPSessionsActive = promauto.NewGauge(prometheus.GaugeOpts{
 		Name: "gitlab_workhorse_git_http_sessions_active",
 		Help: "Number of Git HTTP request-response cycles currently being handled by gitlab-workhorse.",
 	})
 
-	gitHTTPRequests = prometheus.NewCounterVec(
+	gitHTTPRequests = promauto.NewCounterVec(
 		prometheus.CounterOpts{
 			Name: "gitlab_workhorse_git_http_requests",
 			Help: "How many Git HTTP requests have been processed by gitlab-workhorse, partitioned by request type and agent.",
@@ -28,7 +29,7 @@ var (
 		[]string{"method", "code", "service", "agent"},
 	)
 
-	gitHTTPBytes = prometheus.NewCounterVec(
+	gitHTTPBytes = promauto.NewCounterVec(
 		prometheus.CounterOpts{
 			Name: "gitlab_workhorse_git_http_bytes",
 			Help: "How many Git HTTP bytes have been sent by gitlab-workhorse, partitioned by request type, agent and direction.",
@@ -37,12 +38,6 @@ var (
 	)
 )
 
-func init() {
-	prometheus.MustRegister(gitHTTPSessionsActive)
-	prometheus.MustRegister(gitHTTPRequests)
-	prometheus.MustRegister(gitHTTPBytes)
-}
-
 type HttpResponseWriter struct {
 	helper.CountingResponseWriter
 }

workhorse/internal/gitaly/gitaly.go

@@ -10,6 +10,7 @@ import (
 	grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware"
 	grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus"
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 	gitalyauth "gitlab.com/gitlab-org/gitaly/auth"
 	gitalyclient "gitlab.com/gitlab-org/gitaly/client"
 	"gitlab.com/gitlab-org/gitaly/proto/go/gitalypb"
@@ -43,7 +44,7 @@ var (
 		connections: make(map[cacheKey]*grpc.ClientConn),
 	}
 
-	connectionsTotal = prometheus.NewCounterVec(
+	connectionsTotal = promauto.NewCounterVec(
 		prometheus.CounterOpts{
 			Name: "gitlab_workhorse_gitaly_connections_total",
 			Help: "Number of Gitaly connections that have been established",
@@ -52,10 +53,6 @@ var (
 	)
 )
 
-func init() {
-	prometheus.MustRegister(connectionsTotal)
-}
-
 func withOutgoingMetadata(ctx context.Context, features map[string]string) context.Context {
 	md := metadata.New(nil)
 	for k, v := range features {

workhorse/internal/imageresizer/image_resizer.go

@@ -5,7 +5,6 @@ import (
 	"context"
 	"fmt"
 	"io"
-	"math"
 	"net"
 	"net/http"
 	"os"
@@ -17,6 +16,7 @@ import (
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 
 	"gitlab.com/gitlab-org/labkit/correlation"
 	"gitlab.com/gitlab-org/labkit/log"
@@ -96,7 +96,7 @@ const (
 )
 
 var (
-	imageResizeConcurrencyLimitExceeds = prometheus.NewCounter(
+	imageResizeConcurrencyLimitExceeds = promauto.NewCounter(
 		prometheus.CounterOpts{
 			Namespace: namespace,
 			Subsystem: subsystem,
@@ -104,7 +104,7 @@ var (
 			Help:      "Amount of image resizing requests that exceeded the maximum allowed scaler processes",
 		},
 	)
-	imageResizeProcesses = prometheus.NewGauge(
+	imageResizeProcesses = promauto.NewGauge(
 		prometheus.GaugeOpts{
 			Namespace: namespace,
 			Subsystem: subsystem,
@@ -112,7 +112,7 @@ var (
 			Help:      "Amount of image scaler processes working now",
 		},
 	)
-	imageResizeMaxProcesses = prometheus.NewGauge(
+	imageResizeMaxProcesses = promauto.NewGauge(
 		prometheus.GaugeOpts{
 			Namespace: namespace,
 			Subsystem: subsystem,
@@ -120,7 +120,7 @@ var (
 			Help:      "The maximum amount of image scaler processes allowed to run concurrently",
 		},
 	)
-	imageResizeRequests = prometheus.NewCounterVec(
+	imageResizeRequests = promauto.NewCounterVec(
 		prometheus.CounterOpts{
 			Namespace: namespace,
 			Subsystem: subsystem,
@@ -129,7 +129,7 @@ var (
 		},
 		[]string{"status"},
 	)
-	imageResizeDurations = prometheus.NewHistogramVec(
+	imageResizeDurations = promauto.NewHistogramVec(
 		prometheus.HistogramOpts{
 			Namespace: namespace,
 			Subsystem: subsystem,
@@ -154,14 +154,6 @@ const (
 	maxMagicLen = 8                   // 8 first bytes is enough to detect PNG or JPEG
 )
 
-func init() {
-	prometheus.MustRegister(imageResizeConcurrencyLimitExceeds)
-	prometheus.MustRegister(imageResizeProcesses)
-	prometheus.MustRegister(imageResizeMaxProcesses)
-	prometheus.MustRegister(imageResizeRequests)
-	prometheus.MustRegister(imageResizeDurations)
-}
-
 func NewResizer(cfg config.Config) *Resizer {
 	imageResizeMaxProcesses.Set(float64(cfg.ImageResizerConfig.MaxScalerProcs))
 
@@ -279,6 +271,10 @@ func (r *Resizer) tryResizeImage(req *http.Request, f *imageFile, params *resize
 		return f.reader, nil, fmt.Errorf("%d bytes exceeds maximum file size of %d bytes", f.contentLength, cfg.MaxFilesize)
 	}
 
+	if f.contentLength < maxMagicLen {
+		return f.reader, nil, fmt.Errorf("file is too small to resize: %d bytes", f.contentLength)
+	}
+
 	if !r.numScalerProcs.tryIncrement(int32(cfg.MaxScalerProcs)) {
 		return f.reader, nil, fmt.Errorf("too many running scaler processes (%d / %d)", r.numScalerProcs.n, cfg.MaxScalerProcs)
 	}
@@ -289,11 +285,16 @@ func (r *Resizer) tryResizeImage(req *http.Request, f *imageFile, params *resize
 		r.numScalerProcs.decrement()
 	}()
 
-	// Prevents EOF if the file is smaller than 8 bytes
-	bufferSize := int(math.Min(maxMagicLen, float64(f.contentLength)))
-	buffered := bufio.NewReaderSize(f.reader, bufferSize)
+	// Creating buffered Reader is required for us to Peek into first bytes of the image file to detect the format
+	// without advancing the reader (we need to read from the file start in the Scaler binary).
+	// We set `8` as the minimal buffer size by the length of PNG magic bytes sequence (JPEG needs only 2).
+	// In fact, `NewReaderSize` will immediately override it with `16` using its `minReadBufferSize` -
+	// here we are just being explicit about the buffer size required for our code to operate correctly.
+	// Having a reader with such tiny buffer will not hurt the performance during further operations,
+	// because Golang `bufio.Read` avoids double copy: https://golang.org/src/bufio/bufio.go?s=1768:1804#L212
+	buffered := bufio.NewReaderSize(f.reader, maxMagicLen)
 
-	headerBytes, err := buffered.Peek(bufferSize)
+	headerBytes, err := buffered.Peek(maxMagicLen)
 	if err != nil {
 		return buffered, nil, fmt.Errorf("peek stream: %v", err)
 	}

workhorse/internal/imageresizer/image_resizer_test.go

@@ -198,6 +198,29 @@ func TestServeOriginalImageWhenSourceImageFormatIsNotAllowed(t *testing.T) {
 	require.Equal(t, svgImage, responseData, "expected original image")
 }
 
+func TestServeOriginalImageWhenSourceImageIsTooSmall(t *testing.T) {
+	content := []byte("PNG") // 3 bytes only, invalid as PNG/JPEG image
+
+	img, err := ioutil.TempFile("", "*.png")
+	require.NoError(t, err)
+
+	defer img.Close()
+	defer os.Remove(img.Name())
+
+	_, err = img.Write(content)
+	require.NoError(t, err)
+
+	cfg := config.DefaultImageResizerConfig
+	params := resizeParams{Location: img.Name(), ContentType: "image/png", Width: 64}
+
+	resp := requestScaledImage(t, nil, params, cfg)
+	require.Equal(t, http.StatusOK, resp.StatusCode)
+
+	responseData, err := ioutil.ReadAll(resp.Body)
+	require.NoError(t, err)
+	require.Equal(t, content, responseData, "expected original image")
+}
+
 // The Rails applications sends a Base64 encoded JSON string carrying
 // these parameters in an HTTP response header
 func encodeParams(t *testing.T, p *resizeParams) string {

workhorse/internal/objectstore/prometheus.go

 package objectstore
 
-import "github.com/prometheus/client_golang/prometheus"
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
 
 var (
-	objectStorageUploadRequests = prometheus.NewCounterVec(
+	objectStorageUploadRequests = promauto.NewCounterVec(
 		prometheus.CounterOpts{
 			Name: "gitlab_workhorse_object_storage_upload_requests",
 			Help: "How many object storage requests have been processed",
 		},
 		[]string{"status"},
 	)
-	objectStorageUploadsOpen = prometheus.NewGauge(
+	objectStorageUploadsOpen = promauto.NewGauge(
 		prometheus.GaugeOpts{
 			Name: "gitlab_workhorse_object_storage_upload_open",
 			Help: "Describes many object storage requests are open now",
 		},
 	)
-	objectStorageUploadBytes = prometheus.NewCounter(
+	objectStorageUploadBytes = promauto.NewCounter(
 		prometheus.CounterOpts{
 			Name: "gitlab_workhorse_object_storage_upload_bytes",
 			Help: "How many bytes were sent to object storage",
 		},
 	)
-	objectStorageUploadTime = prometheus.NewHistogram(
+	objectStorageUploadTime = promauto.NewHistogram(
 		prometheus.HistogramOpts{
 			Name:    "gitlab_workhorse_object_storage_upload_time",
 			Help:    "How long it took to upload objects",
@@ -34,10 +37,3 @@ var (
 
 	objectStorageUploadTimeBuckets = []float64{.1, .25, .5, 1, 2.5, 5, 10, 25, 50, 100}
 )
-
-func init() {
-	prometheus.MustRegister(
-		objectStorageUploadRequests,
-		objectStorageUploadsOpen,
-		objectStorageUploadBytes)
-}

workhorse/internal/objectstore/uploader.go

@@ -90,6 +90,8 @@ func (u *uploader) Consume(outerCtx context.Context, reader io.Reader, deadline
 		}
 	}
 
+	objectStorageUploadBytes.Add(float64(cr.n))
+
 	return cr.n, nil
 }
 

workhorse/internal/queueing/queue.go

@@ -5,6 +5,7 @@ import (
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 )
 
 type errTooManyRequests struct{ error }
@@ -44,7 +45,7 @@ func newQueueMetrics(name string, timeout time.Duration) *queueMetrics {
 	}
 
 	metrics := &queueMetrics{
-		queueingLimit: prometheus.NewGauge(prometheus.GaugeOpts{
+		queueingLimit: promauto.NewGauge(prometheus.GaugeOpts{
 			Name: "gitlab_workhorse_queueing_limit",
 			Help: "Current limit set for the queueing mechanism",
 			ConstLabels: prometheus.Labels{
@@ -52,7 +53,7 @@ func newQueueMetrics(name string, timeout time.Duration) *queueMetrics {
 			},
 		}),
 
-		queueingQueueLimit: prometheus.NewGauge(prometheus.GaugeOpts{
+		queueingQueueLimit: promauto.NewGauge(prometheus.GaugeOpts{
 			Name: "gitlab_workhorse_queueing_queue_limit",
 			Help: "Current queueLimit set for the queueing mechanism",
 			ConstLabels: prometheus.Labels{
@@ -60,7 +61,7 @@ func newQueueMetrics(name string, timeout time.Duration) *queueMetrics {
 			},
 		}),
 
-		queueingQueueTimeout: prometheus.NewGauge(prometheus.GaugeOpts{
+		queueingQueueTimeout: promauto.NewGauge(prometheus.GaugeOpts{
 			Name: "gitlab_workhorse_queueing_queue_timeout",
 			Help: "Current queueTimeout set for the queueing mechanism",
 			ConstLabels: prometheus.Labels{
@@ -68,7 +69,7 @@ func newQueueMetrics(name string, timeout time.Duration) *queueMetrics {
 			},
 		}),
 
-		queueingBusy: prometheus.NewGauge(prometheus.GaugeOpts{
+		queueingBusy: promauto.NewGauge(prometheus.GaugeOpts{
 			Name: "gitlab_workhorse_queueing_busy",
 			Help: "How many queued requests are now processed",
 			ConstLabels: prometheus.Labels{
@@ -76,7 +77,7 @@ func newQueueMetrics(name string, timeout time.Duration) *queueMetrics {
 			},
 		}),
 
-		queueingWaiting: prometheus.NewGauge(prometheus.GaugeOpts{
+		queueingWaiting: promauto.NewGauge(prometheus.GaugeOpts{
 			Name: "gitlab_workhorse_queueing_waiting",
 			Help: "How many requests are now queued",
 			ConstLabels: prometheus.Labels{
@@ -84,7 +85,7 @@ func newQueueMetrics(name string, timeout time.Duration) *queueMetrics {
 			},
 		}),
 
-		queueingWaitingTime: prometheus.NewHistogram(prometheus.HistogramOpts{
+		queueingWaitingTime: promauto.NewHistogram(prometheus.HistogramOpts{
 			Name: "gitlab_workhorse_queueing_waiting_time",
 			Help: "How many time a request spent in queue",
 			ConstLabels: prometheus.Labels{
@@ -93,7 +94,7 @@ func newQueueMetrics(name string, timeout time.Duration) *queueMetrics {
 			Buckets: waitingTimeBuckets,
 		}),
 
-		queueingErrors: prometheus.NewCounterVec(
+		queueingErrors: promauto.NewCounterVec(
 			prometheus.CounterOpts{
 				Name: "gitlab_workhorse_queueing_errors",
 				Help: "How many times the TooManyRequests or QueueintTimedout errors were returned while queueing, partitioned by error type",
@@ -105,14 +106,6 @@ func newQueueMetrics(name string, timeout time.Duration) *queueMetrics {
 		),
 	}
 
-	prometheus.MustRegister(metrics.queueingLimit)
-	prometheus.MustRegister(metrics.queueingQueueLimit)
-	prometheus.MustRegister(metrics.queueingQueueTimeout)
-	prometheus.MustRegister(metrics.queueingBusy)
-	prometheus.MustRegister(metrics.queueingWaiting)
-	prometheus.MustRegister(metrics.queueingWaitingTime)
-	prometheus.MustRegister(metrics.queueingErrors)
-
 	return metrics
 }
 

workhorse/internal/redis/keywatcher.go

@@ -9,6 +9,7 @@ import (
 	"github.com/gomodule/redigo/redis"
 	"github.com/jpillora/backoff"
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 	"gitlab.com/gitlab-org/labkit/log"
 
 	"gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
@@ -24,13 +25,13 @@ var (
 		Factor: 2,
 		Jitter: true,
 	}
-	keyWatchers = prometheus.NewGauge(
+	keyWatchers = promauto.NewGauge(
 		prometheus.GaugeOpts{
 			Name: "gitlab_workhorse_keywatcher_keywatchers",
 			Help: "The number of keys that is being watched by gitlab-workhorse",
 		},
 	)
-	totalMessages = prometheus.NewCounter(
+	totalMessages = promauto.NewCounter(
 		prometheus.CounterOpts{
 			Name: "gitlab_workhorse_keywatcher_total_messages",
 			Help: "How many messages gitlab-workhorse has received in total on pubsub.",
@@ -38,13 +39,6 @@ var (
 	)
 )
 
-func init() {
-	prometheus.MustRegister(
-		keyWatchers,
-		totalMessages,
-	)
-}
-
 const (
 	keySubChannel = "workhorse:notifications"
 )

workhorse/internal/redis/redis.go

@@ -10,6 +10,7 @@ import (
 	"github.com/FZambia/sentinel"
 	"github.com/gomodule/redigo/redis"
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 	"gitlab.com/gitlab-org/labkit/log"
 
 	"gitlab.com/gitlab-org/gitlab-workhorse/internal/config"
@@ -45,14 +46,14 @@ const (
 )
 
 var (
-	totalConnections = prometheus.NewCounter(
+	totalConnections = promauto.NewCounter(
 		prometheus.CounterOpts{
 			Name: "gitlab_workhorse_redis_total_connections",
 			Help: "How many connections gitlab-workhorse has opened in total. Can be used to track Redis connection rate for this process",
 		},
 	)
 
-	errorCounter = prometheus.NewCounterVec(
+	errorCounter = promauto.NewCounterVec(
 		prometheus.CounterOpts{
 			Name: "gitlab_workhorse_redis_errors",
 			Help: "Counts different types of Redis errors encountered by workhorse, by type and destination (redis, sentinel)",
@@ -61,13 +62,6 @@ var (
 	)
 )
 
-func init() {
-	prometheus.MustRegister(
-		totalConnections,
-		errorCounter,
-	)
-}
-
 func sentinelConn(master string, urls []config.TomlURL) *sentinel.Sentinel {
 	if len(urls) == 0 {
 		return nil

workhorse/internal/senddata/senddata.go

@@ -8,17 +8,18 @@ import (
 	"gitlab.com/gitlab-org/gitlab-workhorse/internal/senddata/contentprocessor"
 
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 )
 
 var (
-	sendDataResponses = prometheus.NewCounterVec(
+	sendDataResponses = promauto.NewCounterVec(
 		prometheus.CounterOpts{
 			Name: "gitlab_workhorse_senddata_responses",
 			Help: "How many HTTP responses have been hijacked by a workhorse senddata injecter",
 		},
 		[]string{"injecter"},
 	)
-	sendDataResponseBytes = prometheus.NewCounterVec(
+	sendDataResponseBytes = promauto.NewCounterVec(
 		prometheus.CounterOpts{
 			Name: "gitlab_workhorse_senddata_response_bytes",
 			Help: "How many bytes have been written by workhorse senddata response injecters",
@@ -27,11 +28,6 @@ var (
 	)
 )
 
-func init() {
-	prometheus.MustRegister(sendDataResponses)
-	prometheus.MustRegister(sendDataResponseBytes)
-}
-
 type sendDataResponseWriter struct {
 	rw        http.ResponseWriter
 	status    int

workhorse/internal/sendfile/sendfile.go

@@ -14,6 +14,7 @@ import (
 	"regexp"
 
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 
 	"gitlab.com/gitlab-org/labkit/log"
 	"gitlab.com/gitlab-org/labkit/mask"
@@ -23,7 +24,7 @@ import (
 )
 
 var (
-	sendFileRequests = prometheus.NewCounterVec(
+	sendFileRequests = promauto.NewCounterVec(
 		prometheus.CounterOpts{
 			Name: "gitlab_workhorse_sendfile_requests",
 			Help: "How many X-Sendfile requests have been processed by gitlab-workhorse, partitioned by sendfile type.",
@@ -31,7 +32,7 @@ var (
 		[]string{"type"},
 	)
 
-	sendFileBytes = prometheus.NewCounterVec(
+	sendFileBytes = promauto.NewCounterVec(
 		prometheus.CounterOpts{
 			Name: "gitlab_workhorse_sendfile_bytes",
 			Help: "How many X-Sendfile bytes have been sent by gitlab-workhorse, partitioned by sendfile type.",
@@ -49,11 +50,6 @@ type sendFileResponseWriter struct {
 	req      *http.Request
 }
 
-func init() {
-	prometheus.MustRegister(sendFileRequests)
-	prometheus.MustRegister(sendFileBytes)
-}
-
 func SendFile(h http.Handler) http.Handler {
 	return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
 		s := &sendFileResponseWriter{

workhorse/internal/sendurl/sendurl.go

@@ -8,6 +8,7 @@ import (
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 
 	"gitlab.com/gitlab-org/labkit/correlation"
 	"gitlab.com/gitlab-org/labkit/log"
@@ -68,20 +69,20 @@ var httpClient = &http.Client{
 }
 
 var (
-	sendURLRequests = prometheus.NewCounterVec(
+	sendURLRequests = promauto.NewCounterVec(
 		prometheus.CounterOpts{
 			Name: "gitlab_workhorse_send_url_requests",
 			Help: "How many send URL requests have been processed",
 		},
 		[]string{"status"},
 	)
-	sendURLOpenRequests = prometheus.NewGauge(
+	sendURLOpenRequests = promauto.NewGauge(
 		prometheus.GaugeOpts{
 			Name: "gitlab_workhorse_send_url_open_requests",
 			Help: "Describes how many send URL requests are open now",
 		},
 	)
-	sendURLBytes = prometheus.NewCounter(
+	sendURLBytes = promauto.NewCounter(
 		prometheus.CounterOpts{
 			Name: "gitlab_workhorse_send_url_bytes",
 			Help: "How many bytes were passed with send URL",
@@ -93,13 +94,6 @@ var (
 	sendURLRequestsSucceeded     = sendURLRequests.WithLabelValues("succeeded")
 )
 
-func init() {
-	prometheus.MustRegister(
-		sendURLRequests,
-		sendURLOpenRequests,
-		sendURLBytes)
-}
-
 func (e *entry) Inject(w http.ResponseWriter, r *http.Request, sendData string) {
 	var params entryParams
 

workhorse/internal/staticpages/error_pages.go

@@ -8,12 +8,13 @@ import (
 	"path/filepath"
 
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 
 	"gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
 )
 
 var (
-	staticErrorResponses = prometheus.NewCounterVec(
+	staticErrorResponses = promauto.NewCounterVec(
 		prometheus.CounterOpts{
 			Name: "gitlab_workhorse_static_error_responses",
 			Help: "How many HTTP responses have been changed to a static error page, by HTTP status code.",
@@ -30,10 +31,6 @@ const (
 	ErrorFormatText
 )
 
-func init() {
-	prometheus.MustRegister(staticErrorResponses)
-}
-
 type errorPageResponseWriter struct {
 	rw       http.ResponseWriter
 	status   int

workhorse/internal/upload/rewrite.go

@@ -11,6 +11,7 @@ import (
 	"strings"
 
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 	"gitlab.com/gitlab-org/labkit/log"
 
 	"gitlab.com/gitlab-org/gitlab-workhorse/internal/api"
@@ -23,7 +24,7 @@ import (
 var ErrInjectedClientParam = errors.New("injected client parameter")
 
 var (
-	multipartUploadRequests = prometheus.NewCounterVec(
+	multipartUploadRequests = promauto.NewCounterVec(
 		prometheus.CounterOpts{
 
 			Name: "gitlab_workhorse_multipart_upload_requests",
@@ -32,7 +33,7 @@ var (
 		[]string{"type"},
 	)
 
-	multipartFileUploadBytes = prometheus.NewCounterVec(
+	multipartFileUploadBytes = promauto.NewCounterVec(
 		prometheus.CounterOpts{
 			Name: "gitlab_workhorse_multipart_upload_bytes",
 			Help: "How many disk bytes of multipart file parts have been successfully written by gitlab-workhorse. Partitioned by type.",
@@ -40,7 +41,7 @@ var (
 		[]string{"type"},
 	)
 
-	multipartFiles = prometheus.NewCounterVec(
+	multipartFiles = promauto.NewCounterVec(
 		prometheus.CounterOpts{
 			Name: "gitlab_workhorse_multipart_upload_files",
 			Help: "How many multipart file parts have been processed by gitlab-workhorse. Partitioned by type.",
@@ -56,12 +57,6 @@ type rewriter struct {
 	finalizedFields map[string]bool
 }
 
-func init() {
-	prometheus.MustRegister(multipartUploadRequests)
-	prometheus.MustRegister(multipartFileUploadBytes)
-	prometheus.MustRegister(multipartFiles)
-}
-
 func rewriteFormFilesFromMultipart(r *http.Request, writer *multipart.Writer, preauth *api.Response, filter MultipartFormProcessor, opts *filestore.SaveFileOpts) error {
 	// Create multipart reader
 	reader, err := r.MultipartReader()

workhorse/internal/upstream/metrics.go

@@ -4,6 +4,7 @@ import (
 	"net/http"
 
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 )
 
@@ -40,14 +41,14 @@ func byteSizeBuckets() []float64 {
 }
 
 var (
-	httpInFlightRequests = prometheus.NewGauge(prometheus.GaugeOpts{
+	httpInFlightRequests = promauto.NewGauge(prometheus.GaugeOpts{
 		Namespace: namespace,
 		Subsystem: httpSubsystem,
 		Name:      "in_flight_requests",
 		Help:      "A gauge of requests currently being served by workhorse.",
 	})
 
-	httpRequestsTotal = prometheus.NewCounterVec(
+	httpRequestsTotal = promauto.NewCounterVec(
 		prometheus.CounterOpts{
 			Namespace: namespace,
 			Subsystem: httpSubsystem,
@@ -57,7 +58,7 @@ var (
 		[]string{"code", "method", "route"},
 	)
 
-	httpRequestDurationSeconds = prometheus.NewHistogramVec(
+	httpRequestDurationSeconds = promauto.NewHistogramVec(
 		prometheus.HistogramOpts{
 			Namespace: namespace,
 			Subsystem: httpSubsystem,
@@ -68,7 +69,7 @@ var (
 		[]string{"code", "method", "route"},
 	)
 
-	httpRequestSizeBytes = prometheus.NewHistogramVec(
+	httpRequestSizeBytes = promauto.NewHistogramVec(
 		prometheus.HistogramOpts{
 			Namespace: namespace,
 			Subsystem: httpSubsystem,
@@ -79,7 +80,7 @@ var (
 		[]string{"code", "method", "route"},
 	)
 
-	httpResponseSizeBytes = prometheus.NewHistogramVec(
+	httpResponseSizeBytes = promauto.NewHistogramVec(
 		prometheus.HistogramOpts{
 			Namespace: namespace,
 			Subsystem: httpSubsystem,
@@ -90,7 +91,7 @@ var (
 		[]string{"code", "method", "route"},
 	)
 
-	httpTimeToWriteHeaderSeconds = prometheus.NewHistogramVec(
+	httpTimeToWriteHeaderSeconds = promauto.NewHistogramVec(
 		prometheus.HistogramOpts{
 			Namespace: namespace,
 			Subsystem: httpSubsystem,
@@ -102,14 +103,6 @@ var (
 	)
 )
 
-func init() {
-	prometheus.MustRegister(httpInFlightRequests)
-	prometheus.MustRegister(httpRequestsTotal)
-	prometheus.MustRegister(httpRequestDurationSeconds)
-	prometheus.MustRegister(httpRequestSizeBytes)
-	prometheus.MustRegister(httpTimeToWriteHeaderSeconds)
-}
-
 func instrumentRoute(next http.Handler, method string, regexpStr string) http.Handler {
 	handler := next
 

workhorse/internal/upstream/routes.go

@@ -192,6 +192,16 @@ func (u *upstream) configureRoutes() {
 	proxy := buildProxy(u.Backend, u.Version, u.RoundTripper, u.Config)
 	cableProxy := proxypkg.NewProxy(u.CableBackend, u.Version, u.CableRoundTripper)
 
+	assetsNotFoundHandler := NotFoundUnless(u.DevelopmentMode, proxy)
+	if u.AltDocumentRoot != "" {
+		altStatic := &staticpages.Static{DocumentRoot: u.AltDocumentRoot}
+		assetsNotFoundHandler = altStatic.ServeExisting(
+			u.URLPrefix,
+			staticpages.CacheExpireMax,
+			NotFoundUnless(u.DevelopmentMode, proxy),
+		)
+	}
+
 	signingTripper := secret.NewRoundTripper(u.RoundTripper, u.Version)
 	signingProxy := buildProxy(u.Backend, u.Version, signingTripper, u.Config)
 
@@ -283,7 +293,7 @@ func (u *upstream) configureRoutes() {
 			static.ServeExisting(
 				u.URLPrefix,
 				staticpages.CacheExpireMax,
-				NotFoundUnless(u.DevelopmentMode, proxy),
+				assetsNotFoundHandler,
 			),
 			withoutTracing(), // Tracing on assets is very noisy
 		),

workhorse/main.go

@@ -143,6 +143,7 @@ func buildConfig(arg0 string, args []string) (*bootConfig, *config.Config, error
 	cfg.Redis = cfgFromFile.Redis
 	cfg.ObjectStorageCredentials = cfgFromFile.ObjectStorageCredentials
 	cfg.ImageResizerConfig = cfgFromFile.ImageResizerConfig
+	cfg.AltDocumentRoot = cfgFromFile.AltDocumentRoot
 
 	return boot, cfg, nil
 }

workhorse/main_test.go

@@ -38,6 +38,7 @@ import (
 const scratchDir = "testdata/scratch"
 const testRepoRoot = "testdata/data"
 const testDocumentRoot = "testdata/public"
+const testAltDocumentRoot = "testdata/alt-public"
 
 var absDocumentRoot string
 
@@ -312,6 +313,72 @@ func TestGzipAssets(t *testing.T) {
 	}
 }
 
+func TestAltDocumentAssets(t *testing.T) {
+	path := "/assets/static.txt"
+	content := "asset"
+	require.NoError(t, setupAltStaticFile(path, content))
+
+	buf := &bytes.Buffer{}
+	gzipWriter := gzip.NewWriter(buf)
+	_, err := gzipWriter.Write([]byte(content))
+	require.NoError(t, err)
+	require.NoError(t, gzipWriter.Close())
+	contentGzip := buf.String()
+	require.NoError(t, setupAltStaticFile(path+".gz", contentGzip))
+
+	proxied := false
+	ts := testhelper.TestServerWithHandler(regexp.MustCompile(`.`), func(w http.ResponseWriter, r *http.Request) {
+		proxied = true
+		w.WriteHeader(404)
+	})
+	defer ts.Close()
+
+	upstreamConfig := newUpstreamConfig(ts.URL)
+	upstreamConfig.AltDocumentRoot = testAltDocumentRoot
+
+	ws := startWorkhorseServerWithConfig(upstreamConfig)
+	defer ws.Close()
+
+	testCases := []struct {
+		desc            string
+		path            string
+		content         string
+		acceptEncoding  string
+		contentEncoding string
+	}{
+		{desc: "plaintext asset", path: path, content: content},
+		{desc: "gzip asset available", path: path, content: contentGzip, acceptEncoding: "gzip", contentEncoding: "gzip"},
+		{desc: "non-existent file", path: "/assets/non-existent"},
+	}
+
+	for _, tc := range testCases {
+		req, err := http.NewRequest("GET", ws.URL+tc.path, nil)
+		require.NoError(t, err)
+
+		if tc.acceptEncoding != "" {
+			req.Header.Set("Accept-Encoding", tc.acceptEncoding)
+		}
+
+		resp, err := http.DefaultTransport.RoundTrip(req)
+		require.NoError(t, err)
+		defer resp.Body.Close()
+		b, err := ioutil.ReadAll(resp.Body)
+		require.NoError(t, err)
+
+		if tc.content != "" {
+			require.Equal(t, 200, resp.StatusCode, "%s: status code", tc.desc)
+			require.Equal(t, tc.content, string(b), "%s: response body", tc.desc)
+			require.False(t, proxied, "%s: should not have made it to backend", tc.desc)
+
+			if tc.contentEncoding != "" {
+				require.Equal(t, tc.contentEncoding, resp.Header.Get("Content-Encoding"))
+			}
+		} else {
+			require.Equal(t, 404, resp.StatusCode, "%s: status code", tc.desc)
+		}
+	}
+}
+
 var sendDataHeader = "Gitlab-Workhorse-Send-Data"
 
 func sendDataResponder(command string, literalJSON string) *httptest.Server {
@@ -576,11 +643,19 @@ func TestPropagateCorrelationIdHeader(t *testing.T) {
 }
 
 func setupStaticFile(fpath, content string) error {
+	return setupStaticFileHelper(fpath, content, testDocumentRoot)
+}
+
+func setupAltStaticFile(fpath, content string) error {
+	return setupStaticFileHelper(fpath, content, testAltDocumentRoot)
+}
+
+func setupStaticFileHelper(fpath, content, directory string) error {
 	cwd, err := os.Getwd()
 	if err != nil {
 		return err
 	}
-	absDocumentRoot = path.Join(cwd, testDocumentRoot)
+	absDocumentRoot = path.Join(cwd, directory)
 	if err := os.MkdirAll(path.Join(absDocumentRoot, path.Dir(fpath)), 0755); err != nil {
 		return err
 	}