Support alternate document root directory

This will be useful for supporting no-downtime upgrades. Admins
attempting to upgrade GitLab via our no-downtime upgrade procedure have
found that CSS and JavaScript often don't load while the upgrade is in
progress. This is because in a mixed deployment scenario with a load
balancer, this can happen:

1. User accesses node version N+1, which then makes a CSS/JS request on
version N.
2. User accesses node version N, which then makes a CSS/JS requests on
version N+1.

In both scenarios, the user gets a 404 since only one version of the
assets exist on a given server.

To fix this, we provide an alternate path where previous and future
assets can be stored.

Relates to https://gitlab.com/gitlab-org/gitlab-workhorse/-/issues/304

.gitignore

 testdata/data
 testdata/scratch
 testdata/public
+testdata/alt-public
 /gitlab-workhorse
 /gitlab-resize-image
 /gitlab-zip-cat

changelogs/unreleased/sh-support-alt-document-root.yml

+---
+title: Support alternate document root directory
+merge_request: 626
+author:
+type: added

config.toml.example

+# alt_document_root = '/home/git/public/assets'
+
 [redis]
 URL = "unix:/home/git/gitlab/redis/redis.socket"
 

internal/config/config.go

@@ -105,6 +105,7 @@ type Config struct {
 	ObjectStorageCredentials ObjectStorageCredentials `toml:"object_storage"`
 	PropagateCorrelationID   bool                     `toml:"-"`
 	ImageResizerConfig       ImageResizerConfig       `toml:"image_resizer"`
+	AltDocumentRoot          string                   `toml:"alt_document_root"`
 }
 
 var DefaultImageResizerConfig = ImageResizerConfig{

internal/config/config_test.go

@@ -21,6 +21,7 @@ func TestLoadEmptyConfig(t *testing.T) {
 	cfg, err := LoadConfig(config)
 	require.NoError(t, err)
 
+	require.Empty(t, cfg.AltDocumentRoot)
 	require.Equal(t, cfg.ImageResizerConfig.MaxFilesize, uint64(250000))
 	require.GreaterOrEqual(t, cfg.ImageResizerConfig.MaxScalerProcs, uint32(2))
 
@@ -97,3 +98,14 @@ max_filesize = 350000
 
 	require.Equal(t, expected, cfg.ImageResizerConfig)
 }
+
+func TestAltDocumentConfig(t *testing.T) {
+	config := `
+alt_document_root = "/path/to/documents"
+`
+
+	cfg, err := LoadConfig(config)
+	require.NoError(t, err)
+
+	require.Equal(t, "/path/to/documents", cfg.AltDocumentRoot)
+}

internal/upstream/routes.go

@@ -192,6 +192,16 @@ func (u *upstream) configureRoutes() {
 	proxy := buildProxy(u.Backend, u.Version, u.RoundTripper, u.Config)
 	cableProxy := proxypkg.NewProxy(u.CableBackend, u.Version, u.CableRoundTripper)
 
+	assetsNotFoundHandler := NotFoundUnless(u.DevelopmentMode, proxy)
+	if u.AltDocumentRoot != "" {
+		altStatic := &staticpages.Static{DocumentRoot: u.AltDocumentRoot}
+		assetsNotFoundHandler = altStatic.ServeExisting(
+			u.URLPrefix,
+			staticpages.CacheExpireMax,
+			NotFoundUnless(u.DevelopmentMode, proxy),
+		)
+	}
+
 	signingTripper := secret.NewRoundTripper(u.RoundTripper, u.Version)
 	signingProxy := buildProxy(u.Backend, u.Version, signingTripper, u.Config)
 
@@ -280,7 +290,7 @@ func (u *upstream) configureRoutes() {
 			static.ServeExisting(
 				u.URLPrefix,
 				staticpages.CacheExpireMax,
-				NotFoundUnless(u.DevelopmentMode, proxy),
+				assetsNotFoundHandler,
 			),
 			withoutTracing(), // Tracing on assets is very noisy
 		),

main.go

@@ -143,6 +143,7 @@ func buildConfig(arg0 string, args []string) (*bootConfig, *config.Config, error
 	cfg.Redis = cfgFromFile.Redis
 	cfg.ObjectStorageCredentials = cfgFromFile.ObjectStorageCredentials
 	cfg.ImageResizerConfig = cfgFromFile.ImageResizerConfig
+	cfg.AltDocumentRoot = cfgFromFile.AltDocumentRoot
 
 	return boot, cfg, nil
 }

main_test.go

@@ -38,6 +38,7 @@ import (
 const scratchDir = "testdata/scratch"
 const testRepoRoot = "testdata/data"
 const testDocumentRoot = "testdata/public"
+const testAltDocumentRoot = "testdata/alt-public"
 
 var absDocumentRoot string
 
@@ -312,6 +313,72 @@ func TestGzipAssets(t *testing.T) {
 	}
 }
 
+func TestAltDocumentAssets(t *testing.T) {
+	path := "/assets/static.txt"
+	content := "asset"
+	require.NoError(t, setupAltStaticFile(path, content))
+
+	buf := &bytes.Buffer{}
+	gzipWriter := gzip.NewWriter(buf)
+	_, err := gzipWriter.Write([]byte(content))
+	require.NoError(t, err)
+	require.NoError(t, gzipWriter.Close())
+	contentGzip := buf.String()
+	require.NoError(t, setupAltStaticFile(path+".gz", contentGzip))
+
+	proxied := false
+	ts := testhelper.TestServerWithHandler(regexp.MustCompile(`.`), func(w http.ResponseWriter, r *http.Request) {
+		proxied = true
+		w.WriteHeader(404)
+	})
+	defer ts.Close()
+
+	upstreamConfig := newUpstreamConfig(ts.URL)
+	upstreamConfig.AltDocumentRoot = testAltDocumentRoot
+
+	ws := startWorkhorseServerWithConfig(upstreamConfig)
+	defer ws.Close()
+
+	testCases := []struct {
+		desc            string
+		path            string
+		content         string
+		acceptEncoding  string
+		contentEncoding string
+	}{
+		{desc: "plaintext asset", path: path, content: content},
+		{desc: "gzip asset available", path: path, content: contentGzip, acceptEncoding: "gzip", contentEncoding: "gzip"},
+		{desc: "non-existent file", path: "/assets/non-existent"},
+	}
+
+	for _, tc := range testCases {
+		req, err := http.NewRequest("GET", ws.URL+tc.path, nil)
+		require.NoError(t, err)
+
+		if tc.acceptEncoding != "" {
+			req.Header.Set("Accept-Encoding", tc.acceptEncoding)
+		}
+
+		resp, err := http.DefaultTransport.RoundTrip(req)
+		require.NoError(t, err)
+		defer resp.Body.Close()
+		b, err := ioutil.ReadAll(resp.Body)
+		require.NoError(t, err)
+
+		if tc.content != "" {
+			require.Equal(t, 200, resp.StatusCode, "%s: status code", tc.desc)
+			require.Equal(t, tc.content, string(b), "%s: response body", tc.desc)
+			require.False(t, proxied, "%s: should not have made it to backend", tc.desc)
+
+			if tc.contentEncoding != "" {
+				require.Equal(t, tc.contentEncoding, resp.Header.Get("Content-Encoding"))
+			}
+		} else {
+			require.Equal(t, 404, resp.StatusCode, "%s: status code", tc.desc)
+		}
+	}
+}
+
 var sendDataHeader = "Gitlab-Workhorse-Send-Data"
 
 func sendDataResponder(command string, literalJSON string) *httptest.Server {
@@ -576,11 +643,19 @@ func TestPropagateCorrelationIdHeader(t *testing.T) {
 }
 
 func setupStaticFile(fpath, content string) error {
+	return setupStaticFileHelper(fpath, content, testDocumentRoot)
+}
+
+func setupAltStaticFile(fpath, content string) error {
+	return setupStaticFileHelper(fpath, content, testAltDocumentRoot)
+}
+
+func setupStaticFileHelper(fpath, content, directory string) error {
 	cwd, err := os.Getwd()
 	if err != nil {
 		return err
 	}
-	absDocumentRoot = path.Join(cwd, testDocumentRoot)
+	absDocumentRoot = path.Join(cwd, directory)
 	if err := os.MkdirAll(path.Join(absDocumentRoot, path.Dir(fpath)), 0755); err != nil {
 		return err
 	}