Introduce a `send-url:` method that allows to serve remote HTTP/GET file, like S3-based file

internal/sendurl/sendurl.go

+package sendurl
+
+import (
+	"fmt"
+	"io"
+	"log"
+	"net"
+	"net/http"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+
+	"gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
+	"gitlab.com/gitlab-org/gitlab-workhorse/internal/senddata"
+)
+
+type entry struct{ senddata.Prefix }
+
+type entryParams struct {
+	URL            string
+	AllowRedirects bool
+}
+
+var SendURL = &entry{"send-url:"}
+
+var rangeHeaderKeys = []string{
+	"If-Match",
+	"If-Unmodified-Since",
+	"If-None-Match",
+	"If-Modified-Since",
+	"If-Range",
+	"Range",
+}
+
+// httpTransport defines a http.Transport with values
+// that are more restrictive than for http.DefaultTransport,
+// they define shorter TLS Handshake, and more agressive connection closing
+// to prevent the connection hanging and reduce FD usage
+var httpTransport = &http.Transport{
+	Proxy: http.ProxyFromEnvironment,
+	DialContext: (&net.Dialer{
+		Timeout:   30 * time.Second,
+		KeepAlive: 10 * time.Second,
+	}).DialContext,
+	MaxIdleConns:          2,
+	IdleConnTimeout:       30 * time.Second,
+	TLSHandshakeTimeout:   10 * time.Second,
+	ExpectContinueTimeout: 10 * time.Second,
+	ResponseHeaderTimeout: 30 * time.Second,
+}
+
+var httpClient = &http.Client{
+	Transport: httpTransport,
+}
+
+var (
+	sendURLRequests = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "gitlab_workhorse_send_url_requests",
+			Help: "How many send URL requests have been processed",
+		},
+		[]string{"status"},
+	)
+	sendURLOpenRequests = prometheus.NewGauge(
+		prometheus.GaugeOpts{
+			Name: "gitlab_workhorse_send_url_open_requests",
+			Help: "Describes how many send URL requests are open now",
+		},
+	)
+	sendURLBytes = prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Name: "gitlab_workhorse_send_url_bytes",
+			Help: "How many bytes were passed with send URL",
+		},
+	)
+
+	sendURLRequestsInvalidData   = sendURLRequests.WithLabelValues("invalid-data")
+	sendURLRequestsRequestFailed = sendURLRequests.WithLabelValues("request-failed")
+	sendURLRequestsSucceeded     = sendURLRequests.WithLabelValues("succeeded")
+)
+
+func init() {
+	prometheus.MustRegister(
+		sendURLRequests,
+		sendURLOpenRequests,
+		sendURLBytes)
+}
+
+func (e *entry) Inject(w http.ResponseWriter, r *http.Request, sendData string) {
+	var params entryParams
+
+	sendURLOpenRequests.Inc()
+	defer sendURLOpenRequests.Dec()
+
+	if err := e.Unpack(&params, sendData); err != nil {
+		helper.Fail500(w, r, fmt.Errorf("SendURL: unpack sendData: %v", err))
+		return
+	}
+
+	log.Printf("SendURL: sending %q for %q", helper.ScrubURLParams(params.URL), r.URL.Path)
+
+	if params.URL == "" {
+		sendURLRequestsInvalidData.Inc()
+		helper.Fail500(w, r, fmt.Errorf("SendURL: URL is empty"))
+		return
+	}
+
+	// create new request and copy range headers
+	newReq, err := http.NewRequest("GET", params.URL, nil)
+	if err != nil {
+		sendURLRequestsInvalidData.Inc()
+		helper.Fail500(w, r, fmt.Errorf("SendURL: NewRequest: %v", err))
+		return
+	}
+
+	for _, header := range rangeHeaderKeys {
+		newReq.Header[header] = r.Header[header]
+	}
+
+	// execute new request
+	var resp *http.Response
+	if params.AllowRedirects {
+		resp, err = httpClient.Do(newReq)
+	} else {
+		resp, err = httpTransport.RoundTrip(newReq)
+	}
+	if err != nil {
+		sendURLRequestsRequestFailed.Inc()
+		helper.Fail500(w, r, fmt.Errorf("SendURL: Do request: %v", err))
+		return
+	}
+
+	// copy response headers and body
+	for key, value := range resp.Header {
+		w.Header()[key] = value
+	}
+	w.WriteHeader(resp.StatusCode)
+
+	defer resp.Body.Close()
+	n, err := io.Copy(w, resp.Body)
+	sendURLBytes.Add(float64(n))
+
+	if err != nil {
+		sendURLRequestsRequestFailed.Inc()
+		helper.Fail500(w, r, fmt.Errorf("SendURL: Copy response: %v", err))
+		return
+	}
+
+	sendURLRequestsSucceeded.Inc()
+}

internal/sendurl/sendurl_test.go

+package sendurl
+
+import (
+	"encoding/base64"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"gitlab.com/gitlab-org/gitlab-workhorse/internal/testhelper"
+)
+
+const testData = `123456789012345678901234567890`
+const testDataEtag = `W/"myetag"`
+
+func testEntryServer(t *testing.T, requestURL string, httpHeaders http.Header, allowRedirects bool) *httptest.ResponseRecorder {
+	requestHandler := func(w http.ResponseWriter, r *http.Request) {
+		require.Equal(t, "GET", r.Method)
+
+		url := r.URL.String() + "/file"
+		jsonParams := fmt.Sprintf(`{"URL":%q,"AllowRedirects":%s}`,
+			url, strconv.FormatBool(allowRedirects))
+		data := base64.URLEncoding.EncodeToString([]byte(jsonParams))
+
+		// The server returns a Content-Disposition
+		w.Header().Set("Content-Disposition", "attachment; filename=\"archive.txt\"")
+
+		SendURL.Inject(w, r, data)
+	}
+	serveFile := func(w http.ResponseWriter, r *http.Request) {
+		require.Equal(t, "GET", r.Method)
+
+		tempFile, err := ioutil.TempFile("", "download_file")
+		require.NoError(t, err)
+		require.NoError(t, os.Remove(tempFile.Name()))
+		defer tempFile.Close()
+		_, err = tempFile.Write([]byte(testData))
+		require.NoError(t, err)
+
+		w.Header().Set("Etag", testDataEtag)
+
+		http.ServeContent(w, r, "archive.txt", time.Now(), tempFile)
+	}
+	redirectFile := func(w http.ResponseWriter, r *http.Request) {
+		require.Equal(t, "GET", r.Method)
+		http.Redirect(w, r, r.URL.String()+"/download", http.StatusTemporaryRedirect)
+	}
+
+	mux := http.NewServeMux()
+	mux.HandleFunc("/get/request", requestHandler)
+	mux.HandleFunc("/get/request/file", serveFile)
+	mux.HandleFunc("/get/redirect", requestHandler)
+	mux.HandleFunc("/get/redirect/file", redirectFile)
+	mux.HandleFunc("/get/redirect/file/download", serveFile)
+	mux.HandleFunc("/get/file-not-existing", requestHandler)
+
+	server := httptest.NewServer(mux)
+	defer server.Close()
+
+	httpRequest, err := http.NewRequest("GET", server.URL+requestURL, nil)
+	require.NoError(t, err)
+	if httpHeaders != nil {
+		httpRequest.Header = httpHeaders
+	}
+
+	response := httptest.NewRecorder()
+	mux.ServeHTTP(response, httpRequest)
+	return response
+}
+
+func TestDownloadingUsingSendURL(t *testing.T) {
+	response := testEntryServer(t, "/get/request", nil, false)
+	testhelper.AssertResponseCode(t, response, http.StatusOK)
+
+	testhelper.AssertResponseWriterHeader(t, response,
+		"Content-Type",
+		"text/plain; charset=utf-8")
+	testhelper.AssertResponseWriterHeader(t, response,
+		"Content-Disposition",
+		"attachment; filename=\"archive.txt\"")
+
+	testhelper.AssertResponseBody(t, response, testData)
+}
+
+func TestDownloadingAChunkOfDataWithSendURL(t *testing.T) {
+	httpHeaders := http.Header{
+		"Range": []string{
+			"bytes=1-2",
+		},
+	}
+
+	response := testEntryServer(t, "/get/request", httpHeaders, false)
+	testhelper.AssertResponseCode(t, response, http.StatusPartialContent)
+
+	testhelper.AssertResponseWriterHeader(t, response,
+		"Content-Type",
+		"text/plain; charset=utf-8")
+	testhelper.AssertResponseWriterHeader(t, response,
+		"Content-Disposition",
+		"attachment; filename=\"archive.txt\"")
+	testhelper.AssertResponseWriterHeader(t, response,
+		"Content-Range",
+		"bytes 1-2/30")
+
+	testhelper.AssertResponseBody(t, response, "23")
+}
+
+func TestAccessingAlreadyDownloadedFileWithSendURL(t *testing.T) {
+	httpHeaders := http.Header{
+		"If-None-Match": []string{testDataEtag},
+	}
+
+	response := testEntryServer(t, "/get/request", httpHeaders, false)
+	testhelper.AssertResponseCode(t, response, http.StatusNotModified)
+}
+
+func TestAccessingRedirectWithSendURL(t *testing.T) {
+	response := testEntryServer(t, "/get/redirect", nil, false)
+	testhelper.AssertResponseCode(t, response, http.StatusTemporaryRedirect)
+}
+
+func TestAccessingAllowedRedirectWithSendURL(t *testing.T) {
+	response := testEntryServer(t, "/get/redirect", nil, true)
+	testhelper.AssertResponseCode(t, response, http.StatusOK)
+
+	testhelper.AssertResponseWriterHeader(t, response,
+		"Content-Type",
+		"text/plain; charset=utf-8")
+	testhelper.AssertResponseWriterHeader(t, response,
+		"Content-Disposition",
+		"attachment; filename=\"archive.txt\"")
+}
+
+func TestAccessingAllowedRedirectWithChunkOfDataWithSendURL(t *testing.T) {
+	httpHeaders := http.Header{
+		"Range": []string{
+			"bytes=1-2",
+		},
+	}
+
+	response := testEntryServer(t, "/get/redirect", httpHeaders, true)
+	testhelper.AssertResponseCode(t, response, http.StatusPartialContent)
+
+	testhelper.AssertResponseWriterHeader(t, response,
+		"Content-Type",
+		"text/plain; charset=utf-8")
+	testhelper.AssertResponseWriterHeader(t, response,
+		"Content-Disposition",
+		"attachment; filename=\"archive.txt\"")
+	testhelper.AssertResponseWriterHeader(t, response,
+		"Content-Range",
+		"bytes 1-2/30")
+
+	testhelper.AssertResponseBody(t, response, "23")
+}
+
+func TestDownloadingNonExistingFileUsingSendURL(t *testing.T) {
+	response := testEntryServer(t, "/invalid/path", nil, false)
+	testhelper.AssertResponseCode(t, response, http.StatusNotFound)
+}
+
+func TestDownloadingNonExistingRemoteFileWithSendURL(t *testing.T) {
+	response := testEntryServer(t, "/get/file-not-existing", nil, false)
+	testhelper.AssertResponseCode(t, response, http.StatusNotFound)
+}

internal/upstream/routes.go

@@ -20,6 +20,7 @@ import (
 	"gitlab.com/gitlab-org/gitlab-workhorse/internal/redis"
 	"gitlab.com/gitlab-org/gitlab-workhorse/internal/senddata"
 	"gitlab.com/gitlab-org/gitlab-workhorse/internal/sendfile"
+	"gitlab.com/gitlab-org/gitlab-workhorse/internal/sendurl"
 	"gitlab.com/gitlab-org/gitlab-workhorse/internal/staticpages"
 	"gitlab.com/gitlab-org/gitlab-workhorse/internal/terminal"
 	"gitlab.com/gitlab-org/gitlab-workhorse/internal/upload"
@@ -140,6 +141,7 @@ func (u *Upstream) configureRoutes() {
 		git.SendDiff,
 		git.SendPatch,
 		artifacts.SendEntry,
+		sendurl.SendURL,
 	)
 
 	uploadAccelerateProxy := upload.Accelerate(path.Join(u.DocumentRoot, "uploads/tmp"), proxy)

main_test.go

@@ -440,6 +440,24 @@ func TestArtifactsGetSingleFile(t *testing.T) {
 	assertNginxResponseBuffering(t, "no", resp, "GET %q: nginx response buffering", resourcePath)
 }
 
+func TestSendURLForArtifacts(t *testing.T) {
+	fileContents := "12345678901234567890\n"
+
+	server := httptest.NewServer(http.FileServer(http.Dir("testdata")))
+	defer server.Close()
+
+	// We manually created this txt file in the gitlab-workhorse Git repository
+	url := server.URL + "/test-file.txt"
+	jsonParams := fmt.Sprintf(`{"URL":%q}`, url)
+
+	resourcePath := `/namespace/project/builds/123/artifacts/file/download`
+	resp, body, err := doSendDataRequest(resourcePath, "send-url", jsonParams)
+	require.NoError(t, err)
+
+	assert.Equal(t, 200, resp.StatusCode, "GET %q: status code", resourcePath)
+	assert.Equal(t, fileContents, string(body), "GET %q: response body", resourcePath)
+}
+
 func TestGetGitBlob(t *testing.T) {
 	blobId := "50b27c6518be44c42c4d87966ae2481ce895624c" // the LICENSE file in the test repository
 	blobLength := 1075

testdata/test-file.txt

+12345678901234567890