Commit 199976a4 authored by Kamil Trzciński's avatar Kamil Trzciński

Merge branch 'artifacts' into 'master'

Implement multipart form rewriting to support upload offloading

1. This parses multipart form data and saves all found files as files in TempPath. TempPath is received from Rails by calling authorize request. The rewritten multipart form data contains `file.path` where the temporary file is stored, and `file.name` the original name of file as stored in Content-Disposition. The temporary file is removed afterwards, if it's not consumed by GitLab Rails. If the body is not multipart, forward request.

2. All artifacts downloads are offloaded by exposing X-Sendfile-Type extension.


See merge request !5
parents 2cc94bc8 86d2264c
package main
func artifactsAuthorizeHandler(handleFunc serviceHandleFunc) serviceHandleFunc {
return preAuthorizeHandler(handleFunc, "/authorize")
}
package main
import (
"fmt"
"net/http"
"net/http/httptest"
"regexp"
"testing"
)
func okHandler(w http.ResponseWriter, r *gitRequest) {
w.WriteHeader(201)
fmt.Fprint(w, "{\"status\":\"ok\"}")
}
func runPreAuthorizeHandler(t *testing.T, suffix string, url *regexp.Regexp, authorizationResponse interface{}, returnCode, expectedCode int) *httptest.ResponseRecorder {
// Prepare test server and backend
ts := testAuthServer(url, returnCode, authorizationResponse)
defer ts.Close()
// Create http request
httpRequest, err := http.NewRequest("GET", "/address", nil)
if err != nil {
t.Fatal(err)
}
request := gitRequest{
Request: httpRequest,
u: newUpstream(ts.URL, nil),
}
response := httptest.NewRecorder()
preAuthorizeHandler(okHandler, suffix)(response, &request)
assertResponseCode(t, response, expectedCode)
return response
}
func TestPreAuthorizeHappyPath(t *testing.T) {
runPreAuthorizeHandler(
t, "/authorize",
regexp.MustCompile(`/authorize\z`),
&authorizationResponse{},
200, 201)
}
func TestPreAuthorizeSuffix(t *testing.T) {
runPreAuthorizeHandler(
t, "/different-authorize",
regexp.MustCompile(`/authorize\z`),
&authorizationResponse{},
200, 404)
}
func TestPreAuthorizeJsonFailure(t *testing.T) {
runPreAuthorizeHandler(
t, "/authorize",
regexp.MustCompile(`/authorize\z`),
"not-json",
200, 500)
}
...@@ -5,7 +5,6 @@ In this file we handle the Git 'smart HTTP' protocol ...@@ -5,7 +5,6 @@ In this file we handle the Git 'smart HTTP' protocol
package main package main
import ( import (
"compress/gzip"
"fmt" "fmt"
"io" "io"
"net/http" "net/http"
...@@ -58,7 +57,6 @@ func handleGetInfoRefs(w http.ResponseWriter, r *gitRequest) { ...@@ -58,7 +57,6 @@ func handleGetInfoRefs(w http.ResponseWriter, r *gitRequest) {
} }
func handlePostRPC(w http.ResponseWriter, r *gitRequest) { func handlePostRPC(w http.ResponseWriter, r *gitRequest) {
var body io.ReadCloser
var err error var err error
// Get Git action from URL // Get Git action from URL
...@@ -69,18 +67,6 @@ func handlePostRPC(w http.ResponseWriter, r *gitRequest) { ...@@ -69,18 +67,6 @@ func handlePostRPC(w http.ResponseWriter, r *gitRequest) {
return return
} }
// The client request body may have been gzipped.
if r.Header.Get("Content-Encoding") == "gzip" {
body, err = gzip.NewReader(r.Body)
if err != nil {
fail500(w, "handlePostRPC", err)
return
}
} else {
body = r.Body
}
defer body.Close()
// Prepare our Git subprocess // Prepare our Git subprocess
cmd := gitCommand(r.GL_ID, "git", subCommand(action), "--stateless-rpc", r.RepoPath) cmd := gitCommand(r.GL_ID, "git", subCommand(action), "--stateless-rpc", r.RepoPath)
stdout, err := cmd.StdoutPipe() stdout, err := cmd.StdoutPipe()
...@@ -102,7 +88,7 @@ func handlePostRPC(w http.ResponseWriter, r *gitRequest) { ...@@ -102,7 +88,7 @@ func handlePostRPC(w http.ResponseWriter, r *gitRequest) {
defer cleanUpProcessGroup(cmd) // Ensure brute force subprocess clean-up defer cleanUpProcessGroup(cmd) // Ensure brute force subprocess clean-up
// Write the client request body to Git's standard input // Write the client request body to Git's standard input
if _, err := io.Copy(stdin, body); err != nil { if _, err := io.Copy(stdin, r.Body); err != nil {
fail500(w, "handlePostRPC write to subprocess", err) fail500(w, "handlePostRPC write to subprocess", err)
return return
} }
...@@ -112,9 +98,6 @@ func handlePostRPC(w http.ResponseWriter, r *gitRequest) { ...@@ -112,9 +98,6 @@ func handlePostRPC(w http.ResponseWriter, r *gitRequest) {
// It may take a while before we return and the deferred closes happen // It may take a while before we return and the deferred closes happen
// so let's free up some resources already. // so let's free up some resources already.
r.Body.Close() r.Body.Close()
// If the body was compressed, body != r.Body and this frees up the
// gzip.Reader.
body.Close()
// Start writing the response // Start writing the response
w.Header().Add("Content-Type", fmt.Sprintf("application/x-%s-result", action)) w.Header().Add("Content-Type", fmt.Sprintf("application/x-%s-result", action))
......
package main
import (
"compress/gzip"
"fmt"
"io"
"net/http"
)
func contentEncodingHandler(handleFunc serviceHandleFunc) serviceHandleFunc {
return func(w http.ResponseWriter, r *gitRequest) {
var body io.ReadCloser
var err error
// The client request body may have been gzipped.
contentEncoding := r.Header.Get("Content-Encoding")
switch contentEncoding {
case "":
body = r.Body
case "gzip":
body, err = gzip.NewReader(r.Body)
default:
err = fmt.Errorf("unsupported content encoding: %s", contentEncoding)
}
if err != nil {
fail500(w, "contentEncodingHandler", err)
return
}
defer body.Close()
r.Body = body
r.Header.Del("Content-Encoding")
handleFunc(w, r)
}
}
package main
import (
"bytes"
"compress/gzip"
"fmt"
"io/ioutil"
"net/http"
"net/http/httptest"
"reflect"
"testing"
)
func TestGzipEncoding(t *testing.T) {
resp := httptest.NewRecorder()
var b bytes.Buffer
w := gzip.NewWriter(&b)
fmt.Fprint(w, "test")
w.Close()
body := ioutil.NopCloser(&b)
req, err := http.NewRequest("POST", "http://address/test", body)
if err != nil {
t.Fatal(err)
}
req.Header.Set("Content-Encoding", "gzip")
request := gitRequest{Request: req}
contentEncodingHandler(func(w http.ResponseWriter, r *gitRequest) {
if _, ok := r.Body.(*gzip.Reader); !ok {
t.Fatal("Expected gzip reader for body, but it's:", reflect.TypeOf(r.Body))
}
if r.Header.Get("Content-Encoding") != "" {
t.Fatal("Content-Encoding should be deleted")
}
})(resp, &request)
assertResponseCode(t, resp, 200)
}
func TestNoEncoding(t *testing.T) {
resp := httptest.NewRecorder()
var b bytes.Buffer
body := ioutil.NopCloser(&b)
req, err := http.NewRequest("POST", "http://address/test", body)
if err != nil {
t.Fatal(err)
}
req.Header.Set("Content-Encoding", "")
request := gitRequest{Request: req}
contentEncodingHandler(func(w http.ResponseWriter, r *gitRequest) {
if r.Body != body {
t.Fatal("Expected the same body")
}
if r.Header.Get("Content-Encoding") != "" {
t.Fatal("Content-Encoding should be deleted")
}
})(resp, &request)
assertResponseCode(t, resp, 200)
}
func TestInvalidEncoding(t *testing.T) {
resp := httptest.NewRecorder()
req, err := http.NewRequest("POST", "http://address/test", nil)
if err != nil {
t.Fatal(err)
}
req.Header.Set("Content-Encoding", "application/unknown")
request := gitRequest{Request: req}
contentEncodingHandler(func(w http.ResponseWriter, r *gitRequest) {
t.Fatal("it shouldn't be executed")
})(resp, &request)
assertResponseCode(t, resp, 500)
}
...@@ -6,13 +6,22 @@ package main ...@@ -6,13 +6,22 @@ package main
import ( import (
"fmt" "fmt"
"io"
"io/ioutil"
"log" "log"
"net/http" "net/http"
"net/url"
"os" "os"
"os/exec" "os/exec"
"strings"
"syscall" "syscall"
) )
func fail400(w http.ResponseWriter, context string, err error) {
http.Error(w, "Bad request", 400)
logContext(context, err)
}
func fail500(w http.ResponseWriter, context string, err error) { func fail500(w http.ResponseWriter, context string, err error) {
http.Error(w, "Internal server error", 500) http.Error(w, "Internal server error", 500)
logContext(context, err) logContext(context, err)
...@@ -52,3 +61,21 @@ func cleanUpProcessGroup(cmd *exec.Cmd) { ...@@ -52,3 +61,21 @@ func cleanUpProcessGroup(cmd *exec.Cmd) {
// reap our child process // reap our child process
cmd.Wait() cmd.Wait()
} }
func forwardResponseToClient(w http.ResponseWriter, r *http.Response) {
log.Printf("PROXY:%s %q %d", r.Request.Method, r.Request.URL, r.StatusCode)
for k, v := range r.Header {
w.Header()[k] = v
}
w.WriteHeader(r.StatusCode)
io.Copy(w, r.Body)
}
func setHttpPostForm(r *http.Request, values url.Values) {
dataBuffer := strings.NewReader(values.Encode())
r.Body = ioutil.NopCloser(dataBuffer)
r.ContentLength = int64(dataBuffer.Len())
r.Header.Set("Content-Type", "application/x-www-form-urlencoded")
}
package main
import (
"net/http/httptest"
"testing"
)
func assertResponseCode(t *testing.T, response *httptest.ResponseRecorder, expectedCode int) {
if response.Code != expectedCode {
t.Fatalf("for HTTP request expected to get %d, got %d instead", expectedCode, response.Code)
}
}
...@@ -2,6 +2,7 @@ package main ...@@ -2,6 +2,7 @@ package main
import ( import (
"bytes" "bytes"
"encoding/json"
"fmt" "fmt"
"io/ioutil" "io/ioutil"
"log" "log"
...@@ -10,6 +11,7 @@ import ( ...@@ -10,6 +11,7 @@ import (
"os" "os"
"os/exec" "os/exec"
"path" "path"
"regexp"
"testing" "testing"
"time" "time"
) )
...@@ -29,7 +31,7 @@ func TestAllowedClone(t *testing.T) { ...@@ -29,7 +31,7 @@ func TestAllowedClone(t *testing.T) {
} }
// Prepare test server and backend // Prepare test server and backend
ts := testAuthServer(200, gitOkBody(t)) ts := testAuthServer(nil, 200, gitOkBody(t))
defer ts.Close() defer ts.Close()
ws := startWorkhorseServer(ts.URL) ws := startWorkhorseServer(ts.URL)
defer ws.Close() defer ws.Close()
...@@ -51,7 +53,7 @@ func TestDeniedClone(t *testing.T) { ...@@ -51,7 +53,7 @@ func TestDeniedClone(t *testing.T) {
} }
// Prepare test server and backend // Prepare test server and backend
ts := testAuthServer(403, "Access denied") ts := testAuthServer(nil, 403, "Access denied")
defer ts.Close() defer ts.Close()
ws := startWorkhorseServer(ts.URL) ws := startWorkhorseServer(ts.URL)
defer ws.Close() defer ws.Close()
...@@ -69,7 +71,7 @@ func TestAllowedPush(t *testing.T) { ...@@ -69,7 +71,7 @@ func TestAllowedPush(t *testing.T) {
preparePushRepo(t) preparePushRepo(t)
// Prepare the test server and backend // Prepare the test server and backend
ts := testAuthServer(200, gitOkBody(t)) ts := testAuthServer(nil, 200, gitOkBody(t))
defer ts.Close() defer ts.Close()
ws := startWorkhorseServer(ts.URL) ws := startWorkhorseServer(ts.URL)
defer ws.Close() defer ws.Close()
...@@ -84,7 +86,7 @@ func TestDeniedPush(t *testing.T) { ...@@ -84,7 +86,7 @@ func TestDeniedPush(t *testing.T) {
preparePushRepo(t) preparePushRepo(t)
// Prepare the test server and backend // Prepare the test server and backend
ts := testAuthServer(403, "Access denied") ts := testAuthServer(nil, 403, "Access denied")
defer ts.Close() defer ts.Close()
ws := startWorkhorseServer(ts.URL) ws := startWorkhorseServer(ts.URL)
defer ws.Close() defer ws.Close()
...@@ -104,7 +106,7 @@ func TestAllowedDownloadZip(t *testing.T) { ...@@ -104,7 +106,7 @@ func TestAllowedDownloadZip(t *testing.T) {
// Prepare test server and backend // Prepare test server and backend
archiveName := "foobar.zip" archiveName := "foobar.zip"
ts := testAuthServer(200, archiveOkBody(t, archiveName)) ts := testAuthServer(nil, 200, archiveOkBody(t, archiveName))
defer ts.Close() defer ts.Close()
ws := startWorkhorseServer(ts.URL) ws := startWorkhorseServer(ts.URL)
defer ws.Close() defer ws.Close()
...@@ -123,7 +125,7 @@ func TestAllowedDownloadTar(t *testing.T) { ...@@ -123,7 +125,7 @@ func TestAllowedDownloadTar(t *testing.T) {
// Prepare test server and backend // Prepare test server and backend
archiveName := "foobar.tar" archiveName := "foobar.tar"
ts := testAuthServer(200, archiveOkBody(t, archiveName)) ts := testAuthServer(nil, 200, archiveOkBody(t, archiveName))
defer ts.Close() defer ts.Close()
ws := startWorkhorseServer(ts.URL) ws := startWorkhorseServer(ts.URL)
defer ws.Close() defer ws.Close()
...@@ -142,7 +144,7 @@ func TestAllowedDownloadTarGz(t *testing.T) { ...@@ -142,7 +144,7 @@ func TestAllowedDownloadTarGz(t *testing.T) {
// Prepare test server and backend // Prepare test server and backend
archiveName := "foobar.tar.gz" archiveName := "foobar.tar.gz"
ts := testAuthServer(200, archiveOkBody(t, archiveName)) ts := testAuthServer(nil, 200, archiveOkBody(t, archiveName))
defer ts.Close() defer ts.Close()
ws := startWorkhorseServer(ts.URL) ws := startWorkhorseServer(ts.URL)
defer ws.Close() defer ws.Close()
...@@ -161,7 +163,7 @@ func TestAllowedDownloadTarBz2(t *testing.T) { ...@@ -161,7 +163,7 @@ func TestAllowedDownloadTarBz2(t *testing.T) {
// Prepare test server and backend // Prepare test server and backend
archiveName := "foobar.tar.bz2" archiveName := "foobar.tar.bz2"
ts := testAuthServer(200, archiveOkBody(t, archiveName)) ts := testAuthServer(nil, 200, archiveOkBody(t, archiveName))
defer ts.Close() defer ts.Close()
ws := startWorkhorseServer(ts.URL) ws := startWorkhorseServer(ts.URL)
defer ws.Close() defer ws.Close()
...@@ -180,7 +182,7 @@ func TestAllowedApiDownloadZip(t *testing.T) { ...@@ -180,7 +182,7 @@ func TestAllowedApiDownloadZip(t *testing.T) {
// Prepare test server and backend // Prepare test server and backend
archiveName := "foobar.zip" archiveName := "foobar.zip"
ts := testAuthServer(200, archiveOkBody(t, archiveName)) ts := testAuthServer(nil, 200, archiveOkBody(t, archiveName))
defer ts.Close() defer ts.Close()
ws := startWorkhorseServer(ts.URL) ws := startWorkhorseServer(ts.URL)
defer ws.Close() defer ws.Close()
...@@ -199,7 +201,7 @@ func TestDownloadCacheHit(t *testing.T) { ...@@ -199,7 +201,7 @@ func TestDownloadCacheHit(t *testing.T) {
// Prepare test server and backend // Prepare test server and backend
archiveName := "foobar.zip" archiveName := "foobar.zip"
ts := testAuthServer(200, archiveOkBody(t, archiveName)) ts := testAuthServer(nil, 200, archiveOkBody(t, archiveName))
defer ts.Close() defer ts.Close()
ws := startWorkhorseServer(ts.URL) ws := startWorkhorseServer(ts.URL)
defer ws.Close() defer ws.Close()
...@@ -230,7 +232,7 @@ func TestDownloadCacheCreate(t *testing.T) { ...@@ -230,7 +232,7 @@ func TestDownloadCacheCreate(t *testing.T) {
// Prepare test server and backend // Prepare test server and backend
archiveName := "foobar.zip" archiveName := "foobar.zip"
ts := testAuthServer(200, archiveOkBody(t, archiveName)) ts := testAuthServer(nil, 200, archiveOkBody(t, archiveName))
defer ts.Close() defer ts.Close()
ws := startWorkhorseServer(ts.URL) ws := startWorkhorseServer(ts.URL)
defer ws.Close() defer ws.Close()
...@@ -281,12 +283,47 @@ func newBranch() string { ...@@ -281,12 +283,47 @@ func newBranch() string {
return fmt.Sprintf("branch-%d", time.Now().UnixNano()) return fmt.Sprintf("branch-%d", time.Now().UnixNano())
} }
func testAuthServer(code int, body string) *httptest.Server { func testServerWithHandler(url *regexp.Regexp, handler http.HandlerFunc) *httptest.Server {
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if url != nil && !url.MatchString(r.URL.Path) {
log.Println("UPSTREAM", r.Method, r.URL, "DENY")
w.WriteHeader(404)
return
}
if version := r.Header.Get("Gitlab-Workhorse"); version == "" {
log.Println("UPSTREAM", r.Method, r.URL, "DENY")
w.WriteHeader(403)
return
}
handler(w, r)
}))
}
func testAuthServer(url *regexp.Regexp, code int, body interface{}) *httptest.Server {
return testServerWithHandler(url, func(w http.ResponseWriter, r *http.Request) {
// Write pure string
if data, ok := body.(string); ok {
log.Println("UPSTREAM", r.Method, r.URL, code)
w.WriteHeader(code)
fmt.Fprint(w, data)
return
}
// Write json string
data, err := json.Marshal(body)
if err != nil {
log.Println("UPSTREAM", r.Method, r.URL, "FAILURE", err)
w.WriteHeader(503)
fmt.Fprint(w, err)
return
}
log.Println("UPSTREAM", r.Method, r.URL, code) log.Println("UPSTREAM", r.Method, r.URL, code)
w.WriteHeader(code) w.WriteHeader(code)
fmt.Fprint(w, body) w.Write(data)
})) })
} }
func startWorkhorseServer(authBackend string) *httptest.Server { func startWorkhorseServer(authBackend string) *httptest.Server {
...@@ -301,23 +338,26 @@ func runOrFail(t *testing.T, cmd *exec.Cmd) { ...@@ -301,23 +338,26 @@ func runOrFail(t *testing.T, cmd *exec.Cmd) {
} }
} }
func gitOkBody(t *testing.T) string { func gitOkBody(t *testing.T) interface{} {
return fmt.Sprintf(`{"GL_ID":"user-123","RepoPath":"%s"}`, repoPath(t)) return &authorizationResponse{
GL_ID: "user-123",
RepoPath: repoPath(t),
}
} }
func archiveOkBody(t *testing.T, archiveName string) string { func archiveOkBody(t *testing.T, archiveName string) interface{} {
cwd, err := os.Getwd() cwd, err := os.Getwd()
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
archivePath := path.Join(cwd, cacheDir, archiveName) archivePath := path.Join(cwd, cacheDir, archiveName)
jsonString := `{
"RepoPath":"%s", return &authorizationResponse{
"ArchivePath":"%s", RepoPath: repoPath(t),
"CommitId":"c7fbe50c7c7419d9701eebe64b1fdacc3df5b9dd", ArchivePath: archivePath,
"ArchivePrefix":"foobar123" CommitId: "c7fbe50c7c7419d9701eebe64b1fdacc3df5b9dd",
}` ArchivePrefix: "foobar123",
return fmt.Sprintf(jsonString, repoPath(t), archivePath) }
} }
func repoPath(t *testing.T) string { func repoPath(t *testing.T) string {
......
package main
import (
"net/http"
)
func proxyRequest(w http.ResponseWriter, r *gitRequest) {
upRequest, err := r.u.newUpstreamRequest(r.Request, r.Body, "")
if err != nil {
fail500(w, "newUpstreamRequest", err)
return
}
upResponse, err := r.u.httpClient.Do(upRequest)
if err != nil {
fail500(w, "do upstream request", err)
return
}
defer upResponse.Body.Close()
forwardResponseToClient(w, upResponse)
}
package main
import (
"bytes"
"fmt"
"io"
"net/http"
"net/http/httptest"
"regexp"
"testing"
)
func TestProxyRequest(t *testing.T) {
ts := testServerWithHandler(regexp.MustCompile(`/url/path\z`), func(w http.ResponseWriter, r *http.Request) {
if r.Method != "POST" {
t.Fatal("Expected POST request")
}
if r.Header.Get("Custom-Header") != "test" {
t.Fatal("Missing custom header")
}
var body bytes.Buffer
io.Copy(&body, r.Body)
if body.String() != "REQUEST" {
t.Fatal("Expected REQUEST in request body")
}
w.Header().Set("Custom-Response-Header", "test")
w.WriteHeader(202)
fmt.Fprint(w, "RESPONSE")
})
httpRequest, err := http.NewRequest("POST", ts.URL+"/url/path", bytes.NewBufferString("REQUEST"))
if err != nil {
t.Fatal(err)
}
httpRequest.Header.Set("Custom-Header", "test")
request := gitRequest{
Request: httpRequest,
u: newUpstream(ts.URL, nil),
}
response := httptest.NewRecorder()
proxyRequest(response, &request)
assertResponseCode(t, response, 202)
if response.Body.String() != "RESPONSE" {
t.Fatal("Expected RESPONSE in response body:", response.Body.String())
}
if response.Header().Get("Custom-Response-Header") != "test" {
t.Fatal("Expected custom response header")
}
}
package main
import (
"bytes"
"errors"
"io"
"io/ioutil"
"mime/multipart"
"net/http"
"os"
)
func rewriteFormFilesFromMultipart(r *gitRequest, writer *multipart.Writer) (cleanup func(), err error) {
// Create multipart reader
reader, err := r.MultipartReader()
if err != nil {
return nil, err
}
var files []string
cleanup = func() {
for _, file := range files {
os.Remove(file)
}
}
// Execute cleanup in case of failure
defer func() {
if err != nil {
cleanup()
}
}()
for {
p, err := reader.NextPart()
if err == io.EOF {
break
}
name := p.FormName()
if name == "" {
continue
}
// Copy form field
if filename := p.FileName(); filename != "" {
// Create temporary directory where the uploaded file will be stored
if err := os.MkdirAll(r.TempPath, 0700); err != nil {
return cleanup, err
}
// Create temporary file in path returned by Authorization filter
file, err := ioutil.TempFile(r.TempPath, "upload_")
if err != nil {
return cleanup, err
}
defer file.Close()
// Add file entry
writer.WriteField(name+".path", file.Name())
writer.WriteField(name+".name", filename)
files = append(files, file.Name())
_, err = io.Copy(file, p)
file.Close()
if err != nil {
return cleanup, err
}
} else {
np, err := writer.CreatePart(p.Header)
if err != nil {
return cleanup, err
}
_, err = io.Copy(np, p)
if err != nil {
return cleanup, err
}
}
}
return cleanup, nil
}
func handleFileUploads(w http.ResponseWriter, r *gitRequest) {
if r.TempPath == "" {
fail500(w, "handleUploadFile", errors.New("missing temporary path"))
return
}
var body bytes.Buffer
writer := multipart.NewWriter(&body)
defer writer.Close()
// Rewrite multipart form data
cleanup, err := rewriteFormFilesFromMultipart(r, writer)
if err != nil {
if err == http.ErrNotMultipart {
proxyRequest(w, r)
} else {
fail500(w, "Couldn't handle upload request.", err)
}
return
}
if cleanup != nil {
defer cleanup()
}
// Close writer
writer.Close()
// Create request
upstreamRequest, err := r.u.newUpstreamRequest(r.Request, nil, "")
if err != nil {
fail500(w, "Couldn't handle artifacts upload request.", err)
return
}
// Set multipart form data
upstreamRequest.Body = ioutil.NopCloser(&body)
upstreamRequest.ContentLength = int64(body.Len())
upstreamRequest.Header.Set("Content-Type", writer.FormDataContentType())
// Forward request to backend
upstreamResponse, err := r.u.httpClient.Do(upstreamRequest)
if err != nil {
fail500(w, "do upstream request", err)
return
}
defer upstreamResponse.Body.Close()
forwardResponseToClient(w, upstreamResponse)
}
package main
import (
"bytes"
"fmt"
"io"
"io/ioutil"
"mime/multipart"
"net/http"
"net/http/httptest"
"os"
"regexp"
"strings"
"testing"
)
func TestUploadTempPathRequirement(t *testing.T) {
response := httptest.NewRecorder()
request := gitRequest{
authorizationResponse: authorizationResponse{
TempPath: "",
},
}
handleFileUploads(response, &request)
assertResponseCode(t, response, 500)
}
func TestUploadHandlerForwardingRawData(t *testing.T) {
ts := testServerWithHandler(regexp.MustCompile(`/url/path\z`), func(w http.ResponseWriter, r *http.Request) {
if r.Method != "PATCH" {
t.Fatal("Expected PATCH request")
}
var body bytes.Buffer
io.Copy(&body, r.Body)
if body.String() != "REQUEST" {
t.Fatal("Expected REQUEST in request body")
}
w.WriteHeader(202)
fmt.Fprint(w, "RESPONSE")
})
httpRequest, err := http.NewRequest("PATCH", ts.URL+"/url/path", bytes.NewBufferString("REQUEST"))
if err != nil {
t.Fatal(err)
}
tempPath, err := ioutil.TempDir("", "uploads")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tempPath)
response := httptest.NewRecorder()
request := gitRequest{
Request: httpRequest,
u: newUpstream(ts.URL, nil),
authorizationResponse: authorizationResponse{
TempPath: tempPath,
},
}
handleFileUploads(response, &request)
assertResponseCode(t, response, 202)
if response.Body.String() != "RESPONSE" {
t.Fatal("Expected RESPONSE in response body")
}
}
func TestUploadHandlerRewritingMultiPartData(t *testing.T) {
var filePath string
tempPath, err := ioutil.TempDir("", "uploads")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tempPath)
ts := testServerWithHandler(regexp.MustCompile(`/url/path\z`), func(w http.ResponseWriter, r *http.Request) {
if r.Method != "PUT" {
t.Fatal("Expected PUT request")
}
err := r.ParseMultipartForm(100000)
if err != nil {
t.Fatal(err)
}
if len(r.MultipartForm.Value) != 3 {
t.Fatal("Expected to receive exactly 3 values")
}
if len(r.MultipartForm.File) != 0 {
t.Fatal("Expected to not receive any files")
}
if r.FormValue("token") != "test" {
t.Fatal("Expected to receive token")
}
if r.FormValue("file.name") != "my.file" {
t.Fatal("Expected to receive a filename")
}
filePath = r.FormValue("file.path")
if !strings.HasPrefix(r.FormValue("file.path"), tempPath) {
t.Fatal("Expected to the file to be in tempPath")
}
w.WriteHeader(202)
fmt.Fprint(w, "RESPONSE")
})
var buffer bytes.Buffer
writer := multipart.NewWriter(&buffer)
writer.WriteField("token", "test")
file, err := writer.CreateFormFile("file", "my.file")
if err != nil {
t.Fatal(err)
}
fmt.Fprint(file, "test")
writer.Close()
httpRequest, err := http.NewRequest("PUT", ts.URL+"/url/path", nil)
if err != nil {
t.Fatal(err)
}
httpRequest.Body = ioutil.NopCloser(&buffer)
httpRequest.ContentLength = int64(buffer.Len())
httpRequest.Header.Set("Content-Type", writer.FormDataContentType())
response := httptest.NewRecorder()
request := gitRequest{
Request: httpRequest,
u: newUpstream(ts.URL, nil),
authorizationResponse: authorizationResponse{
TempPath: tempPath,
},
}
handleFileUploads(response, &request)
assertResponseCode(t, response, 202)
if _, err := os.Stat(filePath); !os.IsNotExist(err) {
t.Fatal("expected the file to be deleted")
}
}
...@@ -51,6 +51,9 @@ type authorizationResponse struct { ...@@ -51,6 +51,9 @@ type authorizationResponse struct {
LfsOid string LfsOid string
// LFS object size // LFS object size
LfsSize int64 LfsSize int64
// TmpPath is the path where we should store temporary files
// This is set by authorization middleware
TempPath string
} }
// A gitReqest is an *http.Request decorated with attributes returned by the // A gitReqest is an *http.Request decorated with attributes returned by the
...@@ -64,16 +67,24 @@ type gitRequest struct { ...@@ -64,16 +67,24 @@ type gitRequest struct {
// Routing table // Routing table
var gitServices = [...]gitService{ var gitServices = [...]gitService{
gitService{"GET", regexp.MustCompile(`/info/refs\z`), repoPreAuthorizeHandler(handleGetInfoRefs)}, gitService{"GET", regexp.MustCompile(`/info/refs\z`), repoPreAuthorizeHandler(handleGetInfoRefs)},
gitService{"POST", regexp.MustCompile(`/git-upload-pack\z`), repoPreAuthorizeHandler(handlePostRPC)}, gitService{"POST", regexp.MustCompile(`/git-upload-pack\z`), repoPreAuthorizeHandler(contentEncodingHandler(handlePostRPC))},
gitService{"POST", regexp.MustCompile(`/git-receive-pack\z`), repoPreAuthorizeHandler(handlePostRPC)}, gitService{"POST", regexp.MustCompile(`/git-receive-pack\z`), repoPreAuthorizeHandler(contentEncodingHandler(handlePostRPC))},
gitService{"GET", regexp.MustCompile(`/repository/archive\z`), repoPreAuthorizeHandler(handleGetArchive)}, gitService{"GET", regexp.MustCompile(`/repository/archive\z`), repoPreAuthorizeHandler(handleGetArchive)},
gitService{"GET", regexp.MustCompile(`/repository/archive.zip\z`), repoPreAuthorizeHandler(handleGetArchive)}, gitService{"GET", regexp.MustCompile(`/repository/archive.zip\z`), repoPreAuthorizeHandler(handleGetArchive)},
gitService{"GET", regexp.MustCompile(`/repository/archive.tar\z`), repoPreAuthorizeHandler(handleGetArchive)}, gitService{"GET", regexp.MustCompile(`/repository/archive.tar\z`), repoPreAuthorizeHandler(handleGetArchive)},
gitService{"GET", regexp.MustCompile(`/repository/archive.tar.gz\z`), repoPreAuthorizeHandler(handleGetArchive)}, gitService{"GET", regexp.MustCompile(`/repository/archive.tar.gz\z`), repoPreAuthorizeHandler(handleGetArchive)},
gitService{"GET", regexp.MustCompile(`/repository/archive.tar.bz2\z`), repoPreAuthorizeHandler(handleGetArchive)}, gitService{"GET", regexp.MustCompile(`/repository/archive.tar.bz2\z`), repoPreAuthorizeHandler(handleGetArchive)},
gitService{"GET", regexp.MustCompile(`/uploads/`), handleSendFile}, gitService{"GET", regexp.MustCompile(`/uploads/`), handleSendFile},
// Git LFS
gitService{"PUT", regexp.MustCompile(`/gitlab-lfs/objects/([0-9a-f]{64})/([0-9]+)\z`), lfsAuthorizeHandler(handleStoreLfsObject)}, gitService{"PUT", regexp.MustCompile(`/gitlab-lfs/objects/([0-9a-f]{64})/([0-9]+)\z`), lfsAuthorizeHandler(handleStoreLfsObject)},
gitService{"GET", regexp.MustCompile(`/gitlab-lfs/objects/([0-9a-f]{64})\z`), handleSendFile}, gitService{"GET", regexp.MustCompile(`/gitlab-lfs/objects/([0-9a-f]{64})\z`), handleSendFile},
// CI artifacts
gitService{"GET", regexp.MustCompile(`/builds/download\z`), handleSendFile},
gitService{"GET", regexp.MustCompile(`/ci/api/v1/builds/[0-9]+/artifacts\z`), handleSendFile},
gitService{"POST", regexp.MustCompile(`/ci/api/v1/builds/[0-9]+/artifacts\z`), artifactsAuthorizeHandler(contentEncodingHandler(handleFileUploads))},
gitService{"DELETE", regexp.MustCompile(`/ci/api/v1/builds/[0-9]+/artifacts\z`), proxyRequest},
} }
func newUpstream(authBackend string, authTransport http.RoundTripper) *upstream { func newUpstream(authBackend string, authTransport http.RoundTripper) *upstream {
...@@ -135,6 +146,7 @@ func (u *upstream) newUpstreamRequest(r *http.Request, body io.Reader, suffix st ...@@ -135,6 +146,7 @@ func (u *upstream) newUpstreamRequest(r *http.Request, body io.Reader, suffix st
authReq.Header.Del("Content-Type") authReq.Header.Del("Content-Type")
authReq.Header.Del("Content-Encoding") authReq.Header.Del("Content-Encoding")
authReq.Header.Del("Content-Length") authReq.Header.Del("Content-Length")
authReq.Header.Del("Content-Disposition")
authReq.Header.Del("Accept-Encoding") authReq.Header.Del("Accept-Encoding")
authReq.Header.Del("Transfer-Encoding") authReq.Header.Del("Transfer-Encoding")
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment