Commit 575e78b9 authored by Jacob Vosmaer's avatar Jacob Vosmaer

Merge branch 'raw-blob' into 'master'

Blobs via workhorse

Works together with https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/2451

See merge request !30
parents e5250487 8167a305
...@@ -3,7 +3,6 @@ package api ...@@ -3,7 +3,6 @@ package api
import ( import (
"../badgateway" "../badgateway"
"../helper" "../helper"
"../proxy"
"encoding/json" "encoding/json"
"fmt" "fmt"
"io" "io"
...@@ -97,7 +96,7 @@ func (api *API) newRequest(r *http.Request, body io.Reader, suffix string) (*htt ...@@ -97,7 +96,7 @@ func (api *API) newRequest(r *http.Request, body io.Reader, suffix string) (*htt
authReq := &http.Request{ authReq := &http.Request{
Method: r.Method, Method: r.Method,
URL: rebaseUrl(r.URL, api.URL, suffix), URL: rebaseUrl(r.URL, api.URL, suffix),
Header: proxy.HeaderClone(r.Header), Header: helper.HeaderClone(r.Header),
} }
if body != nil { if body != nil {
authReq.Body = ioutil.NopCloser(body) authReq.Body = ioutil.NopCloser(body)
......
package git
import (
"../helper"
"encoding/base64"
"encoding/json"
"fmt"
"io"
"log"
"net/http"
"strings"
)
type blobParams struct {
RepoPath string
BlobId string
}
const SendBlobPrefix = "git-blob:"
func SendBlob(w http.ResponseWriter, r *http.Request, sendData string) {
params, err := unpackSendData(sendData)
if err != nil {
helper.Fail500(w, fmt.Errorf("SendBlob: unpack sendData: %v", err))
return
}
log.Printf("SendBlob: sending %q for %q", params.BlobId, r.URL.Path)
gitShowCmd := gitCommand("", "git", "--git-dir="+params.RepoPath, "cat-file", "blob", params.BlobId)
stdout, err := gitShowCmd.StdoutPipe()
if err != nil {
helper.Fail500(w, fmt.Errorf("SendBlob: git stdout: %v", err))
return
}
if err := gitShowCmd.Start(); err != nil {
helper.Fail500(w, fmt.Errorf("SendBlob: start %v: %v", gitShowCmd, err))
return
}
defer helper.CleanUpProcessGroup(gitShowCmd)
if _, err := io.Copy(w, stdout); err != nil {
helper.LogError(fmt.Errorf("SendBlob: copy git cat-file stdout: %v", err))
return
}
if err := gitShowCmd.Wait(); err != nil {
helper.LogError(fmt.Errorf("SendBlob: wait for git cat-file: %v", err))
return
}
}
func unpackSendData(sendData string) (*blobParams, error) {
jsonBytes, err := base64.URLEncoding.DecodeString(strings.TrimPrefix(sendData, SendBlobPrefix))
if err != nil {
return nil, err
}
result := &blobParams{}
if err := json.Unmarshal([]byte(jsonBytes), result); err != nil {
return nil, err
}
return result, nil
}
...@@ -72,6 +72,16 @@ func HTTPError(w http.ResponseWriter, r *http.Request, error string, code int) { ...@@ -72,6 +72,16 @@ func HTTPError(w http.ResponseWriter, r *http.Request, error string, code int) {
http.Error(w, error, code) http.Error(w, error, code)
} }
func HeaderClone(h http.Header) http.Header {
h2 := make(http.Header, len(h))
for k, vv := range h {
vv2 := make([]string, len(vv))
copy(vv2, vv)
h2[k] = vv2
}
return h2
}
func CleanUpProcessGroup(cmd *exec.Cmd) { func CleanUpProcessGroup(cmd *exec.Cmd) {
if cmd == nil { if cmd == nil {
return return
......
...@@ -2,6 +2,8 @@ package proxy ...@@ -2,6 +2,8 @@ package proxy
import ( import (
"../badgateway" "../badgateway"
"../helper"
"../senddata"
"net/http" "net/http"
"net/http/httputil" "net/http/httputil"
"net/url" "net/url"
...@@ -25,24 +27,14 @@ func NewProxy(myURL *url.URL, version string, roundTripper *badgateway.RoundTrip ...@@ -25,24 +27,14 @@ func NewProxy(myURL *url.URL, version string, roundTripper *badgateway.RoundTrip
return &p return &p
} }
func HeaderClone(h http.Header) http.Header {
h2 := make(http.Header, len(h))
for k, vv := range h {
vv2 := make([]string, len(vv))
copy(vv2, vv)
h2[k] = vv2
}
return h2
}
func (p *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { func (p *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
// Clone request // Clone request
req := *r req := *r
req.Header = HeaderClone(r.Header) req.Header = helper.HeaderClone(r.Header)
// Set Workhorse version // Set Workhorse version
req.Header.Set("Gitlab-Workhorse", p.Version) req.Header.Set("Gitlab-Workhorse", p.Version)
rw := newSendFileResponseWriter(w, &req) rw := senddata.NewSendFileResponseWriter(w, &req)
defer rw.Flush() defer rw.Flush()
p.reverseProxy.ServeHTTP(&rw, &req) p.reverseProxy.ServeHTTP(&rw, &req)
......
...@@ -4,12 +4,19 @@ via the X-Sendfile mechanism. All that is needed in the Rails code is the ...@@ -4,12 +4,19 @@ via the X-Sendfile mechanism. All that is needed in the Rails code is the
'send_file' method. 'send_file' method.
*/ */
package proxy package senddata
import ( import (
"../git"
"../helper" "../helper"
"log" "log"
"net/http" "net/http"
"strings"
)
const (
sendDataResponseHeader = "Gitlab-Workhorse-Send-Data"
sendFileResponseHeader = "X-Sendfile"
) )
type sendFileResponseWriter struct { type sendFileResponseWriter struct {
...@@ -19,11 +26,12 @@ type sendFileResponseWriter struct { ...@@ -19,11 +26,12 @@ type sendFileResponseWriter struct {
req *http.Request req *http.Request
} }
func newSendFileResponseWriter(rw http.ResponseWriter, req *http.Request) sendFileResponseWriter { func NewSendFileResponseWriter(rw http.ResponseWriter, req *http.Request) sendFileResponseWriter {
s := sendFileResponseWriter{ s := sendFileResponseWriter{
rw: rw, rw: rw,
req: req, req: req,
} }
// Advertise to upstream (Rails) that we support X-Sendfile
req.Header.Set("X-Sendfile-Type", "X-Sendfile") req.Header.Set("X-Sendfile-Type", "X-Sendfile")
return s return s
} }
...@@ -48,30 +56,41 @@ func (s *sendFileResponseWriter) WriteHeader(status int) { ...@@ -48,30 +56,41 @@ func (s *sendFileResponseWriter) WriteHeader(status int) {
} }
s.status = status s.status = status
if s.status != http.StatusOK {
s.rw.WriteHeader(s.status)
return
}
// Check X-Sendfile header if file := s.Header().Get(sendFileResponseHeader); file != "" {
file := s.Header().Get("X-Sendfile") s.Header().Del(sendFileResponseHeader)
s.Header().Del("X-Sendfile") // Mark this connection as hijacked
s.hijacked = true
// If file is empty or status is not 200 pass through header // Serve the file
if file == "" || s.status != http.StatusOK { sendFileFromDisk(s.rw, s.req, file)
s.rw.WriteHeader(s.status) return
}
if sendData := s.Header().Get(sendDataResponseHeader); strings.HasPrefix(sendData, git.SendBlobPrefix) {
s.Header().Del(sendDataResponseHeader)
s.hijacked = true
git.SendBlob(s.rw, s.req, sendData)
return return
} }
// Mark this connection as hijacked s.rw.WriteHeader(s.status)
s.hijacked = true return
}
// Serve the file func sendFileFromDisk(w http.ResponseWriter, r *http.Request, file string) {
log.Printf("Send file %q for %s %q", file, s.req.Method, s.req.RequestURI) log.Printf("Send file %q for %s %q", file, r.Method, r.RequestURI)
content, fi, err := helper.OpenFile(file) content, fi, err := helper.OpenFile(file)
if err != nil { if err != nil {
http.NotFound(s.rw, s.req) http.NotFound(w, r)
return return
} }
defer content.Close() defer content.Close()
http.ServeContent(s.rw, s.req, "", fi.ModTime(), content) http.ServeContent(w, r, "", fi.ModTime(), content)
} }
func (s *sendFileResponseWriter) Flush() { func (s *sendFileResponseWriter) Flush() {
......
...@@ -555,6 +555,48 @@ func TestArtifactsGetSingleFile(t *testing.T) { ...@@ -555,6 +555,48 @@ func TestArtifactsGetSingleFile(t *testing.T) {
} }
} }
func TestGetGitBlob(t *testing.T) {
blobId := "50b27c6518be44c42c4d87966ae2481ce895624c" // the LICENSE file in the test repository
blobLength := 1075
headerKey := http.CanonicalHeaderKey("Gitlab-Workhorse-Send-Data")
ts := testhelper.TestServerWithHandler(regexp.MustCompile(`.`), func(w http.ResponseWriter, r *http.Request) {
responseJSON := fmt.Sprintf(`{"RepoPath":"%s","BlobId":"%s"}`, path.Join(testRepoRoot, testRepo), blobId)
encodedJSON := base64.StdEncoding.EncodeToString([]byte(responseJSON))
w.Header().Set(headerKey, "git-blob:"+encodedJSON)
// Prevent the Go HTTP server from setting the Content-Length to 0.
w.Header().Set("Transfer-Encoding", "chunked")
if _, err := fmt.Fprintf(w, "GNU General Public License"); err != nil {
t.Fatal(err)
}
return
})
defer ts.Close()
ws := startWorkhorseServer(ts.URL)
defer ws.Close()
resourcePath := "/something"
resp, err := http.Get(ws.URL + resourcePath)
if err != nil {
t.Error(err)
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
t.Errorf("GET %q: expected 200, got %d", resourcePath, resp.StatusCode)
}
if len(resp.Header[headerKey]) != 0 {
t.Fatalf("Unexpected response header: %s: %q", headerKey, resp.Header.Get(headerKey))
}
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
t.Fatal(err)
}
if len(body) != blobLength {
t.Fatalf("Expected body of %d bytes, got %d", blobLength, len(body))
}
if !strings.HasPrefix(string(body), "The MIT License (MIT)") {
t.Fatalf("Expected MIT license, got %q", body)
}
}
func setupStaticFile(fpath, content string) error { func setupStaticFile(fpath, content string) error {
cwd, err := os.Getwd() cwd, err := os.Getwd()
if err != nil { if err != nil {
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment