Commit b89c2257 authored by Patrick Bajao's avatar Patrick Bajao

Merge branch 'jv-refactor-uploads-2' into 'master'

Workhorse: change upload method names and add comments

See merge request gitlab-org/gitlab!80305
parents dfddb423 d0bc5fbe
...@@ -162,6 +162,6 @@ func UploadArtifacts(myAPI *api.API, h http.Handler, p upload.Preparer) http.Han ...@@ -162,6 +162,6 @@ func UploadArtifacts(myAPI *api.API, h http.Handler, p upload.Preparer) http.Han
format := r.URL.Query().Get(ArtifactFormatKey) format := r.URL.Query().Get(ArtifactFormatKey)
mg := &artifactsUploadProcessor{opts: opts, format: format, SavedFileTracker: upload.SavedFileTracker{Request: r}} mg := &artifactsUploadProcessor{opts: opts, format: format, SavedFileTracker: upload.SavedFileTracker{Request: r}}
upload.HandleFileUploads(w, r, h, a, mg, opts) upload.InterceptMultipartFiles(w, r, h, a, mg, opts)
}, "/authorize") }, "/authorize")
} }
...@@ -91,12 +91,12 @@ func TestInject(t *testing.T) { ...@@ -91,12 +91,12 @@ func TestInject(t *testing.T) {
})) }))
defer originResourceServer.Close() defer originResourceServer.Close()
// BodyUploader expects http.Handler as its second param, we can create a stub function and verify that // RequestBody expects http.Handler as its second param, we can create a stub function and verify that
// it's only called for successful requests // it's only called for successful requests
handlerIsCalled := false handlerIsCalled := false
handlerFunc := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { handlerIsCalled = true }) handlerFunc := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { handlerIsCalled = true })
bodyUploader := upload.BodyUploader(&fakePreAuthHandler{}, handlerFunc, &upload.DefaultPreparer{}) bodyUploader := upload.RequestBody(&fakePreAuthHandler{}, handlerFunc, &upload.DefaultPreparer{})
injector := NewInjector() injector := NewInjector()
injector.SetUploadHandler(bodyUploader) injector.SetUploadHandler(bodyUploader)
......
...@@ -6,7 +6,6 @@ package lfs ...@@ -6,7 +6,6 @@ package lfs
import ( import (
"fmt" "fmt"
"net/http"
"gitlab.com/gitlab-org/gitlab/workhorse/internal/api" "gitlab.com/gitlab-org/gitlab/workhorse/internal/api"
"gitlab.com/gitlab-org/gitlab/workhorse/internal/config" "gitlab.com/gitlab-org/gitlab/workhorse/internal/config"
...@@ -49,7 +48,3 @@ func (l *uploadPreparer) Prepare(a *api.Response) (*filestore.SaveFileOpts, uplo ...@@ -49,7 +48,3 @@ func (l *uploadPreparer) Prepare(a *api.Response) (*filestore.SaveFileOpts, uplo
return opts, &object{oid: a.LfsOid, size: a.LfsSize}, nil return opts, &object{oid: a.LfsOid, size: a.LfsSize}, nil
} }
func PutStore(a *api.API, h http.Handler, p upload.Preparer) http.Handler {
return upload.BodyUploader(a, h, p)
}
...@@ -17,16 +17,21 @@ type MultipartClaims struct { ...@@ -17,16 +17,21 @@ type MultipartClaims struct {
jwt.StandardClaims jwt.StandardClaims
} }
func Accelerate(rails PreAuthorizer, h http.Handler, p Preparer) http.Handler { // Multipart is a request middleware. If the request has a MIME multipart
// request body, the middleware will iterate through the multipart parts.
// When it finds a file part (filename != ""), the middleware will save
// the file contents to a temporary location and replace the file part
// with a reference to the temporary location.
func Multipart(rails PreAuthorizer, h http.Handler, p Preparer) http.Handler {
return rails.PreAuthorizeHandler(func(w http.ResponseWriter, r *http.Request, a *api.Response) { return rails.PreAuthorizeHandler(func(w http.ResponseWriter, r *http.Request, a *api.Response) {
s := &SavedFileTracker{Request: r} s := &SavedFileTracker{Request: r}
opts, _, err := p.Prepare(a) opts, _, err := p.Prepare(a)
if err != nil { if err != nil {
helper.Fail500(w, r, fmt.Errorf("Accelerate: error preparing file storage options")) helper.Fail500(w, r, fmt.Errorf("Multipart: error preparing file storage options"))
return return
} }
HandleFileUploads(w, r, h, a, s, opts) InterceptMultipartFiles(w, r, h, a, s, opts)
}, "/authorize") }, "/authorize")
} }
...@@ -16,16 +16,23 @@ type PreAuthorizer interface { ...@@ -16,16 +16,23 @@ type PreAuthorizer interface {
PreAuthorizeHandler(next api.HandleFunc, suffix string) http.Handler PreAuthorizeHandler(next api.HandleFunc, suffix string) http.Handler
} }
// Verifier allows to check an upload before sending it to rails // Verifier is an optional pluggable behavior for upload paths. If
// Verify() returns an error, Workhorse will return an error response to
// the client instead of propagating the request to Rails. The motivating
// use case is Git LFS, where Workhorse checks the size and SHA256
// checksum of the uploaded file.
type Verifier interface { type Verifier interface {
// Verify can abort the upload returning an error // Verify can abort the upload by returning an error
Verify(handler *filestore.FileHandler) error Verify(handler *filestore.FileHandler) error
} }
// Preparer allows to customize BodyUploader configuration // Preparer is a pluggable behavior that interprets a Rails API response
// and either tells Workhorse how to handle the upload, via the
// SaveFileOpts and Verifier, or it rejects the request by returning a
// non-nil error. Its intended use is to make sure the upload gets stored
// in the right location: either a local directory, or one of several
// supported object storage backends.
type Preparer interface { type Preparer interface {
// Prepare converts api.Response into a *SaveFileOpts, it can optionally return an Verifier that will be
// invoked after the real upload, before the finalization with rails
Prepare(a *api.Response) (*filestore.SaveFileOpts, Verifier, error) Prepare(a *api.Response) (*filestore.SaveFileOpts, Verifier, error)
} }
...@@ -36,26 +43,26 @@ func (s *DefaultPreparer) Prepare(a *api.Response) (*filestore.SaveFileOpts, Ver ...@@ -36,26 +43,26 @@ func (s *DefaultPreparer) Prepare(a *api.Response) (*filestore.SaveFileOpts, Ver
return opts, nil, err return opts, nil, err
} }
// BodyUploader is an http.Handler that perform a pre authorization call to rails before hijacking the request body and // RequestBody is a request middleware. It will store the request body to
// uploading it. // a location by determined an api.Response value. It then forwards the
// Providing an Preparer allows to customize the upload process // request to gitlab-rails without the original request body.
func BodyUploader(rails PreAuthorizer, h http.Handler, p Preparer) http.Handler { func RequestBody(rails PreAuthorizer, h http.Handler, p Preparer) http.Handler {
return rails.PreAuthorizeHandler(func(w http.ResponseWriter, r *http.Request, a *api.Response) { return rails.PreAuthorizeHandler(func(w http.ResponseWriter, r *http.Request, a *api.Response) {
opts, verifier, err := p.Prepare(a) opts, verifier, err := p.Prepare(a)
if err != nil { if err != nil {
helper.Fail500(w, r, fmt.Errorf("BodyUploader: preparation failed: %v", err)) helper.Fail500(w, r, fmt.Errorf("RequestBody: preparation failed: %v", err))
return return
} }
fh, err := filestore.SaveFileFromReader(r.Context(), r.Body, r.ContentLength, opts) fh, err := filestore.SaveFileFromReader(r.Context(), r.Body, r.ContentLength, opts)
if err != nil { if err != nil {
helper.Fail500(w, r, fmt.Errorf("BodyUploader: upload failed: %v", err)) helper.Fail500(w, r, fmt.Errorf("RequestBody: upload failed: %v", err))
return return
} }
if verifier != nil { if verifier != nil {
if err := verifier.Verify(fh); err != nil { if err := verifier.Verify(fh); err != nil {
helper.Fail500(w, r, fmt.Errorf("BodyUploader: verification failed: %v", err)) helper.Fail500(w, r, fmt.Errorf("RequestBody: verification failed: %v", err))
return return
} }
} }
...@@ -63,7 +70,7 @@ func BodyUploader(rails PreAuthorizer, h http.Handler, p Preparer) http.Handler ...@@ -63,7 +70,7 @@ func BodyUploader(rails PreAuthorizer, h http.Handler, p Preparer) http.Handler
data := url.Values{} data := url.Values{}
fields, err := fh.GitLabFinalizeFields("file") fields, err := fh.GitLabFinalizeFields("file")
if err != nil { if err != nil {
helper.Fail500(w, r, fmt.Errorf("BodyUploader: finalize fields failed: %v", err)) helper.Fail500(w, r, fmt.Errorf("RequestBody: finalize fields failed: %v", err))
return return
} }
...@@ -80,7 +87,7 @@ func BodyUploader(rails PreAuthorizer, h http.Handler, p Preparer) http.Handler ...@@ -80,7 +87,7 @@ func BodyUploader(rails PreAuthorizer, h http.Handler, p Preparer) http.Handler
sft := SavedFileTracker{Request: r} sft := SavedFileTracker{Request: r}
sft.Track("file", fh.LocalPath) sft.Track("file", fh.LocalPath)
if err := sft.Finalize(r.Context()); err != nil { if err := sft.Finalize(r.Context()); err != nil {
helper.Fail500(w, r, fmt.Errorf("BodyUploader: finalize failed: %v", err)) helper.Fail500(w, r, fmt.Errorf("RequestBody: finalize failed: %v", err))
return return
} }
......
...@@ -24,7 +24,7 @@ const ( ...@@ -24,7 +24,7 @@ const (
fileLen = len(fileContent) fileLen = len(fileContent)
) )
func TestBodyUploader(t *testing.T) { func TestRequestBody(t *testing.T) {
testhelper.ConfigureSecret() testhelper.ConfigureSecret()
body := strings.NewReader(fileContent) body := strings.NewReader(fileContent)
...@@ -38,7 +38,7 @@ func TestBodyUploader(t *testing.T) { ...@@ -38,7 +38,7 @@ func TestBodyUploader(t *testing.T) {
require.Equal(t, fileContent, string(uploadEcho)) require.Equal(t, fileContent, string(uploadEcho))
} }
func TestBodyUploaderCustomPreparer(t *testing.T) { func TestRequestBodyCustomPreparer(t *testing.T) {
body := strings.NewReader(fileContent) body := strings.NewReader(fileContent)
resp := testUpload(&rails{}, &alwaysLocalPreparer{}, echoProxy(t, fileLen), body) resp := testUpload(&rails{}, &alwaysLocalPreparer{}, echoProxy(t, fileLen), body)
...@@ -49,7 +49,7 @@ func TestBodyUploaderCustomPreparer(t *testing.T) { ...@@ -49,7 +49,7 @@ func TestBodyUploaderCustomPreparer(t *testing.T) {
require.Equal(t, fileContent, string(uploadEcho)) require.Equal(t, fileContent, string(uploadEcho))
} }
func TestBodyUploaderCustomVerifier(t *testing.T) { func TestRequestBodyCustomVerifier(t *testing.T) {
body := strings.NewReader(fileContent) body := strings.NewReader(fileContent)
verifier := &mockVerifier{} verifier := &mockVerifier{}
...@@ -62,11 +62,11 @@ func TestBodyUploaderCustomVerifier(t *testing.T) { ...@@ -62,11 +62,11 @@ func TestBodyUploaderCustomVerifier(t *testing.T) {
require.True(t, verifier.invoked, "Verifier.Verify not invoked") require.True(t, verifier.invoked, "Verifier.Verify not invoked")
} }
func TestBodyUploaderAuthorizationFailure(t *testing.T) { func TestRequestBodyAuthorizationFailure(t *testing.T) {
testNoProxyInvocation(t, http.StatusUnauthorized, &rails{unauthorized: true}, &alwaysLocalPreparer{}) testNoProxyInvocation(t, http.StatusUnauthorized, &rails{unauthorized: true}, &alwaysLocalPreparer{})
} }
func TestBodyUploaderErrors(t *testing.T) { func TestRequestBodyErrors(t *testing.T) {
tests := []struct { tests := []struct {
name string name string
preparer *alwaysLocalPreparer preparer *alwaysLocalPreparer
...@@ -95,7 +95,7 @@ func testUpload(auth PreAuthorizer, preparer Preparer, proxy http.Handler, body ...@@ -95,7 +95,7 @@ func testUpload(auth PreAuthorizer, preparer Preparer, proxy http.Handler, body
req := httptest.NewRequest("POST", "http://example.com/upload", body) req := httptest.NewRequest("POST", "http://example.com/upload", body)
w := httptest.NewRecorder() w := httptest.NewRecorder()
BodyUploader(auth, proxy, preparer).ServeHTTP(w, req) RequestBody(auth, proxy, preparer).ServeHTTP(w, req)
return w.Result() return w.Result()
} }
......
...@@ -6,15 +6,15 @@ import ( ...@@ -6,15 +6,15 @@ import (
"gitlab.com/gitlab-org/gitlab/workhorse/internal/api" "gitlab.com/gitlab-org/gitlab/workhorse/internal/api"
) )
// SkipRailsAuthorizer implements a fake PreAuthorizer that do not calls rails API and // SkipRailsAuthorizer implements a fake PreAuthorizer that does not call
// authorize each call as a local only upload to TempPath // the gitlab-rails API. It must be fast because it gets called on each
// request proxied to Rails.
type SkipRailsAuthorizer struct { type SkipRailsAuthorizer struct {
// TempPath is the temporary path for a local only upload // TempPath is a directory where workhorse can store files that can later
// be accessed by gitlab-rails.
TempPath string TempPath string
} }
// PreAuthorizeHandler implements PreAuthorizer. It always grant the upload.
// The fake API response contains only TempPath
func (l *SkipRailsAuthorizer) PreAuthorizeHandler(next api.HandleFunc, _ string) http.Handler { func (l *SkipRailsAuthorizer) PreAuthorizeHandler(next api.HandleFunc, _ string) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
next(w, r, &api.Response{TempPath: l.TempPath}) next(w, r, &api.Response{TempPath: l.TempPath})
......
...@@ -15,7 +15,8 @@ import ( ...@@ -15,7 +15,8 @@ import (
"gitlab.com/gitlab-org/gitlab/workhorse/internal/zipartifacts" "gitlab.com/gitlab-org/gitlab/workhorse/internal/zipartifacts"
) )
// These methods are allowed to have thread-unsafe implementations. // MultipartFormProcessor abstracts away implementation differences
// between generic MIME multipart file uploads and CI artifact uploads.
type MultipartFormProcessor interface { type MultipartFormProcessor interface {
ProcessFile(ctx context.Context, formName string, file *filestore.FileHandler, writer *multipart.Writer) error ProcessFile(ctx context.Context, formName string, file *filestore.FileHandler, writer *multipart.Writer) error
ProcessField(ctx context.Context, formName string, writer *multipart.Writer) error ProcessField(ctx context.Context, formName string, writer *multipart.Writer) error
...@@ -24,7 +25,10 @@ type MultipartFormProcessor interface { ...@@ -24,7 +25,10 @@ type MultipartFormProcessor interface {
Count() int Count() int
} }
func HandleFileUploads(w http.ResponseWriter, r *http.Request, h http.Handler, preauth *api.Response, filter MultipartFormProcessor, opts *filestore.SaveFileOpts) { // InterceptMultipartFiles is the core of the implementation of
// Multipart. Because it is also used for CI artifact uploads it is a
// public function.
func InterceptMultipartFiles(w http.ResponseWriter, r *http.Request, h http.Handler, preauth *api.Response, filter MultipartFormProcessor, opts *filestore.SaveFileOpts) {
var body bytes.Buffer var body bytes.Buffer
writer := multipart.NewWriter(&body) writer := multipart.NewWriter(&body)
defer writer.Close() defer writer.Close()
......
...@@ -78,7 +78,7 @@ func TestUploadHandlerForwardingRawData(t *testing.T) { ...@@ -78,7 +78,7 @@ func TestUploadHandlerForwardingRawData(t *testing.T) {
opts, _, err := preparer.Prepare(apiResponse) opts, _, err := preparer.Prepare(apiResponse)
require.NoError(t, err) require.NoError(t, err)
HandleFileUploads(response, httpRequest, handler, apiResponse, nil, opts) InterceptMultipartFiles(response, httpRequest, handler, apiResponse, nil, opts)
require.Equal(t, 202, response.Code) require.Equal(t, 202, response.Code)
require.Equal(t, "RESPONSE", response.Body.String(), "response body") require.Equal(t, "RESPONSE", response.Body.String(), "response body")
...@@ -149,7 +149,7 @@ func TestUploadHandlerRewritingMultiPartData(t *testing.T) { ...@@ -149,7 +149,7 @@ func TestUploadHandlerRewritingMultiPartData(t *testing.T) {
opts, _, err := preparer.Prepare(apiResponse) opts, _, err := preparer.Prepare(apiResponse)
require.NoError(t, err) require.NoError(t, err)
HandleFileUploads(response, httpRequest, handler, apiResponse, &testFormProcessor{}, opts) InterceptMultipartFiles(response, httpRequest, handler, apiResponse, &testFormProcessor{}, opts)
require.Equal(t, 202, response.Code) require.Equal(t, 202, response.Code)
cancel() // this will trigger an async cleanup cancel() // this will trigger an async cleanup
...@@ -218,7 +218,7 @@ func TestUploadHandlerDetectingInjectedMultiPartData(t *testing.T) { ...@@ -218,7 +218,7 @@ func TestUploadHandlerDetectingInjectedMultiPartData(t *testing.T) {
opts, _, err := preparer.Prepare(apiResponse) opts, _, err := preparer.Prepare(apiResponse)
require.NoError(t, err) require.NoError(t, err)
HandleFileUploads(response, httpRequest, handler, apiResponse, &testFormProcessor{}, opts) InterceptMultipartFiles(response, httpRequest, handler, apiResponse, &testFormProcessor{}, opts)
require.Equal(t, test.response, response.Code) require.Equal(t, test.response, response.Code)
cancel() // this will trigger an async cleanup cancel() // this will trigger an async cleanup
...@@ -248,7 +248,7 @@ func TestUploadProcessingField(t *testing.T) { ...@@ -248,7 +248,7 @@ func TestUploadProcessingField(t *testing.T) {
opts, _, err := preparer.Prepare(apiResponse) opts, _, err := preparer.Prepare(apiResponse)
require.NoError(t, err) require.NoError(t, err)
HandleFileUploads(response, httpRequest, nilHandler, apiResponse, &testFormProcessor{}, opts) InterceptMultipartFiles(response, httpRequest, nilHandler, apiResponse, &testFormProcessor{}, opts)
require.Equal(t, 500, response.Code) require.Equal(t, 500, response.Code)
} }
...@@ -279,7 +279,7 @@ func TestUploadingMultipleFiles(t *testing.T) { ...@@ -279,7 +279,7 @@ func TestUploadingMultipleFiles(t *testing.T) {
opts, _, err := preparer.Prepare(apiResponse) opts, _, err := preparer.Prepare(apiResponse)
require.NoError(t, err) require.NoError(t, err)
HandleFileUploads(response, httpRequest, nilHandler, apiResponse, &testFormProcessor{}, opts) InterceptMultipartFiles(response, httpRequest, nilHandler, apiResponse, &testFormProcessor{}, opts)
require.Equal(t, 400, response.Code) require.Equal(t, 400, response.Code)
require.Equal(t, "upload request contains more than 10 files\n", response.Body.String()) require.Equal(t, "upload request contains more than 10 files\n", response.Body.String())
...@@ -335,7 +335,7 @@ func TestUploadProcessingFile(t *testing.T) { ...@@ -335,7 +335,7 @@ func TestUploadProcessingFile(t *testing.T) {
opts, _, err := preparer.Prepare(apiResponse) opts, _, err := preparer.Prepare(apiResponse)
require.NoError(t, err) require.NoError(t, err)
HandleFileUploads(response, httpRequest, nilHandler, apiResponse, &testFormProcessor{}, opts) InterceptMultipartFiles(response, httpRequest, nilHandler, apiResponse, &testFormProcessor{}, opts)
require.Equal(t, 200, response.Code) require.Equal(t, 200, response.Code)
}) })
...@@ -381,7 +381,7 @@ func TestInvalidFileNames(t *testing.T) { ...@@ -381,7 +381,7 @@ func TestInvalidFileNames(t *testing.T) {
opts, _, err := preparer.Prepare(apiResponse) opts, _, err := preparer.Prepare(apiResponse)
require.NoError(t, err) require.NoError(t, err)
HandleFileUploads(response, httpRequest, nilHandler, apiResponse, &SavedFileTracker{Request: httpRequest}, opts) InterceptMultipartFiles(response, httpRequest, nilHandler, apiResponse, &SavedFileTracker{Request: httpRequest}, opts)
require.Equal(t, testCase.code, response.Code) require.Equal(t, testCase.code, response.Code)
require.Equal(t, testCase.expectedPrefix, opts.TempFilePrefix) require.Equal(t, testCase.expectedPrefix, opts.TempFilePrefix)
} }
...@@ -447,7 +447,7 @@ func TestContentDispositionRewrite(t *testing.T) { ...@@ -447,7 +447,7 @@ func TestContentDispositionRewrite(t *testing.T) {
opts, _, err := preparer.Prepare(apiResponse) opts, _, err := preparer.Prepare(apiResponse)
require.NoError(t, err) require.NoError(t, err)
HandleFileUploads(response, httpRequest, customHandler, apiResponse, &SavedFileTracker{Request: httpRequest}, opts) InterceptMultipartFiles(response, httpRequest, customHandler, apiResponse, &SavedFileTracker{Request: httpRequest}, opts)
upstreamRequest, err := http.ReadRequest(bufio.NewReader(&upstreamRequestBuffer)) upstreamRequest, err := http.ReadRequest(bufio.NewReader(&upstreamRequestBuffer))
require.NoError(t, err) require.NoError(t, err)
...@@ -570,7 +570,7 @@ func runUploadTest(t *testing.T, image []byte, filename string, httpCode int, ts ...@@ -570,7 +570,7 @@ func runUploadTest(t *testing.T, image []byte, filename string, httpCode int, ts
opts, _, err := preparer.Prepare(apiResponse) opts, _, err := preparer.Prepare(apiResponse)
require.NoError(t, err) require.NoError(t, err)
HandleFileUploads(response, httpRequest, handler, apiResponse, &testFormProcessor{}, opts) InterceptMultipartFiles(response, httpRequest, handler, apiResponse, &testFormProcessor{}, opts)
require.Equal(t, httpCode, response.Code) require.Equal(t, httpCode, response.Code)
} }
......
...@@ -228,17 +228,17 @@ func configureRoutes(u *upstream) { ...@@ -228,17 +228,17 @@ func configureRoutes(u *upstream) {
preparers := createUploadPreparers(u.Config) preparers := createUploadPreparers(u.Config)
uploadPath := path.Join(u.DocumentRoot, "uploads/tmp") uploadPath := path.Join(u.DocumentRoot, "uploads/tmp")
uploadAccelerateProxy := upload.Accelerate(&upload.SkipRailsAuthorizer{TempPath: uploadPath}, proxy, preparers.uploads) tempfileMultipartProxy := upload.Multipart(&upload.SkipRailsAuthorizer{TempPath: uploadPath}, proxy, preparers.uploads)
ciAPIProxyQueue := queueing.QueueRequests("ci_api_job_requests", uploadAccelerateProxy, u.APILimit, u.APIQueueLimit, u.APIQueueTimeout) ciAPIProxyQueue := queueing.QueueRequests("ci_api_job_requests", tempfileMultipartProxy, u.APILimit, u.APIQueueLimit, u.APIQueueTimeout)
ciAPILongPolling := builds.RegisterHandler(ciAPIProxyQueue, redis.WatchKey, u.APICILongPollingDuration) ciAPILongPolling := builds.RegisterHandler(ciAPIProxyQueue, redis.WatchKey, u.APICILongPollingDuration)
dependencyProxyInjector.SetUploadHandler(upload.BodyUploader(api, signingProxy, preparers.packages)) dependencyProxyInjector.SetUploadHandler(upload.RequestBody(api, signingProxy, preparers.packages))
// Serve static files or forward the requests // Serve static files or forward the requests
defaultUpstream := static.ServeExisting( defaultUpstream := static.ServeExisting(
u.URLPrefix, u.URLPrefix,
staticpages.CacheDisabled, staticpages.CacheDisabled,
static.DeployPage(static.ErrorPagesUnless(u.DevelopmentMode, staticpages.ErrorFormatHTML, uploadAccelerateProxy)), static.DeployPage(static.ErrorPagesUnless(u.DevelopmentMode, staticpages.ErrorFormatHTML, tempfileMultipartProxy)),
) )
probeUpstream := static.ErrorPagesUnless(u.DevelopmentMode, staticpages.ErrorFormatJSON, proxy) probeUpstream := static.ErrorPagesUnless(u.DevelopmentMode, staticpages.ErrorFormatJSON, proxy)
healthUpstream := static.ErrorPagesUnless(u.DevelopmentMode, staticpages.ErrorFormatText, proxy) healthUpstream := static.ErrorPagesUnless(u.DevelopmentMode, staticpages.ErrorFormatText, proxy)
...@@ -248,7 +248,7 @@ func configureRoutes(u *upstream) { ...@@ -248,7 +248,7 @@ func configureRoutes(u *upstream) {
u.route("GET", gitProjectPattern+`info/refs\z`, git.GetInfoRefsHandler(api)), u.route("GET", gitProjectPattern+`info/refs\z`, git.GetInfoRefsHandler(api)),
u.route("POST", gitProjectPattern+`git-upload-pack\z`, contentEncodingHandler(git.UploadPack(api)), withMatcher(isContentType("application/x-git-upload-pack-request"))), u.route("POST", gitProjectPattern+`git-upload-pack\z`, contentEncodingHandler(git.UploadPack(api)), withMatcher(isContentType("application/x-git-upload-pack-request"))),
u.route("POST", gitProjectPattern+`git-receive-pack\z`, contentEncodingHandler(git.ReceivePack(api)), withMatcher(isContentType("application/x-git-receive-pack-request"))), u.route("POST", gitProjectPattern+`git-receive-pack\z`, contentEncodingHandler(git.ReceivePack(api)), withMatcher(isContentType("application/x-git-receive-pack-request"))),
u.route("PUT", gitProjectPattern+`gitlab-lfs/objects/([0-9a-f]{64})/([0-9]+)\z`, lfs.PutStore(api, signingProxy, preparers.lfs), withMatcher(isContentType("application/octet-stream"))), u.route("PUT", gitProjectPattern+`gitlab-lfs/objects/([0-9a-f]{64})/([0-9]+)\z`, upload.RequestBody(api, signingProxy, preparers.lfs), withMatcher(isContentType("application/octet-stream"))),
// CI Artifacts // CI Artifacts
u.route("POST", apiPattern+`v4/jobs/[0-9]+/artifacts\z`, contentEncodingHandler(artifacts.UploadArtifacts(api, signingProxy, preparers.artifacts))), u.route("POST", apiPattern+`v4/jobs/[0-9]+/artifacts\z`, contentEncodingHandler(artifacts.UploadArtifacts(api, signingProxy, preparers.artifacts))),
...@@ -276,56 +276,56 @@ func configureRoutes(u *upstream) { ...@@ -276,56 +276,56 @@ func configureRoutes(u *upstream) {
// https://gitlab.com/gitlab-org/gitlab/-/merge_requests/56731. // https://gitlab.com/gitlab-org/gitlab/-/merge_requests/56731.
// Maven Artifact Repository // Maven Artifact Repository
u.route("PUT", apiProjectPattern+`packages/maven/`, upload.BodyUploader(api, signingProxy, preparers.packages)), u.route("PUT", apiProjectPattern+`packages/maven/`, upload.RequestBody(api, signingProxy, preparers.packages)),
// Conan Artifact Repository // Conan Artifact Repository
u.route("PUT", apiPattern+`v4/packages/conan/`, upload.BodyUploader(api, signingProxy, preparers.packages)), u.route("PUT", apiPattern+`v4/packages/conan/`, upload.RequestBody(api, signingProxy, preparers.packages)),
u.route("PUT", apiProjectPattern+`packages/conan/`, upload.BodyUploader(api, signingProxy, preparers.packages)), u.route("PUT", apiProjectPattern+`packages/conan/`, upload.RequestBody(api, signingProxy, preparers.packages)),
// Generic Packages Repository // Generic Packages Repository
u.route("PUT", apiProjectPattern+`packages/generic/`, upload.BodyUploader(api, signingProxy, preparers.packages)), u.route("PUT", apiProjectPattern+`packages/generic/`, upload.RequestBody(api, signingProxy, preparers.packages)),
// NuGet Artifact Repository // NuGet Artifact Repository
u.route("PUT", apiProjectPattern+`packages/nuget/`, upload.Accelerate(api, signingProxy, preparers.packages)), u.route("PUT", apiProjectPattern+`packages/nuget/`, upload.Multipart(api, signingProxy, preparers.packages)),
// PyPI Artifact Repository // PyPI Artifact Repository
u.route("POST", apiProjectPattern+`packages/pypi`, upload.Accelerate(api, signingProxy, preparers.packages)), u.route("POST", apiProjectPattern+`packages/pypi`, upload.Multipart(api, signingProxy, preparers.packages)),
// Debian Artifact Repository // Debian Artifact Repository
u.route("PUT", apiProjectPattern+`packages/debian/`, upload.BodyUploader(api, signingProxy, preparers.packages)), u.route("PUT", apiProjectPattern+`packages/debian/`, upload.RequestBody(api, signingProxy, preparers.packages)),
// Gem Artifact Repository // Gem Artifact Repository
u.route("POST", apiProjectPattern+`packages/rubygems/`, upload.BodyUploader(api, signingProxy, preparers.packages)), u.route("POST", apiProjectPattern+`packages/rubygems/`, upload.RequestBody(api, signingProxy, preparers.packages)),
// Terraform Module Package Repository // Terraform Module Package Repository
u.route("PUT", apiProjectPattern+`packages/terraform/modules/`, upload.BodyUploader(api, signingProxy, preparers.packages)), u.route("PUT", apiProjectPattern+`packages/terraform/modules/`, upload.RequestBody(api, signingProxy, preparers.packages)),
// Helm Artifact Repository // Helm Artifact Repository
u.route("POST", apiProjectPattern+`packages/helm/api/[^/]+/charts\z`, upload.Accelerate(api, signingProxy, preparers.packages)), u.route("POST", apiProjectPattern+`packages/helm/api/[^/]+/charts\z`, upload.Multipart(api, signingProxy, preparers.packages)),
// We are porting API to disk acceleration // We are porting API to disk acceleration
// we need to declare each routes until we have fixed all the routes on the rails codebase. // we need to declare each routes until we have fixed all the routes on the rails codebase.
// Overall status can be seen at https://gitlab.com/groups/gitlab-org/-/epics/1802#current-status // Overall status can be seen at https://gitlab.com/groups/gitlab-org/-/epics/1802#current-status
u.route("POST", apiProjectPattern+`wikis/attachments\z`, uploadAccelerateProxy), u.route("POST", apiProjectPattern+`wikis/attachments\z`, tempfileMultipartProxy),
u.route("POST", apiPattern+`graphql\z`, uploadAccelerateProxy), u.route("POST", apiPattern+`graphql\z`, tempfileMultipartProxy),
u.route("POST", apiTopicPattern, uploadAccelerateProxy), u.route("POST", apiTopicPattern, tempfileMultipartProxy),
u.route("PUT", apiTopicPattern, uploadAccelerateProxy), u.route("PUT", apiTopicPattern, tempfileMultipartProxy),
u.route("POST", apiPattern+`v4/groups/import`, upload.Accelerate(api, signingProxy, preparers.uploads)), u.route("POST", apiPattern+`v4/groups/import`, upload.Multipart(api, signingProxy, preparers.uploads)),
u.route("POST", apiPattern+`v4/projects/import`, upload.Accelerate(api, signingProxy, preparers.uploads)), u.route("POST", apiPattern+`v4/projects/import`, upload.Multipart(api, signingProxy, preparers.uploads)),
// Project Import via UI upload acceleration // Project Import via UI upload acceleration
u.route("POST", importPattern+`gitlab_project`, upload.Accelerate(api, signingProxy, preparers.uploads)), u.route("POST", importPattern+`gitlab_project`, upload.Multipart(api, signingProxy, preparers.uploads)),
// Group Import via UI upload acceleration // Group Import via UI upload acceleration
u.route("POST", importPattern+`gitlab_group`, upload.Accelerate(api, signingProxy, preparers.uploads)), u.route("POST", importPattern+`gitlab_group`, upload.Multipart(api, signingProxy, preparers.uploads)),
// Metric image upload // Metric image upload
u.route("POST", apiProjectPattern+`issues/[0-9]+/metric_images\z`, upload.Accelerate(api, signingProxy, preparers.uploads)), u.route("POST", apiProjectPattern+`issues/[0-9]+/metric_images\z`, upload.Multipart(api, signingProxy, preparers.uploads)),
// Requirements Import via UI upload acceleration // Requirements Import via UI upload acceleration
u.route("POST", projectPattern+`requirements_management/requirements/import_csv`, upload.Accelerate(api, signingProxy, preparers.uploads)), u.route("POST", projectPattern+`requirements_management/requirements/import_csv`, upload.Multipart(api, signingProxy, preparers.uploads)),
// Uploads via API // Uploads via API
u.route("POST", apiProjectPattern+`uploads\z`, upload.Accelerate(api, signingProxy, preparers.uploads)), u.route("POST", apiProjectPattern+`uploads\z`, upload.Multipart(api, signingProxy, preparers.uploads)),
// Explicitly proxy API requests // Explicitly proxy API requests
u.route("", apiPattern, proxy), u.route("", apiPattern, proxy),
...@@ -343,9 +343,9 @@ func configureRoutes(u *upstream) { ...@@ -343,9 +343,9 @@ func configureRoutes(u *upstream) {
), ),
// Uploads // Uploads
u.route("POST", projectPattern+`uploads\z`, upload.Accelerate(api, signingProxy, preparers.uploads)), u.route("POST", projectPattern+`uploads\z`, upload.Multipart(api, signingProxy, preparers.uploads)),
u.route("POST", snippetUploadPattern, upload.Accelerate(api, signingProxy, preparers.uploads)), u.route("POST", snippetUploadPattern, upload.Multipart(api, signingProxy, preparers.uploads)),
u.route("POST", userUploadPattern, upload.Accelerate(api, signingProxy, preparers.uploads)), u.route("POST", userUploadPattern, upload.Multipart(api, signingProxy, preparers.uploads)),
// health checks don't intercept errors and go straight to rails // health checks don't intercept errors and go straight to rails
// TODO: We should probably not return a HTML deploy page? // TODO: We should probably not return a HTML deploy page?
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment