Commit e0a2ea65 authored by Kamil Trzcinski's avatar Kamil Trzcinski

Refactor and implement tests for artifacts upload (injecting the metadata) and single file download

parent 49b0cdef
......@@ -3,7 +3,6 @@ package artifacts
import (
"../api"
"../helper"
"../upload"
"archive/zip"
"encoding/base64"
"errors"
......@@ -15,15 +14,58 @@ import (
"strconv"
)
func UploadArtifacts(myAPI *api.API, h http.Handler) http.Handler {
return myAPI.PreAuthorizeHandler(func(w http.ResponseWriter, r *http.Request, a *api.Response) {
if a.TempPath == "" {
helper.Fail500(w, errors.New("UploadArtifacts: TempPath is empty"))
return
func decodeFileEntry(entry string) (string, error) {
decoded, err := base64.StdEncoding.DecodeString(entry)
if err != nil {
return "", err
}
return string(decoded), nil
}
func detectFileContentType(fileName string) string {
contentType := mime.TypeByExtension(filepath.Ext(fileName))
if contentType == "" {
contentType = "application/octet-stream"
}
return contentType
}
func findFileInZip(fileName string, archive *zip.Reader) *zip.File {
for _, file := range archive.File {
if file.Name == fileName {
return file
}
}
return nil
}
func unpackFileFromZip(archiveFileName, fileName string, headers http.Header, output io.Writer) error {
archive, err := zip.OpenReader(archiveFileName)
if err != nil {
return err
}
defer archive.Close()
file := findFileInZip(fileName, &archive.Reader)
if file == nil {
return os.ErrNotExist
}
upload.HandleFileUploads(w, r, h, a.TempPath, &artifactsFormFilter{})
}, "/authorize")
// Start decompressing the file
reader, err := file.Open()
if err != nil {
return err
}
defer reader.Close()
// Write http headers about the file
headers.Set("Content-Length", strconv.FormatInt(int64(file.UncompressedSize64), 10))
headers.Set("Content-Type", detectFileContentType(file.Name))
headers.Set("Content-Disposition", "attachment; filename=\""+filepath.Base(file.Name)+"\"")
// Copy file body to client
_, err = io.Copy(output, reader)
return err
}
// Artifacts downloader doesn't support ranges when downloading a single file
......@@ -34,51 +76,18 @@ func DownloadArtifact(myAPI *api.API) http.Handler {
return
}
fileNameDecoded, err := base64.StdEncoding.DecodeString(a.Entry)
fileName, err := decodeFileEntry(a.Entry)
if err != nil {
helper.Fail500(w, err)
return
}
fileName := string(fileNameDecoded)
// TODO:
// This should be moved to sub process to reduce memory pressue on workhorse
archive, err := zip.OpenReader(a.Archive)
err = unpackFileFromZip(a.Archive, fileName, w.Header(), w)
if os.IsNotExist(err) {
http.NotFound(w, r)
return
} else if err != nil {
helper.Fail500(w, err)
}
defer archive.Close()
var file *zip.File
for _, file = range archive.File {
if file.Name == fileName {
break
}
}
if file == nil {
http.NotFound(w, r)
return
}
contentType := mime.TypeByExtension(filepath.Ext(file.Name))
if contentType == "" {
contentType = "application/octet-stream"
}
w.Header().Set("Content-Length", strconv.FormatInt(int64(file.UncompressedSize64), 10))
w.Header().Set("Content-Type", contentType)
w.Header().Set("Content-Disposition", "attachment; filename=\""+filepath.Base(file.Name)+"\"")
reader, err := file.Open()
if err != nil {
helper.Fail500(w, err)
}
defer reader.Close()
// Copy file body
io.Copy(w, reader)
}, "")
}
package artifacts
import (
"../api"
"../helper"
"../testhelper"
"archive/zip"
"encoding/base64"
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"net/http/httptest"
"os"
"testing"
)
func testArtifactDownloadServer(t *testing.T, archive string, entry string) *httptest.Server {
mux := http.NewServeMux()
mux.HandleFunc("/url/path", func(w http.ResponseWriter, r *http.Request) {
if r.Method != "GET" {
t.Fatal("Expected GET request")
}
w.Header().Set("Content-Type", "application/json")
data, err := json.Marshal(&api.Response{
Archive: archive,
Entry: base64.StdEncoding.EncodeToString([]byte(entry)),
})
if err != nil {
t.Fatal("Expected to marshal")
}
w.Write(data)
})
return testhelper.TestServerWithHandler(nil, mux.ServeHTTP)
}
func testDownloadArtifact(t *testing.T, ts *httptest.Server) *httptest.ResponseRecorder {
httpRequest, err := http.NewRequest("GET", ts.URL+"/url/path", nil)
if err != nil {
t.Fatal(err)
}
response := httptest.NewRecorder()
apiClient := api.NewAPI(helper.URLMustParse(ts.URL), "123", nil)
DownloadArtifact(apiClient).ServeHTTP(response, httpRequest)
return response
}
func TestDownloadingFromValidArchive(t *testing.T) {
tempFile, err := ioutil.TempFile("", "uploads")
if err != nil {
t.Fatal(err)
}
defer tempFile.Close()
defer os.Remove(tempFile.Name())
archive := zip.NewWriter(tempFile)
defer archive.Close()
fileInArchive, err := archive.Create("test.txt")
if err != nil {
t.Fatal(err)
}
fmt.Fprint(fileInArchive, "testtest")
archive.Close()
ts := testArtifactDownloadServer(t, tempFile.Name(), "test.txt")
defer ts.Close()
response := testDownloadArtifact(t, ts)
testhelper.AssertResponseCode(t, response, 200)
testhelper.AssertResponseHeader(t, response, "Content-Type", "text/plain; charset=utf-8")
testhelper.AssertResponseBody(t, response, "testtest")
}
func TestDownloadingNonExistingFile(t *testing.T) {
tempFile, err := ioutil.TempFile("", "uploads")
if err != nil {
t.Fatal(err)
}
defer tempFile.Close()
defer os.Remove(tempFile.Name())
archive := zip.NewWriter(tempFile)
defer archive.Close()
archive.Close()
ts := testArtifactDownloadServer(t, tempFile.Name(), "test")
defer ts.Close()
response := testDownloadArtifact(t, ts)
testhelper.AssertResponseCode(t, response, 404)
}
func TestDownloadingFromInvalidArchive(t *testing.T) {
ts := testArtifactDownloadServer(t, "path/to/non/existing/file", "test")
defer ts.Close()
response := testDownloadArtifact(t, ts)
testhelper.AssertResponseCode(t, response, 404)
}
func TestIncompleteApiResponse(t *testing.T) {
ts := testArtifactDownloadServer(t, "", "")
defer ts.Close()
response := testDownloadArtifact(t, ts)
testhelper.AssertResponseCode(t, response, 500)
}
package artifacts
import (
"../api"
"../helper"
"../upload"
"errors"
"fmt"
"io/ioutil"
"mime/multipart"
"net/http"
"os"
)
// The artifactsFormFilter allows to pass only the `file` as file in body
type artifactsUploadProcessor struct {
TempPath string
metadataFile string
}
func (a *artifactsUploadProcessor) ProcessFile(formName, fileName string, writer *multipart.Writer) error {
if formName != "file" {
return fmt.Errorf("Invalid form field: %q", formName)
}
if a.metadataFile != "" {
return fmt.Errorf("Multiple files")
}
// Create temporary file for metadata and store it's path
tempFile, err := ioutil.TempFile(a.TempPath, "metadata_")
if err != nil {
return err
}
defer tempFile.Close()
a.metadataFile = tempFile.Name()
// Generate metadata and save to file
err = generateZipMetadataFromFile(fileName, tempFile)
if err == os.ErrInvalid {
return nil
} else if err != nil {
return err
}
// Pass metadata file path to Rails
writer.WriteField("metadata.path", a.metadataFile)
writer.WriteField("metadata.name", "metadata.gz")
return nil
}
func (a *artifactsUploadProcessor) ProcessField(formName string, writer *multipart.Writer) error {
return nil
}
func (a *artifactsUploadProcessor) Cleanup() {
if a.metadataFile != "" {
os.Remove(a.metadataFile)
}
}
func UploadArtifacts(myAPI *api.API, h http.Handler) http.Handler {
return myAPI.PreAuthorizeHandler(func(w http.ResponseWriter, r *http.Request, a *api.Response) {
if a.TempPath == "" {
helper.Fail500(w, errors.New("UploadArtifacts: TempPath is empty"))
return
}
mg := &artifactsUploadProcessor{TempPath: a.TempPath}
defer mg.Cleanup()
upload.HandleFileUploads(w, r, h, a.TempPath, mg)
}, "/authorize")
}
package artifacts
import (
"../api"
"../helper"
"../proxy"
"../testhelper"
"archive/zip"
"bytes"
"compress/gzip"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"mime/multipart"
"net/http"
"net/http/httptest"
"os"
"testing"
)
func testArtifactsUploadServer(t *testing.T, tempPath string) *httptest.Server {
mux := http.NewServeMux()
mux.HandleFunc("/url/path/authorize", func(w http.ResponseWriter, r *http.Request) {
if r.Method != "POST" {
t.Fatal("Expected POST request")
}
w.Header().Set("Content-Type", "application/json")
data, err := json.Marshal(&api.Response{
TempPath: tempPath,
})
if err != nil {
t.Fatal("Expected to marshal")
}
w.Write(data)
})
mux.HandleFunc("/url/path", func(w http.ResponseWriter, r *http.Request) {
if r.Method != "POST" {
t.Fatal("Expected POST request")
}
if r.FormValue("file.path") == "" {
w.WriteHeader(501)
return
}
if r.FormValue("metadata.path") == "" {
w.WriteHeader(502)
return
}
_, err := ioutil.ReadFile(r.FormValue("file.path"))
if err != nil {
w.WriteHeader(404)
return
}
metadata, err := ioutil.ReadFile(r.FormValue("metadata.path"))
if err != nil {
w.WriteHeader(404)
return
}
gz, err := gzip.NewReader(bytes.NewReader(metadata))
if err != nil {
w.WriteHeader(405)
return
}
defer gz.Close()
metadata, err = ioutil.ReadAll(gz)
if err != nil {
w.WriteHeader(404)
return
}
if !bytes.Contains(metadata, []byte(metadataHeader)) {
w.WriteHeader(400)
return
}
w.WriteHeader(200)
})
return testhelper.TestServerWithHandler(nil, mux.ServeHTTP)
}
func testUploadArtifacts(contentType string, body io.Reader, t *testing.T, ts *httptest.Server) *httptest.ResponseRecorder {
httpRequest, err := http.NewRequest("POST", ts.URL+"/url/path", body)
if err != nil {
t.Fatal(err)
}
httpRequest.Header.Set("Content-Type", contentType)
response := httptest.NewRecorder()
apiClient := api.NewAPI(helper.URLMustParse(ts.URL), "123", nil)
proxyClient := proxy.NewProxy(helper.URLMustParse(ts.URL), "123", nil)
UploadArtifacts(apiClient, proxyClient).ServeHTTP(response, httpRequest)
return response
}
func TestUploadHandlerAddingMetadata(t *testing.T) {
tempPath, err := ioutil.TempDir("", "uploads")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tempPath)
ts := testArtifactsUploadServer(t, tempPath)
defer ts.Close()
var buffer bytes.Buffer
writer := multipart.NewWriter(&buffer)
file, err := writer.CreateFormFile("file", "my.file")
if err != nil {
t.Fatal(err)
}
archive := zip.NewWriter(file)
defer archive.Close()
fileInArchive, err := archive.Create("test.file")
if err != nil {
t.Fatal(err)
}
fmt.Fprint(fileInArchive, "test")
archive.Close()
writer.Close()
response := testUploadArtifacts(writer.FormDataContentType(), &buffer, t, ts)
testhelper.AssertResponseCode(t, response, 200)
}
func TestUploadHandlerForUnsupportedArchive(t *testing.T) {
tempPath, err := ioutil.TempDir("", "uploads")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tempPath)
ts := testArtifactsUploadServer(t, tempPath)
defer ts.Close()
var buffer bytes.Buffer
writer := multipart.NewWriter(&buffer)
file, err := writer.CreateFormFile("file", "my.file")
if err != nil {
t.Fatal(err)
}
fmt.Fprint(file, "test")
writer.Close()
response := testUploadArtifacts(writer.FormDataContentType(), &buffer, t, ts)
testhelper.AssertResponseCode(t, response, 502)
}
func TestUploadFormProcessing(t *testing.T) {
tempPath, err := ioutil.TempDir("", "uploads")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tempPath)
ts := testArtifactsUploadServer(t, tempPath)
defer ts.Close()
var buffer bytes.Buffer
writer := multipart.NewWriter(&buffer)
file, err := writer.CreateFormFile("metadata", "my.file")
if err != nil {
t.Fatal(err)
}
fmt.Fprint(file, "test")
writer.Close()
response := testUploadArtifacts(writer.FormDataContentType(), &buffer, t, ts)
testhelper.AssertResponseCode(t, response, 500)
}
......@@ -2,9 +2,11 @@ package artifacts
import (
"archive/zip"
"compress/gzip"
"encoding/binary"
"encoding/json"
"io"
"os"
"strconv"
)
......@@ -17,6 +19,8 @@ type metadata struct {
Comment string `json:"comment,omitempty"`
}
const metadataHeader = "GitLab Build Artifacts Metadata 0.0.2\n"
func newMetadata(file *zip.File) metadata {
return metadata{
Modified: file.ModTime().Unix(),
......@@ -28,7 +32,7 @@ func newMetadata(file *zip.File) metadata {
}
}
func (m metadata) write(output io.Writer) error {
func (m metadata) writeEncoded(output io.Writer) error {
j, err := json.Marshal(m)
if err != nil {
return err
......@@ -37,23 +41,34 @@ func (m metadata) write(output io.Writer) error {
return writeBytes(output, j)
}
func writeZipEntryMetadata(output io.Writer, entry *zip.File) error {
err := writeString(output, entry.Name)
if err != nil {
return err
}
err = newMetadata(entry).writeEncoded(output)
if err != nil {
return err
}
return nil
}
func generateZipMetadata(output io.Writer, archive *zip.Reader) error {
err := writeString(output, "GitLab Build Artifacts Metadata 0.0.2\n")
err := writeString(output, metadataHeader)
if err != nil {
return err
}
// Write empty error string
err = writeString(output, "{}")
if err != nil {
return err
}
// Write all files
for _, entry := range archive.File {
err = writeString(output, entry.Name)
if err != nil {
return err
}
err = newMetadata(entry).write(output)
err = writeZipEntryMetadata(output, entry)
if err != nil {
return err
}
......@@ -61,6 +76,20 @@ func generateZipMetadata(output io.Writer, archive *zip.Reader) error {
return nil
}
func generateZipMetadataFromFile(fileName string, w io.Writer) error {
archive, err := zip.OpenReader(fileName)
if err != nil {
// Ignore non-zip archives
return os.ErrInvalid
}
defer archive.Close()
gz := gzip.NewWriter(w)
defer gz.Close()
return generateZipMetadata(gz, &archive.Reader)
}
func writeBytes(output io.Writer, data []byte) error {
err := binary.Write(output, binary.BigEndian, uint32(len(data)))
if err == nil {
......
package artifacts
import (
"mime/multipart"
"fmt"
"archive/zip"
"compress/gzip"
)
type artifactsFormFilter struct {
}
func (a *artifactsFormFilter) FilterFile(formName, fileName string, writer *multipart.Writer) error {
if formName != "file" {
return fmt.Errorf("Invalid form field: %q", formName)
}
archive, err := zip.OpenReader(fileName)
if err != nil {
// Ignore non-zip archives
return nil
}
defer archive.Close()
// TODO:
// we could create a temporary file and save to this file instead of writing to mulipart.Writer
// doing it like this is simpler, but puts more pressure on memory
metadataFile, err := writer.CreateFormFile("metadata", "metadata.gz")
if err != nil {
return err
}
defer writer.Close()
gz := gzip.NewWriter(metadataFile)
defer gz.Close()
err = generateZipMetadata(gz, &archive.Reader)
if err != nil {
return err
}
return nil
}
func (a *artifactsFormFilter) FilterField(formName string, writer *multipart.Writer) error {
return nil
}
......@@ -11,14 +11,12 @@ import (
"os"
)
const tempPathHeader = "Gitlab-Workhorse-Temp-Path"
type MultipartFormFilter interface {
FilterFile(formName, fileName string, writer *multipart.Writer) error
FilterField(formName string, writer *multipart.Writer) error
type MultipartFormProcessor interface {
ProcessFile(formName, fileName string, writer *multipart.Writer) error
ProcessField(formName string, writer *multipart.Writer) error
}
func rewriteFormFilesFromMultipart(r *http.Request, writer *multipart.Writer, tempPath string, filter MultipartFormFilter) (cleanup func(), err error) {
func rewriteFormFilesFromMultipart(r *http.Request, writer *multipart.Writer, tempPath string, filter MultipartFormProcessor) (cleanup func(), err error) {
// Create multipart reader
reader, err := r.MultipartReader()
if err != nil {
......@@ -78,7 +76,7 @@ func rewriteFormFilesFromMultipart(r *http.Request, writer *multipart.Writer, te
file.Close()
if filter != nil {
err = filter.FilterFile(name, file.Name(), writer)
err = filter.ProcessFile(name, file.Name(), writer)
if err != nil {
return cleanup, err
}
......@@ -95,7 +93,7 @@ func rewriteFormFilesFromMultipart(r *http.Request, writer *multipart.Writer, te
}
if filter != nil {
err = filter.FilterField(name, writer)
err = filter.ProcessField(name, writer)
if err != nil {
return cleanup, err
}
......@@ -105,7 +103,12 @@ func rewriteFormFilesFromMultipart(r *http.Request, writer *multipart.Writer, te
return cleanup, nil
}
func HandleFileUploads(w http.ResponseWriter, r *http.Request, h http.Handler, tempPath string, filter MultipartFormFilter) {
func HandleFileUploads(w http.ResponseWriter, r *http.Request, h http.Handler, tempPath string, filter MultipartFormProcessor) {
if tempPath == "" {
helper.Fail500(w, fmt.Errorf("handleFileUploads: temporary path not defined"))
return
}
var body bytes.Buffer
writer := multipart.NewWriter(&body)
defer writer.Close()
......
......@@ -5,6 +5,7 @@ import (
"../proxy"
"../testhelper"
"bytes"
"errors"
"fmt"
"io"
"io/ioutil"
......@@ -19,10 +20,27 @@ import (
var nilHandler = http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})
type testFormProcessor struct {
}
func (a *testFormProcessor) ProcessFile(formName, fileName string, writer *multipart.Writer) error {
if formName != "file" && fileName != "my.file" {
return errors.New("illegal file")
}
return nil
}
func (a *testFormProcessor) ProcessField(formName string, writer *multipart.Writer) error {
if formName != "token" {
return errors.New("illegal field")
}
return nil
}
func TestUploadTempPathRequirement(t *testing.T) {
response := httptest.NewRecorder()
request := &http.Request{}
handleFileUploads(nilHandler).ServeHTTP(response, request)
HandleFileUploads(response, request, nilHandler, "", nil)
testhelper.AssertResponseCode(t, response, 500)
}
......@@ -56,9 +74,8 @@ func TestUploadHandlerForwardingRawData(t *testing.T) {
response := httptest.NewRecorder()
httpRequest.Header.Set(tempPathHeader, tempPath)
handleFileUploads(proxy.NewProxy(helper.URLMustParse(ts.URL), "123", nil)).ServeHTTP(response, httpRequest)
handler := proxy.NewProxy(helper.URLMustParse(ts.URL), "123", nil)
HandleFileUploads(response, httpRequest, handler, tempPath, nil)
testhelper.AssertResponseCode(t, response, 202)
if response.Body.String() != "RESPONSE" {
t.Fatal("Expected RESPONSE in response body")
......@@ -129,13 +146,65 @@ func TestUploadHandlerRewritingMultiPartData(t *testing.T) {
httpRequest.Body = ioutil.NopCloser(&buffer)
httpRequest.ContentLength = int64(buffer.Len())
httpRequest.Header.Set("Content-Type", writer.FormDataContentType())
httpRequest.Header.Set(tempPathHeader, tempPath)
response := httptest.NewRecorder()
handleFileUploads(proxy.NewProxy(helper.URLMustParse(ts.URL), "123", nil)).ServeHTTP(response, httpRequest)
handler := proxy.NewProxy(helper.URLMustParse(ts.URL), "123", nil)
HandleFileUploads(response, httpRequest, handler, tempPath, &testFormProcessor{})
testhelper.AssertResponseCode(t, response, 202)
if _, err := os.Stat(filePath); !os.IsNotExist(err) {
t.Fatal("expected the file to be deleted")
}
}
func TestUploadProcessingField(t *testing.T) {
tempPath, err := ioutil.TempDir("", "uploads")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tempPath)
var buffer bytes.Buffer
writer := multipart.NewWriter(&buffer)
writer.WriteField("token2", "test")
writer.Close()
httpRequest, err := http.NewRequest("PUT", "/url/path", &buffer)
if err != nil {
t.Fatal(err)
}
httpRequest.Header.Set("Content-Type", writer.FormDataContentType())
response := httptest.NewRecorder()
HandleFileUploads(response, httpRequest, nilHandler, tempPath, &testFormProcessor{})
testhelper.AssertResponseCode(t, response, 500)
}
func TestUploadProcessingFile(t *testing.T) {
tempPath, err := ioutil.TempDir("", "uploads")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tempPath)
var buffer bytes.Buffer
writer := multipart.NewWriter(&buffer)
file, err := writer.CreateFormFile("file2", "my.file")
if err != nil {
t.Fatal(err)
}
fmt.Fprint(file, "test")
writer.Close()
httpRequest, err := http.NewRequest("PUT", "/url/path", &buffer)
if err != nil {
t.Fatal(err)
}
httpRequest.Header.Set("Content-Type", writer.FormDataContentType())
response := httptest.NewRecorder()
HandleFileUploads(response, httpRequest, nilHandler, tempPath, &testFormProcessor{})
testhelper.AssertResponseCode(t, response, 500)
}
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment