Commit 8aa52cff authored by Jacob Vosmaer's avatar Jacob Vosmaer

Generate zip metadata in a subprocess

parent 1629812b
...@@ -2,11 +2,14 @@ PREFIX=/usr/local ...@@ -2,11 +2,14 @@ PREFIX=/usr/local
VERSION=$(shell git describe)-$(shell date -u +%Y%m%d.%H%M%S) VERSION=$(shell git describe)-$(shell date -u +%Y%m%d.%H%M%S)
GOBUILD=go build -ldflags "-X main.Version=${VERSION}" GOBUILD=go build -ldflags "-X main.Version=${VERSION}"
all: gitlab-zip-cat gitlab-workhorse all: gitlab-zip-cat gitlab-zip-metadata gitlab-workhorse
gitlab-zip-cat: $(shell find cmd/gitlab-zip-cat/ -name '*.go') gitlab-zip-cat: $(shell find cmd/gitlab-zip-cat/ -name '*.go')
${GOBUILD} -o $@ ./cmd/$@ ${GOBUILD} -o $@ ./cmd/$@
gitlab-zip-metadata: $(shell find cmd/gitlab-zip-metadata/ -name '*.go')
${GOBUILD} -o $@ ./cmd/$@
gitlab-workhorse: $(shell find . -name '*.go') gitlab-workhorse: $(shell find . -name '*.go')
${GOBUILD} -o $@ ${GOBUILD} -o $@
...@@ -14,7 +17,7 @@ install: gitlab-workhorse gitlab-zip-cat ...@@ -14,7 +17,7 @@ install: gitlab-workhorse gitlab-zip-cat
install gitlab-workhorse gitlab-zip-cat ${PREFIX}/bin/ install gitlab-workhorse gitlab-zip-cat ${PREFIX}/bin/
.PHONY: test .PHONY: test
test: testdata/data/group/test.git clean-workhorse gitlab-workhorse gitlab-zip-cat test: testdata/data/group/test.git clean-workhorse all
go fmt ./... | awk '{ print } END { if (NR > 0) { print "Please run go fmt"; exit 1 } }' go fmt ./... | awk '{ print } END { if (NR > 0) { print "Please run go fmt"; exit 1 } }'
support/path-add-current go test ./... support/path-add-current go test ./...
@echo SUCCESS @echo SUCCESS
...@@ -36,4 +39,4 @@ clean: clean-workhorse ...@@ -36,4 +39,4 @@ clean: clean-workhorse
.PHONY: clean-workhorse .PHONY: clean-workhorse
clean-workhorse: clean-workhorse:
rm -f gitlab-workhorse gitlab-zip-cat rm -f gitlab-workhorse gitlab-zip-cat gitlab-zip-metadata
...@@ -25,6 +25,11 @@ func main() { ...@@ -25,6 +25,11 @@ func main() {
os.Exit(0) os.Exit(0)
} }
if len(os.Args) != 3 {
fmt.Fprintf(os.Stderr, "Usage: %s FILE.ZIP ENTRY", progName)
os.Exit(1)
}
archiveFileName := os.Args[1] archiveFileName := os.Args[1]
fileName := os.Args[2] fileName := os.Args[2]
archive, err := zip.OpenReader(archiveFileName) archive, err := zip.OpenReader(archiveFileName)
......
package main
import (
"../../internal/zipmetadata"
"flag"
"fmt"
"os"
)
const progName = "gitlab-zip-metadata"
var Version = "unknown"
var printVersion = flag.Bool("version", false, "Print version and exit")
func main() {
flag.Parse()
version := fmt.Sprintf("%s %s", progName, Version)
if *printVersion {
fmt.Println(version)
os.Exit(0)
}
if len(os.Args) != 2 {
fmt.Fprintf(os.Stderr, "Usage: %s FILE.ZIP", progName)
os.Exit(1)
}
if err := zipmetadata.GenerateZipMetadataFromFile(os.Args[1], os.Stdout); err != nil {
fmt.Fprintf(os.Stderr, "%s: %v\n", progName, err)
if err == os.ErrInvalid {
os.Exit(zipmetadata.StatusNotZip)
}
os.Exit(1)
}
}
...@@ -39,6 +39,7 @@ func detectFileContentType(fileName string) string { ...@@ -39,6 +39,7 @@ func detectFileContentType(fileName string) string {
func unpackFileFromZip(archiveFileName, fileName string, headers http.Header, output io.Writer) error { func unpackFileFromZip(archiveFileName, fileName string, headers http.Header, output io.Writer) error {
catFile := exec.Command("gitlab-zip-cat", archiveFileName, fileName) catFile := exec.Command("gitlab-zip-cat", archiveFileName, fileName)
catFile.Stderr = os.Stderr
catFile.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} catFile.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
stdout, err := catFile.StdoutPipe() stdout, err := catFile.StdoutPipe()
if err != nil { if err != nil {
......
...@@ -4,12 +4,15 @@ import ( ...@@ -4,12 +4,15 @@ import (
"../api" "../api"
"../helper" "../helper"
"../upload" "../upload"
"../zipmetadata"
"errors" "errors"
"fmt" "fmt"
"io/ioutil" "io/ioutil"
"mime/multipart" "mime/multipart"
"net/http" "net/http"
"os" "os"
"os/exec"
"syscall"
) )
type artifactsUploadProcessor struct { type artifactsUploadProcessor struct {
...@@ -36,10 +39,19 @@ func (a *artifactsUploadProcessor) ProcessFile(formName, fileName string, writer ...@@ -36,10 +39,19 @@ func (a *artifactsUploadProcessor) ProcessFile(formName, fileName string, writer
a.metadataFile = tempFile.Name() a.metadataFile = tempFile.Name()
// Generate metadata and save to file // Generate metadata and save to file
err = generateZipMetadataFromFile(fileName, tempFile) zipMd := exec.Command("gitlab-zip-metadata", fileName)
if err == os.ErrInvalid { zipMd.Stderr = os.Stderr
return nil zipMd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
} else if err != nil { zipMd.Stdout = tempFile
if err := zipMd.Start(); err != nil {
return err
}
defer helper.CleanUpProcessGroup(zipMd)
if err := zipMd.Wait(); err != nil {
if st, ok := helper.ExitStatus(err); ok && st == zipmetadata.StatusNotZip {
return nil
}
return err return err
} }
......
...@@ -5,6 +5,7 @@ import ( ...@@ -5,6 +5,7 @@ import (
"../helper" "../helper"
"../proxy" "../proxy"
"../testhelper" "../testhelper"
"../zipmetadata"
"archive/zip" "archive/zip"
"bytes" "bytes"
"compress/gzip" "compress/gzip"
...@@ -71,7 +72,7 @@ func testArtifactsUploadServer(t *testing.T, tempPath string) *httptest.Server { ...@@ -71,7 +72,7 @@ func testArtifactsUploadServer(t *testing.T, tempPath string) *httptest.Server {
w.WriteHeader(404) w.WriteHeader(404)
return return
} }
if !bytes.HasPrefix(metadata, []byte(metadataHeaderPrefix+metadataHeader)) { if !bytes.HasPrefix(metadata, []byte(zipmetadata.HeaderPrefix+zipmetadata.Header)) {
w.WriteHeader(400) w.WriteHeader(400)
return return
} }
......
package artifacts package zipmetadata
import ( import (
"archive/zip" "archive/zip"
...@@ -19,8 +19,9 @@ type metadata struct { ...@@ -19,8 +19,9 @@ type metadata struct {
Comment string `json:"comment,omitempty"` Comment string `json:"comment,omitempty"`
} }
const metadataHeaderPrefix = "\x00\x00\x00&" // length of string below, encoded properly const HeaderPrefix = "\x00\x00\x00&" // length of string below, encoded properly
const metadataHeader = "GitLab Build Artifacts Metadata 0.0.2\n" const Header = "GitLab Build Artifacts Metadata 0.0.2\n"
const StatusNotZip = 2
func newMetadata(file *zip.File) metadata { func newMetadata(file *zip.File) metadata {
return metadata{ return metadata{
...@@ -56,7 +57,7 @@ func writeZipEntryMetadata(output io.Writer, entry *zip.File) error { ...@@ -56,7 +57,7 @@ func writeZipEntryMetadata(output io.Writer, entry *zip.File) error {
} }
func generateZipMetadata(output io.Writer, archive *zip.Reader) error { func generateZipMetadata(output io.Writer, archive *zip.Reader) error {
err := writeString(output, metadataHeader) err := writeString(output, Header)
if err != nil { if err != nil {
return err return err
} }
...@@ -77,7 +78,7 @@ func generateZipMetadata(output io.Writer, archive *zip.Reader) error { ...@@ -77,7 +78,7 @@ func generateZipMetadata(output io.Writer, archive *zip.Reader) error {
return nil return nil
} }
func generateZipMetadataFromFile(fileName string, w io.Writer) error { func GenerateZipMetadataFromFile(fileName string, w io.Writer) error {
archive, err := zip.OpenReader(fileName) archive, err := zip.OpenReader(fileName)
if err != nil { if err != nil {
// Ignore non-zip archives // Ignore non-zip archives
......
...@@ -466,8 +466,9 @@ func TestArtifactsUpload(t *testing.T) { ...@@ -466,8 +466,9 @@ func TestArtifactsUpload(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
if len(r.MultipartForm.Value) != 2 { // 1 file name, 1 file path nValues := 2 // filename + path for just the upload (no metadata because we are not POSTing a valid zip file)
t.Error("Expected to receive exactly 2 values") if len(r.MultipartForm.Value) != nValues {
t.Errorf("Expected to receive exactly %d values", nValues)
} }
if len(r.MultipartForm.File) != 0 { if len(r.MultipartForm.File) != 0 {
t.Error("Expected to not receive any files") t.Error("Expected to not receive any files")
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment