Generate zip metadata in a subprocess

Makefile

@@ -2,11 +2,14 @@ PREFIX=/usr/local
 VERSION=$(shell git describe)-$(shell date -u +%Y%m%d.%H%M%S)
 GOBUILD=go build -ldflags "-X main.Version=${VERSION}"
 
-all: gitlab-zip-cat gitlab-workhorse
+all: gitlab-zip-cat gitlab-zip-metadata gitlab-workhorse
 
 gitlab-zip-cat:	$(shell find cmd/gitlab-zip-cat/ -name '*.go')
 	${GOBUILD} -o $@ ./cmd/$@
 	
+gitlab-zip-metadata:	$(shell find cmd/gitlab-zip-metadata/ -name '*.go')
+	${GOBUILD} -o $@ ./cmd/$@
+
 gitlab-workhorse: $(shell find . -name '*.go')
 	${GOBUILD} -o $@
 
@@ -14,7 +17,7 @@ install: gitlab-workhorse gitlab-zip-cat
 	install gitlab-workhorse gitlab-zip-cat ${PREFIX}/bin/
 
 .PHONY: test
-test: testdata/data/group/test.git clean-workhorse gitlab-workhorse gitlab-zip-cat
+test: testdata/data/group/test.git clean-workhorse all
 	go fmt ./... | awk '{ print } END { if (NR > 0) { print "Please run go fmt"; exit 1 } }'
 	support/path-add-current go test ./...
 	@echo SUCCESS
@@ -36,4 +39,4 @@ clean:	clean-workhorse
 
 .PHONY:	clean-workhorse
 clean-workhorse:
-	rm -f gitlab-workhorse gitlab-zip-cat
+	rm -f gitlab-workhorse gitlab-zip-cat gitlab-zip-metadata

cmd/gitlab-zip-cat/main.go

@@ -25,6 +25,11 @@ func main() {
 		os.Exit(0)
 	}
 
+	if len(os.Args) != 3 {
+		fmt.Fprintf(os.Stderr, "Usage: %s FILE.ZIP ENTRY", progName)
+		os.Exit(1)
+	}
+
 	archiveFileName := os.Args[1]
 	fileName := os.Args[2]
 	archive, err := zip.OpenReader(archiveFileName)

cmd/gitlab-zip-metadata/main.go

+package main
+
+import (
+	"../../internal/zipmetadata"
+	"flag"
+	"fmt"
+	"os"
+)
+
+const progName = "gitlab-zip-metadata"
+
+var Version = "unknown"
+
+var printVersion = flag.Bool("version", false, "Print version and exit")
+
+func main() {
+	flag.Parse()
+
+	version := fmt.Sprintf("%s %s", progName, Version)
+	if *printVersion {
+		fmt.Println(version)
+		os.Exit(0)
+	}
+
+	if len(os.Args) != 2 {
+		fmt.Fprintf(os.Stderr, "Usage: %s FILE.ZIP", progName)
+		os.Exit(1)
+	}
+	if err := zipmetadata.GenerateZipMetadataFromFile(os.Args[1], os.Stdout); err != nil {
+		fmt.Fprintf(os.Stderr, "%s: %v\n", progName, err)
+		if err == os.ErrInvalid {
+			os.Exit(zipmetadata.StatusNotZip)
+		}
+		os.Exit(1)
+	}
+}

internal/artifacts/artifact_download.go

@@ -39,6 +39,7 @@ func detectFileContentType(fileName string) string {
 
 func unpackFileFromZip(archiveFileName, fileName string, headers http.Header, output io.Writer) error {
 	catFile := exec.Command("gitlab-zip-cat", archiveFileName, fileName)
+	catFile.Stderr = os.Stderr
 	catFile.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
 	stdout, err := catFile.StdoutPipe()
 	if err != nil {

internal/artifacts/artifacts_upload.go

@@ -4,12 +4,15 @@ import (
 	"../api"
 	"../helper"
 	"../upload"
+	"../zipmetadata"
 	"errors"
 	"fmt"
 	"io/ioutil"
 	"mime/multipart"
 	"net/http"
 	"os"
+	"os/exec"
+	"syscall"
 )
 
 type artifactsUploadProcessor struct {
@@ -36,10 +39,19 @@ func (a *artifactsUploadProcessor) ProcessFile(formName, fileName string, writer
 	a.metadataFile = tempFile.Name()
 
 	// Generate metadata and save to file
-	err = generateZipMetadataFromFile(fileName, tempFile)
-	if err == os.ErrInvalid {
-		return nil
-	} else if err != nil {
+	zipMd := exec.Command("gitlab-zip-metadata", fileName)
+	zipMd.Stderr = os.Stderr
+	zipMd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
+	zipMd.Stdout = tempFile
+
+	if err := zipMd.Start(); err != nil {
+		return err
+	}
+	defer helper.CleanUpProcessGroup(zipMd)
+	if err := zipMd.Wait(); err != nil {
+		if st, ok := helper.ExitStatus(err); ok && st == zipmetadata.StatusNotZip {
+			return nil
+		}
 		return err
 	}
 

internal/artifacts/artifacts_upload_test.go

@@ -5,6 +5,7 @@ import (
 	"../helper"
 	"../proxy"
 	"../testhelper"
+	"../zipmetadata"
 	"archive/zip"
 	"bytes"
 	"compress/gzip"
@@ -71,7 +72,7 @@ func testArtifactsUploadServer(t *testing.T, tempPath string) *httptest.Server {
 			w.WriteHeader(404)
 			return
 		}
-		if !bytes.HasPrefix(metadata, []byte(metadataHeaderPrefix+metadataHeader)) {
+		if !bytes.HasPrefix(metadata, []byte(zipmetadata.HeaderPrefix+zipmetadata.Header)) {
 			w.WriteHeader(400)
 			return
 		}

internal/artifacts/metadata.go → internal/zipmetadata/metadata.go

-package artifacts
+package zipmetadata
 
 import (
 	"archive/zip"
@@ -19,8 +19,9 @@ type metadata struct {
 	Comment  string `json:"comment,omitempty"`
 }
 
-const metadataHeaderPrefix = "\x00\x00\x00&" // length of string below, encoded properly
-const metadataHeader = "GitLab Build Artifacts Metadata 0.0.2\n"
+const HeaderPrefix = "\x00\x00\x00&" // length of string below, encoded properly
+const Header = "GitLab Build Artifacts Metadata 0.0.2\n"
+const StatusNotZip = 2
 
 func newMetadata(file *zip.File) metadata {
 	return metadata{
@@ -56,7 +57,7 @@ func writeZipEntryMetadata(output io.Writer, entry *zip.File) error {
 }
 
 func generateZipMetadata(output io.Writer, archive *zip.Reader) error {
-	err := writeString(output, metadataHeader)
+	err := writeString(output, Header)
 	if err != nil {
 		return err
 	}
@@ -77,7 +78,7 @@ func generateZipMetadata(output io.Writer, archive *zip.Reader) error {
 	return nil
 }
 
-func generateZipMetadataFromFile(fileName string, w io.Writer) error {
+func GenerateZipMetadataFromFile(fileName string, w io.Writer) error {
 	archive, err := zip.OpenReader(fileName)
 	if err != nil {
 		// Ignore non-zip archives

main_test.go

@@ -466,8 +466,9 @@ func TestArtifactsUpload(t *testing.T) {
 		if err != nil {
 			t.Fatal(err)
 		}
-		if len(r.MultipartForm.Value) != 2 { // 1 file name, 1 file path
-			t.Error("Expected to receive exactly 2 values")
+		nValues := 2 // filename + path for just the upload (no metadata because we are not POSTing a valid zip file)
+		if len(r.MultipartForm.Value) != nValues {
+			t.Errorf("Expected to receive exactly %d values", nValues)
 		}
 		if len(r.MultipartForm.File) != 0 {
 			t.Error("Expected to not receive any files")