storage/app: add BenchmarkReader struct

BenchmarkReader is capable of reading standard benchmark files into
Result objects and writing those Result objects back out to a Writer.

Change-Id: I022221f53b5d3ce1de7e8e7b74d265a50ac4a0eb
Reviewed-on: https://go-review.googlesource.com/34627Reviewed-by: Russ Cox <rsc@golang.org>

storage/app/benchfmt.go

+// Copyright 2016 The Go Authors.  All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package app
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"sort"
+	"strconv"
+	"strings"
+	"unicode"
+)
+
+// BenchmarkReader reads benchmark results from an io.Reader.
+type BenchmarkReader struct {
+	s       *bufio.Scanner
+	labels  map[string]string
+	lineNum int
+}
+
+// NewBenchmarkReader creates a BenchmarkReader that reads from r.
+func NewBenchmarkReader(r io.Reader) *BenchmarkReader {
+	return &BenchmarkReader{
+		s:      bufio.NewScanner(r),
+		labels: make(map[string]string),
+	}
+}
+
+// AddLabels adds additional labels as if they had been read from the file.
+// It must be called before the first call to r.Next.
+func (r *BenchmarkReader) AddLabels(labels map[string]string) {
+	for k, v := range labels {
+		r.labels[k] = v
+	}
+}
+
+// TODO: It would probably be helpful to add a named type for
+// map[string]string with String(), Keys(), and Equal() methods.
+
+// Result represents a single line from a benchmark file.
+// All information about that line is self-contained in the Result.
+type Result struct {
+	// Labels is the set of persistent labels that apply to the result.
+	// Labels must not be modified.
+	Labels map[string]string
+	// NameLabels is the set of ephemeral labels that were parsed
+	// from the benchmark name/line.
+	// NameLabels must not be modified.
+	NameLabels map[string]string
+	// LineNum is the line number on which the result was found
+	LineNum int
+	// Content is the verbatim input line of the benchmark file, beginning with the string "Benchmark".
+	Content string
+}
+
+// A BenchmarkPrinter prints a sequence of benchmark results.
+type BenchmarkPrinter struct {
+	w      io.Writer
+	labels map[string]string
+}
+
+// NewBenchmarkPrinter constructs a BenchmarkPrinter writing to w.
+func NewBenchmarkPrinter(w io.Writer) *BenchmarkPrinter {
+	return &BenchmarkPrinter{w: w}
+}
+
+// Print writes the lines necessary to recreate r.
+func (bp *BenchmarkPrinter) Print(r *Result) error {
+	var keys []string
+	// Print removed keys first.
+	for k := range bp.labels {
+		if r.Labels[k] == "" {
+			keys = append(keys, k)
+		}
+	}
+	sort.Strings(keys)
+	for _, k := range keys {
+		if _, err := fmt.Fprintf(bp.w, "%s:\n", k); err != nil {
+			return err
+		}
+	}
+	// Then print new or changed keys.
+	keys = keys[:0]
+	for k, v := range r.Labels {
+		if v != "" && bp.labels[k] != v {
+			keys = append(keys, k)
+		}
+	}
+	sort.Strings(keys)
+	for _, k := range keys {
+		if _, err := fmt.Fprintf(bp.w, "%s: %s\n", k, r.Labels[k]); err != nil {
+			return err
+		}
+	}
+	// Finally print the actual line itself.
+	if _, err := fmt.Fprintf(bp.w, "%s\n", r.Content); err != nil {
+		return err
+	}
+	bp.labels = r.Labels
+	return nil
+}
+
+// parseNameLabels extracts extra labels from a benchmark name and sets them in labels.
+func parseNameLabels(name string, labels map[string]string) {
+	dash := strings.LastIndex(name, "-")
+	if dash >= 0 {
+		// Accept -N as an alias for /GOMAXPROCS=N
+		_, err := strconv.Atoi(name[dash+1:])
+		if err == nil {
+			labels["GOMAXPROCS"] = name[dash+1:]
+			name = name[:dash]
+		}
+	}
+	parts := strings.Split(name, "/")
+	labels["name"] = parts[0]
+	for i, sub := range parts[1:] {
+		equals := strings.Index(sub, "=")
+		var key string
+		if equals >= 0 {
+			key, sub = sub[:equals], sub[equals+1:]
+		} else {
+			key = fmt.Sprintf("sub%d", i+1)
+		}
+		labels[key] = sub
+	}
+}
+
+// newResult parses a line and returns a Result object for the line.
+func newResult(labels map[string]string, lineNum int, name, content string) *Result {
+	r := &Result{
+		Labels:     labels,
+		NameLabels: make(map[string]string),
+		LineNum:    lineNum,
+		Content:    content,
+	}
+	parseNameLabels(name, r.NameLabels)
+	return r
+}
+
+// copyLabels makes a new copy of the labels map, to protect against
+// future modifications to labels.
+func copyLabels(labels map[string]string) map[string]string {
+	new := make(map[string]string)
+	for k, v := range labels {
+		new[k] = v
+	}
+	return new
+}
+
+// TODO(quentin): How to represent and efficiently group multiple lines?
+
+// Next returns the next benchmark result from the file. If there are
+// no further results, it returns nil, io.EOF.
+func (r *BenchmarkReader) Next() (*Result, error) {
+	copied := false
+	for r.s.Scan() {
+		r.lineNum++
+		line := r.s.Text()
+		if key, value, ok := parseKeyValueLine(line); ok {
+			if !copied {
+				copied = true
+				r.labels = copyLabels(r.labels)
+			}
+			// TODO(quentin): Spec says empty value is valid, but
+			// we need a way to cancel previous labels, so we'll
+			// treat an empty value as a removal.
+			if value == "" {
+				delete(r.labels, key)
+			} else {
+				r.labels[key] = value
+			}
+			continue
+		}
+		if fullName, ok := parseBenchmarkLine(line); ok {
+			return newResult(r.labels, r.lineNum, fullName, line), nil
+		}
+	}
+	if err := r.s.Err(); err != nil {
+		return nil, err
+	}
+	return nil, io.EOF
+}
+
+// parseKeyValueLine attempts to parse line as a key: value pair. ok
+// indicates whether the line could be parsed.
+func parseKeyValueLine(line string) (key, val string, ok bool) {
+	for i, c := range line {
+		if i == 0 && !unicode.IsLower(c) {
+			return
+		}
+		if unicode.IsSpace(c) || unicode.IsUpper(c) {
+			return
+		}
+		if i > 0 && c == ':' {
+			key = line[:i]
+			val = line[i+1:]
+			break
+		}
+	}
+	if val == "" {
+		ok = true
+		return
+	}
+	for len(val) > 0 && (val[0] == ' ' || val[0] == '\t') {
+		val = val[1:]
+		ok = true
+	}
+	return
+}
+
+// parseBenchmarkLine attempts to parse line as a benchmark result. If
+// successful, fullName is the name of the benchmark with the
+// "Benchmark" prefix stripped, and ok is true.
+func parseBenchmarkLine(line string) (fullName string, ok bool) {
+	space := strings.IndexFunc(line, unicode.IsSpace)
+	if space < 0 {
+		return
+	}
+	name := line[:space]
+	if !strings.HasPrefix(name, "Benchmark") {
+		return
+	}
+	return name[len("Benchmark"):], true
+}

storage/app/benchfmt_test.go

+// Copyright 2016 The Go Authors.  All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package app
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"reflect"
+	"strings"
+	"testing"
+)
+
+func readAllResults(t *testing.T, r *BenchmarkReader) []*Result {
+	var out []*Result
+	for {
+		result, err := r.Next()
+		switch err {
+		case io.EOF:
+			return out
+		case nil:
+			out = append(out, result)
+		default:
+			t.Fatal(err)
+		}
+	}
+}
+
+func TestBenchmarkReader(t *testing.T) {
+	type kv map[string]string
+	tests := []struct {
+		name, input string
+		want        []*Result
+	}{
+		{
+			"basic",
+			`key: value
+BenchmarkOne 1 ns/sec
+`,
+			[]*Result{{
+				kv{"key": "value"},
+				kv{"name": "One"},
+				2,
+				"BenchmarkOne 1 ns/sec",
+			}},
+		},
+		{
+			"two results with indexed and named subnames",
+			`key: value
+BenchmarkOne/foo/bar=1-2 1 ns/sec
+BenchmarkTwo 2 ns/sec
+`,
+			[]*Result{
+				{
+					kv{"key": "value"},
+					kv{"name": "One", "sub1": "foo", "bar": "1", "GOMAXPROCS": "2"},
+					2,
+					"BenchmarkOne/foo/bar=1-2 1 ns/sec",
+				},
+				{
+					kv{"key": "value"},
+					kv{"name": "Two"},
+					3,
+					"BenchmarkTwo 2 ns/sec",
+				},
+			},
+		},
+		{
+			"remove existing label",
+			`key: value
+key:
+BenchmarkOne 1 ns/sec
+`,
+			[]*Result{
+				{
+					kv{},
+					kv{"name": "One"},
+					3,
+					"BenchmarkOne 1 ns/sec",
+				},
+			},
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			r := NewBenchmarkReader(strings.NewReader(test.input))
+			have := readAllResults(t, r)
+			want := test.want
+			diff := ""
+			mismatch := false
+			for i := 0; i < len(have) || i < len(want); i++ {
+				if i < len(have) && i < len(want) && reflect.DeepEqual(have[i], want[i]) {
+					diff += fmt.Sprintf(" %+v\n", have[i])
+					continue
+				}
+				mismatch = true
+				if i < len(have) {
+					diff += fmt.Sprintf("-%+v\n", have[i])
+				}
+				if i < len(want) {
+					diff += fmt.Sprintf("+%+v\n", want[i])
+				}
+			}
+			if mismatch {
+				t.Errorf("wrong results: (- have/+ want)\n%s", diff)
+			}
+		})
+	}
+}
+
+func TestBenchmarkPrinter(t *testing.T) {
+	tests := []struct {
+		name, input, want string
+	}{
+		{
+			"basic",
+			`key: value
+BenchmarkOne 1 ns/sec
+`,
+			`key: value
+BenchmarkOne 1 ns/sec
+`,
+		},
+		{
+			"missing newline",
+			`key: value
+BenchmarkOne 1 ns/sec`,
+			`key: value
+BenchmarkOne 1 ns/sec
+`,
+		},
+		{
+			"duplicate and removed fields",
+			`one: 1
+two: 2
+BenchmarkOne 1 ns/sec
+one: 1
+two: 3
+BenchmarkOne 1 ns/sec
+two:
+BenchmarkOne 1 ns/sec
+`,
+			`one: 1
+two: 2
+BenchmarkOne 1 ns/sec
+two: 3
+BenchmarkOne 1 ns/sec
+two:
+BenchmarkOne 1 ns/sec
+`,
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			r := NewBenchmarkReader(strings.NewReader(test.input))
+			results := readAllResults(t, r)
+			var have bytes.Buffer
+			bp := NewBenchmarkPrinter(&have)
+			for _, result := range results {
+				if err := bp.Print(result); err != nil {
+					t.Errorf("Print returned %v", err)
+				}
+			}
+			if diff := diff(have.String(), test.want); diff != "" {
+				t.Errorf("wrong output: (- got/+ want)\n%s", diff)
+			}
+		})
+	}
+}
+
+// diff returns the output of unified diff on s1 and s2. If the result
+// is non-empty, the strings differ or the diff command failed.
+func diff(s1, s2 string) string {
+	f1, err := ioutil.TempFile("", "benchfmt_test")
+	if err != nil {
+		return err.Error()
+	}
+	defer os.Remove(f1.Name())
+	defer f1.Close()
+
+	f2, err := ioutil.TempFile("", "benchfmt_test")
+	if err != nil {
+		return err.Error()
+	}
+	defer os.Remove(f2.Name())
+	defer f2.Close()
+
+	f1.Write([]byte(s1))
+	f2.Write([]byte(s2))
+
+	data, err := exec.Command("diff", "-u", f1.Name(), f2.Name()).CombinedOutput()
+	if len(data) > 0 {
+		// diff exits with a non-zero status when the files don't match.
+		// Ignore that failure as long as we get output.
+		err = nil
+	}
+	if err != nil {
+		data = append(data, []byte(err.Error())...)
+	}
+	return string(data)
+
+}

storage/app/upload.go

@@ -10,6 +10,7 @@ import (
 	"io"
 	"mime/multipart"
 	"net/http"
+	"sort"
 
 	"golang.org/x/net/context"
 )
@@ -102,7 +103,18 @@ func (a *App) processUpload(ctx context.Context, mr *multipart.Reader) (*uploadS
 			return nil, err
 		}
 
-		// TODO(quentin): Write metadata at top of file
+		var keys []string
+		for k := range meta {
+			keys = append(keys, k)
+		}
+		sort.Strings(keys)
+		for _, k := range keys {
+			if _, err := fmt.Fprintf(fw, "%s: %s\n", k, meta[k]); err != nil {
+				fw.CloseWithError(err)
+				return nil, err
+			}
+		}
+
 		if _, err := io.Copy(fw, p); err != nil {
 			fw.CloseWithError(err)
 			return nil, err