Skip to content

G
go.perf
Commit a7a7b5b2 authored by Russ Cox's avatar Russ Cox
Browse files
Options

benchstat: split table generation into table.go, text.go, html.go 

Change-Id: I33d4b6006f4df6b253133c4467488fd6da68a34f
Reviewed-on: https://go-review.googlesource.com/35937Reviewed-by: default avatarQuentin Smith <quentin@golang.org>
parent b4c600e9
Hide whitespace changes
Inline Side-by-side
Showing with 323 additions and 225 deletions
cmd/benchstat/data.go
View file @ a7a7b5b2
......@@ -59,7 +59,7 @@ func (m *Metrics) FormatMean(scaler Scaler) string {
// FormatDiff computes and formats the percent variation of max and min compared to mean.
// If b.Mean or b.Max is zero, FormatDiff returns an empty string.
func (m *Metrics) FormatDiff() string {
if m.Mean == 0 || m.Max == 0 {
if m == nil || m.Mean == 0 || m.Max == 0 {
return ""
}
diff := 1 - m.Min/m.Mean
......@@ -71,6 +71,9 @@ func (m *Metrics) FormatDiff() string {
// Format returns a textual formatting of "Mean ±Diff" using scaler.
func (m *Metrics) Format(scaler Scaler) string {
if m == nil {
return ""
}
mean := m.FormatMean(scaler)
diff := m.FormatDiff()
if diff == "" {
......
cmd/benchstat/html.go 0 → 100644
View file @ a7a7b5b2
// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package main
import (
"bytes"
"fmt"
"html"
)
// FormatHTML appends an HTML formatting of the tables to buf.
func FormatHTML(buf *bytes.Buffer, tables []*Table) {
var textTables [][]*textRow
for _, t := range tables {
textTables = append(textTables, toText(t))
}
for i, table := range textTables {
if i > 0 {
fmt.Fprintf(buf, "\n")
}
fmt.Fprintf(buf, "<style>.benchstat tbody td:nth-child(1n+2) { text-align: right; padding: 0em 1em; }</style>\n")
fmt.Fprintf(buf, "<table class='benchstat'>\n")
printRow := func(row *textRow, tag string) {
fmt.Fprintf(buf, "<tr>")
for _, cell := range row.cols {
fmt.Fprintf(buf, "<%s>%s</%s>", tag, html.EscapeString(cell), tag)
}
fmt.Fprintf(buf, "\n")
}
printRow(table[0], "th")
for _, row := range table[1:] {
printRow(row, "td")
}
fmt.Fprintf(buf, "</table>\n")
}
}
cmd/benchstat/main.go
View file @ a7a7b5b2
......@@ -94,14 +94,10 @@ import (
"bytes"
"flag"
"fmt"
"html"
"io/ioutil"
"log"
"os"
"strings"
"unicode/utf8"
"golang.org/x/perf/internal/stats"
)
func usage() {
......@@ -128,24 +124,6 @@ var deltaTestNames = map[string]DeltaTest{
"ttest": TTest,
}
type row struct {
cols []string
}
func newRow(cols ...string) *row {
return &row{cols: cols}
}
func (r *row) add(col string) {
r.cols = append(r.cols, col)
}
func (r *row) trim() {
for len(r.cols) > 0 && r.cols[len(r.cols)-1] == "" {
r.cols = r.cols[:len(r.cols)-1]
}
}
func main() {
log.SetPrefix("benchstat: ")
log.SetFlags(0)
......@@ -156,7 +134,6 @@ func main() {
flag.Usage()
}
// Read in benchmark data.
c := new(Collection)
for _, file := range flag.Args() {
data, err := ioutil.ReadFile(file)
......@@ -165,210 +142,14 @@ func main() {
}
c.AddConfig(file, data)
}
for _, m := range c.Metrics {
m.computeStats()
}
var tables [][]*row
switch len(c.Configs) {
case 2:
before, after := c.Configs[0], c.Configs[1]
key := Key{}
for _, key.Unit = range c.Units {
var table []*row
metric := metricOf(key.Unit)
for _, key.Benchmark = range c.Benchmarks {
key.Config = before
old := c.Metrics[key]
key.Config = after
new := c.Metrics[key]
if old == nil || new == nil {
continue
}
if len(table) == 0 {
table = append(table, newRow("name", "old "+metric, "new "+metric, "delta"))
}
pval, testerr := deltaTest(old, new)
scaler := NewScaler(old.Mean, old.Unit)
row := newRow(key.Benchmark, old.Format(scaler), new.Format(scaler), "~ ")
if testerr != nil {
row.add(fmt.Sprintf("(%s)", testerr))
} else if pval < *flagAlpha {
row.cols[3] = fmt.Sprintf("%+.2f%%", ((new.Mean/old.Mean)-1.0)*100.0)
}
if len(row.cols) == 4 && pval != -1 {
row.add(fmt.Sprintf("(p=%0.3f n=%d+%d)", pval, len(old.RValues), len(new.RValues)))
}
table = append(table, row)
}
if len(table) > 0 {
table = addGeomean(table, c, key.Unit, true)
tables = append(tables, table)
}
}
default:
key := Key{}
for _, key.Unit = range c.Units {
var table []*row
metric := metricOf(key.Unit)
if len(c.Configs) > 1 {
hdr := newRow("name \\ " + metric)
for _, config := range c.Configs {
hdr.add(config)
}
table = append(table, hdr)
} else {
table = append(table, newRow("name", metric))
}
for _, key.Benchmark = range c.Benchmarks {
row := newRow(key.Benchmark)
var scaler Scaler
for _, key.Config = range c.Configs {
m := c.Metrics[key]
if m == nil {
row.add("")
continue
}
if scaler == nil {
scaler = NewScaler(m.Mean, m.Unit)
}
row.add(m.Format(scaler))
}
row.trim()
if len(row.cols) > 1 {
table = append(table, row)
}
}
table = addGeomean(table, c, key.Unit, false)
tables = append(tables, table)
}
}
numColumn := 0
for _, table := range tables {
for _, row := range table {
if numColumn < len(row.cols) {
numColumn = len(row.cols)
}
}
}
max := make([]int, numColumn)
for _, table := range tables {
for _, row := range table {
for i, s := range row.cols {
n := utf8.RuneCountInString(s)
if max[i] < n {
max[i] = n
}
}
}
}
tables := c.Tables(deltaTest)
var buf bytes.Buffer
for i, table := range tables {
if i > 0 {
fmt.Fprintf(&buf, "\n")
}
if *flagHTML {
fmt.Fprintf(&buf, "<style>.benchstat tbody td:nth-child(1n+2) { text-align: right; padding: 0em 1em; }</style>\n")
fmt.Fprintf(&buf, "<table class='benchstat'>\n")
printRow := func(row *row, tag string) {
fmt.Fprintf(&buf, "<tr>")
for _, cell := range row.cols {
fmt.Fprintf(&buf, "<%s>%s</%s>", tag, html.EscapeString(cell), tag)
}
fmt.Fprintf(&buf, "\n")
}
printRow(table[0], "th")
for _, row := range table[1:] {
printRow(row, "td")
}
fmt.Fprintf(&buf, "</table>\n")
continue
}
// headings
row := table[0]
for i, s := range row.cols {
switch i {
case 0:
fmt.Fprintf(&buf, "%-*s", max[i], s)
default:
fmt.Fprintf(&buf, " %-*s", max[i], s)
case len(row.cols) - 1:
fmt.Fprintf(&buf, " %s\n", s)
}
}
// data
for _, row := range table[1:] {
for i, s := range row.cols {
switch i {
case 0:
fmt.Fprintf(&buf, "%-*s", max[i], s)
default:
if i == len(row.cols)-1 && len(s) > 0 && s[0] == '(' {
// Left-align p value.
fmt.Fprintf(&buf, " %s", s)
break
}
fmt.Fprintf(&buf, " %*s", max[i], s)
}
}
fmt.Fprintf(&buf, "\n")
}
if *flagHTML {
FormatHTML(&buf, tables)
} else {
FormatText(&buf, tables)
}
os.Stdout.Write(buf.Bytes())
}
func addGeomean(table []*row, c *Collection, unit string, delta bool) []*row {
if !*flagGeomean {
return table
}
row := newRow("[Geo mean]")
key := Key{Unit: unit}
geomeans := []float64{}
for _, key.Config = range c.Configs {
var means []float64
for _, key.Benchmark = range c.Benchmarks {
m := c.Metrics[key]
if m != nil {
means = append(means, m.Mean)
}
}
if len(means) == 0 {
row.add("")
delta = false
} else {
geomean := stats.GeoMean(means)
geomeans = append(geomeans, geomean)
row.add(NewScaler(geomean, unit)(geomean) + " ")
}
}
if delta {
row.add(fmt.Sprintf("%+.2f%%", ((geomeans[1]/geomeans[0])-1.0)*100.0))
}
return append(table, row)
}
func metricOf(unit string) string {
switch unit {
case "ns/op":
return "time/op"
case "B/op":
return "alloc/op"
case "MB/s":
return "speed"
default:
return unit
}
}
cmd/benchstat/table.go 0 → 100644
View file @ a7a7b5b2
// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package main
import (
"fmt"
"golang.org/x/perf/internal/stats"
)
// A Table is a table for display in the benchstat output.
type Table struct {
Metric string
Configs []string
Rows []*Row
}
// A Row is a table row for display in the benchstat output.
type Row struct {
Benchmark string // benchmark name
Scaler Scaler // formatter for stats means
Metrics []*Metrics // columns of statistics (nil slice entry means no data)
Delta string // formatted percent change
Note string // additional information
Same bool // likely no change
}
// Tables returns tables comparing the benchmarks in the collection.
func (c *Collection) Tables(deltaTest DeltaTest) []*Table {
// Update statistics.
for _, m := range c.Metrics {
m.computeStats()
}
var tables []*Table
key := Key{}
for _, key.Unit = range c.Units {
table := new(Table)
table.Configs = c.Configs
table.Metric = metricOf(key.Unit)
for _, key.Benchmark = range c.Benchmarks {
row := &Row{Benchmark: key.Benchmark}
for _, key.Config = range c.Configs {
m := c.Metrics[key]
row.Metrics = append(row.Metrics, m)
if m == nil {
continue
}
if row.Scaler == nil {
row.Scaler = NewScaler(m.Mean, m.Unit)
}
}
// If there are only two configs being compared, add stats.
// If one is missing, omit line entirely.
// TODO: Control this better.
if len(c.Configs) == 2 {
k0 := key
k0.Config = c.Configs[0]
k1 := key
k1.Config = c.Configs[1]
old := c.Metrics[k0]
new := c.Metrics[k1]
if old == nil || new == nil {
continue
}
pval, testerr := deltaTest(old, new)
row.Delta = "~"
if testerr == stats.ErrZeroVariance {
row.Note = "(zero variance)"
} else if testerr == stats.ErrSampleSize {
row.Note = "(too few samples)"
} else if testerr == stats.ErrSamplesEqual {
row.Note = "(all equal)"
} else if testerr != nil {
row.Note = fmt.Sprintf("(%s)", testerr)
} else if pval < *flagAlpha {
row.Delta = fmt.Sprintf("%+.2f%%", ((new.Mean/old.Mean)-1.0)*100.0)
}
if row.Note == "" && pval != -1 {
row.Note = fmt.Sprintf("(p=%0.3f n=%d+%d)", pval, len(old.RValues), len(new.RValues))
}
}
table.Rows = append(table.Rows, row)
}
if len(table.Rows) > 0 {
if *flagGeomean {
addGeomean(c, table, key.Unit, len(c.Configs) == 2)
}
tables = append(tables, table)
}
}
return tables
}
// metricOf returns the name of the metric with the given unit.
func metricOf(unit string) string {
switch unit {
case "ns/op":
return "time/op"
case "B/op":
return "alloc/op"
case "MB/s":
return "speed"
default:
return unit
}
}
// addGeomean adds a "geomean" row to the table,
// showing the geometric mean of all the benchmarks.
func addGeomean(c *Collection, t *Table, unit string, delta bool) {
row := &Row{Benchmark: "[Geo mean]"}
key := Key{Unit: unit}
geomeans := []float64{}
for _, key.Config = range c.Configs {
var means []float64
for _, key.Benchmark = range c.Benchmarks {
m := c.Metrics[key]
if m != nil {
means = append(means, m.Mean)
}
}
if len(means) == 0 {
row.Metrics = append(row.Metrics, nil)
delta = false
} else {
geomean := stats.GeoMean(means)
geomeans = append(geomeans, geomean)
if row.Scaler == nil {
row.Scaler = NewScaler(geomean, unit)
}
row.Metrics = append(row.Metrics, &Metrics{
Unit: unit,
Mean: geomean,
})
}
}
if delta {
row.Delta = fmt.Sprintf("%+.2f%%", ((geomeans[1]/geomeans[0])-1.0)*100.0)
}
t.Rows = append(t.Rows, row)
}
cmd/benchstat/text.go 0 → 100644
View file @ a7a7b5b2
// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package main
import (
"bytes"
"fmt"
"unicode/utf8"
)
// FormatText appends a fixed-width text formatting of the tables to buf.
func FormatText(buf *bytes.Buffer, tables []*Table) {
var textTables [][]*textRow
for _, t := range tables {
textTables = append(textTables, toText(t))
}
var max []int
for _, table := range textTables {
for _, row := range table {
for len(max) < len(row.cols) {
max = append(max, 0)
}
for i, s := range row.cols {
n := utf8.RuneCountInString(s)
if max[i] < n {
max[i] = n
}
}
}
}
for i, table := range textTables {
if i > 0 {
fmt.Fprintf(buf, "\n")
}
// headings
row := table[0]
for i, s := range row.cols {
switch i {
case 0:
fmt.Fprintf(buf, "%-*s", max[i], s)
default:
fmt.Fprintf(buf, " %-*s", max[i], s)
case len(row.cols) - 1:
fmt.Fprintf(buf, " %s\n", s)
}
}
// data
for _, row := range table[1:] {
for i, s := range row.cols {
switch i {
case 0:
fmt.Fprintf(buf, "%-*s", max[i], s)
default:
if i == len(row.cols)-1 && len(s) > 0 && s[0] == '(' {
// Left-align p value.
fmt.Fprintf(buf, " %s", s)
break
}
fmt.Fprintf(buf, " %*s", max[i], s)
}
}
fmt.Fprintf(buf, "\n")
}
}
}
// A textRow is a row of printed text columns.
type textRow struct {
cols []string
}
func newTextRow(cols ...string) *textRow {
return &textRow{cols: cols}
}
func (r *textRow) add(col string) {
r.cols = append(r.cols, col)
}
func (r *textRow) trim() {
for len(r.cols) > 0 && r.cols[len(r.cols)-1] == "" {
r.cols = r.cols[:len(r.cols)-1]
}
}
// toText converts the Table to a textual grid of cells,
// which can then be printed in fixed-width output.
func toText(t *Table) []*textRow {
var textRows []*textRow
switch len(t.Configs) {
case 1:
textRows = append(textRows, newTextRow("name", t.Metric))
case 2:
textRows = append(textRows, newTextRow("name", "old "+t.Metric, "new "+t.Metric, "delta"))
default:
row := newTextRow("name \\ " + t.Metric)
row.cols = append(row.cols, t.Configs...)
textRows = append(textRows, row)
}
for _, row := range t.Rows {
text := newTextRow(row.Benchmark)
for _, m := range row.Metrics {
text.cols = append(text.cols, m.Format(row.Scaler))
}
if len(t.Configs) == 2 {
delta := row.Delta
if delta == "~" {
delta = "~ "
}
text.cols = append(text.cols, delta)
text.cols = append(text.cols, row.Note)
}
textRows = append(textRows, text)
}
for _, r := range textRows {
r.trim()
}
return textRows
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7