Commit a03b4c90 authored by Quentin Smith's avatar Quentin Smith

storage/db: optimize query terms matching "upload"

Since the upload ID is part of the primary key, we can avoid using the
RecordLabels table to find matching records. This takes the query
"upload:20170127.22 name:Gzip" from ~0.3s to ~0.04s on our production
database.

Change-Id: Id31624ec1b81f3a7f648c6597392718c56a10edd
Reviewed-on: https://go-review.googlesource.com/37236
Run-TryBot: Quentin Smith <quentin@golang.org>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: default avatarRuss Cox <rsc@golang.org>
parent 9614811b
...@@ -155,7 +155,7 @@ func (db *DB) prepareStatements(driverName string) error { ...@@ -155,7 +155,7 @@ func (db *DB) prepareStatements(driverName string) error {
// An Upload is a collection of files that share an upload ID. // An Upload is a collection of files that share an upload ID.
type Upload struct { type Upload struct {
// ID is the value of the "uploadid" key that should be // ID is the value of the "upload" key that should be
// associated with every record in this upload. // associated with every record in this upload.
ID string ID string
......
...@@ -250,6 +250,7 @@ func TestQuery(t *testing.T) { ...@@ -250,6 +250,7 @@ func TestQuery(t *testing.T) {
for i := 0; i < 1024; i++ { for i := 0; i < 1024; i++ {
allRecords = append(allRecords, i) allRecords = append(allRecords, i)
r := &benchfmt.Result{Labels: make(map[string]string), NameLabels: make(map[string]string), Content: "BenchmarkName 1 ns/op"} r := &benchfmt.Result{Labels: make(map[string]string), NameLabels: make(map[string]string), Content: "BenchmarkName 1 ns/op"}
r.Labels["upload"] = u.ID
for j := uint(0); j < 10; j++ { for j := uint(0); j < 10; j++ {
r.Labels[fmt.Sprintf("label%d", j)] = fmt.Sprintf("%d", i/(1<<j)) r.Labels[fmt.Sprintf("label%d", j)] = fmt.Sprintf("%d", i/(1<<j))
} }
...@@ -280,6 +281,11 @@ func TestQuery(t *testing.T) { ...@@ -280,6 +281,11 @@ func TestQuery(t *testing.T) {
{"", allRecords}, {"", allRecords},
{"missing>", []int{}}, {"missing>", []int{}},
{"label0>", allRecords}, {"label0>", allRecords},
{"upload:" + u.ID, allRecords},
{"upload:none", []int{}},
{"upload>" + u.ID, []int{}},
{"upload<" + u.ID, []int{}},
{"label0:0 upload:" + u.ID, []int{0}},
} }
for _, test := range tests { for _, test := range tests {
t.Run("query="+test.q, func(t *testing.T) { t.Run("query="+test.q, func(t *testing.T) {
......
...@@ -133,6 +133,18 @@ func (p part) merge(p2 part) (part, error) { ...@@ -133,6 +133,18 @@ func (p part) merge(p2 part) (part, error) {
// sql returns a SQL expression and a list of arguments for finding records matching p. // sql returns a SQL expression and a list of arguments for finding records matching p.
func (p part) sql() (sql string, args []interface{}, err error) { func (p part) sql() (sql string, args []interface{}, err error) {
if p.key == "upload" {
switch p.operator {
case equals:
return "SELECT UploadID, RecordID FROM Records WHERE UploadID = ?", []interface{}{p.value}, nil
case lt:
return "SELECT UploadID, RecordID FROM Records WHERE UploadID < ?", []interface{}{p.value}, nil
case gt:
return "SELECT UploadID, RecordID FROM Records WHERE UploadID > ?", []interface{}{p.value}, nil
case ltgt:
return "SELECT UploadID, RecordID FROM Records WHERE UploadID < ? AND UploadID > ?", []interface{}{p.value, p.value2}, nil
}
}
switch p.operator { switch p.operator {
case equals: case equals:
if p.value == "" { if p.value == "" {
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment