Merge branch 'id-increase-lsif-scanner-buffer' into 'master'

Increase LSIF scanner buffer

See merge request gitlab-org/gitlab-workhorse!609

changelogs/unreleased/id-increase-lsif-scanner-buffer.yml

+---
+title: Increase LSIF scanner buffer
+merge_request: 609
+author:
+type: fixed

internal/lsif_transformer/parser/docs.go

@@ -8,6 +8,8 @@ import (
 	"strings"
 )
 
+const maxScanTokenSize = 1024 * 1024
+
 type Line struct {
 	Type string `json:"label"`
 }
@@ -49,6 +51,8 @@ func NewDocs(config Config) (*Docs, error) {
 
 func (d *Docs) Parse(r io.Reader) error {
 	scanner := bufio.NewScanner(r)
+	buf := make([]byte, 0, bufio.MaxScanTokenSize)
+	scanner.Buffer(buf, maxScanTokenSize)
 
 	for scanner.Scan() {
 		if err := d.process(scanner.Bytes()); err != nil {

internal/lsif_transformer/parser/docs_test.go

@@ -3,6 +3,7 @@ package parser
 import (
 	"bytes"
 	"fmt"
+	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -40,3 +41,13 @@ func TestParseContainsLine(t *testing.T) {
 
 	require.Equal(t, []Id{2, 3}, d.DocRanges[1])
 }
+
+func TestParsingVeryLongLine(t *testing.T) {
+	d, err := NewDocs(Config{})
+	require.NoError(t, err)
+	defer d.Close()
+
+	line := []byte(`{"id": "` + strings.Repeat("a", 64*1024) + `"}`)
+
+	require.NoError(t, d.Parse(bytes.NewReader(line)))
+}