Fix some missing Close() calls and unnecessary allocations

internal/lsif_transformer/parser/docs.go

@@ -2,7 +2,9 @@ package parser
 
 import (
 	"archive/zip"
+	"bufio"
 	"encoding/json"
+	"io"
 	"strings"
 )
 
@@ -45,7 +47,19 @@ func NewDocs(config Config) (*Docs, error) {
 	}, nil
 }
 
-func (d *Docs) Read(line []byte) error {
+func (d *Docs) Parse(r io.Reader) error {
+	scanner := bufio.NewScanner(r)
+
+	for scanner.Scan() {
+		if err := d.process(scanner.Bytes()); err != nil {
+			return err
+		}
+	}
+
+	return scanner.Err()
+}
+
+func (d *Docs) process(line []byte) error {
 	l := Line{}
 	if err := json.Unmarshal(line, &l); err != nil {
 		return err

internal/lsif_transformer/parser/docs_test.go

 package parser
 
 import (
+	"bytes"
 	"fmt"
 	"testing"
 
@@ -8,34 +9,34 @@ import (
 )
 
 func createLine(id, label, uri string) []byte {
-	return []byte(fmt.Sprintf(`{"id":"%s","label":"%s","uri":"%s"}`, id, label, uri))
+	return []byte(fmt.Sprintf(`{"id":"%s","label":"%s","uri":"%s"}`+"\n", id, label, uri))
 }
 
-func TestRead(t *testing.T) {
+func TestParse(t *testing.T) {
 	d, err := NewDocs(Config{})
 	require.NoError(t, err)
 	defer d.Close()
 
-	metadataLine := []byte(`{"id":"1","label":"metaData","projectRoot":"file:///Users/nested"}`)
+	data := []byte(`{"id":"1","label":"metaData","projectRoot":"file:///Users/nested"}` + "\n")
+	data = append(data, createLine("2", "document", "file:///Users/nested/file.rb")...)
+	data = append(data, createLine("3", "document", "file:///Users/nested/folder/file.rb")...)
+	data = append(data, createLine("4", "document", "file:///Users/wrong/file.rb")...)
 
-	require.NoError(t, d.Read(metadataLine))
-	require.NoError(t, d.Read(createLine("2", "document", "file:///Users/nested/file.rb")))
-	require.NoError(t, d.Read(createLine("3", "document", "file:///Users/nested/folder/file.rb")))
-	require.NoError(t, d.Read(createLine("4", "document", "file:///Users/wrong/file.rb")))
+	require.NoError(t, d.Parse(bytes.NewReader(data)))
 
 	require.Equal(t, d.Entries[2], "file.rb")
 	require.Equal(t, d.Entries[3], "folder/file.rb")
 	require.Equal(t, d.Entries[4], "file:///Users/wrong/file.rb")
 }
 
-func TestReadContainsLine(t *testing.T) {
+func TestParseContainsLine(t *testing.T) {
 	d, err := NewDocs(Config{})
 	require.NoError(t, err)
 	defer d.Close()
 
-	line := []byte(`{"id":"5","label":"contains","outV":"1", "inVs": ["2", "3"]}`)
+	line := []byte(`{"id":"5","label":"contains","outV":"1", "inVs": ["2", "3"]}` + "\n")
 
-	require.NoError(t, d.Read(line))
+	require.NoError(t, d.Parse(bytes.NewReader(line)))
 
 	require.Equal(t, []Id{2, 3}, d.DocRanges[1])
 }

internal/lsif_transformer/parser/parser.go

@@ -2,7 +2,6 @@ package parser
 
 import (
 	"archive/zip"
-	"bufio"
 	"errors"
 	"fmt"
 	"io"
@@ -30,21 +29,41 @@ func NewParser(r io.Reader, config Config) (io.ReadCloser, error) {
 		return nil, err
 	}
 
-	zr, err := openZipReader(r, config.TempPath)
+	// ZIP files need to be seekable. Don't hold it all in RAM, use a tempfile
+	tempFile, err := ioutil.TempFile(config.TempPath, Lsif)
 	if err != nil {
 		return nil, err
 	}
-	reader := bufio.NewReader(zr)
 
-	for {
-		line, err := reader.ReadBytes('\n')
+	defer tempFile.Close()
+
+	if err := os.Remove(tempFile.Name()); err != nil {
+		return nil, err
+	}
+
+	size, err := io.Copy(tempFile, r)
 	if err != nil {
-			break
+		return nil, err
 	}
 
-		if err := docs.Read(line); err != nil {
+	zr, err := zip.NewReader(tempFile, size)
+	if err != nil {
 		return nil, err
 	}
+
+	if len(zr.File) == 0 {
+		return nil, errors.New("empty zip file")
+	}
+
+	file, err := zr.File[0].Open()
+	if err != nil {
+		return nil, err
+	}
+
+	defer file.Close()
+
+	if err := docs.Parse(file); err != nil {
+		return nil, err
 	}
 
 	pr, pw := io.Pipe()
@@ -53,7 +72,7 @@ func NewParser(r io.Reader, config Config) (io.ReadCloser, error) {
 		pr:   pr,
 	}
 
-	go parser.parse(pw)
+	go parser.transform(pw)
 
 	return parser, nil
 }
@@ -68,7 +87,7 @@ func (p *Parser) Close() error {
 	return p.Docs.Close()
 }
 
-func (p *Parser) parse(pw *io.PipeWriter) {
+func (p *Parser) transform(pw *io.PipeWriter) {
 	zw := zip.NewWriter(pw)
 
 	if err := p.Docs.SerializeEntries(zw); err != nil {
@@ -84,34 +103,3 @@ func (p *Parser) parse(pw *io.PipeWriter) {
 
 	pw.Close()
 }
-
-func openZipReader(reader io.Reader, tempDir string) (io.Reader, error) {
-	tempFile, err := ioutil.TempFile(tempDir, Lsif)
-	if err != nil {
-		return nil, err
-	}
-
-	if err := os.Remove(tempFile.Name()); err != nil {
-		return nil, err
-	}
-
-	size, err := io.Copy(tempFile, reader)
-	if err != nil {
-		return nil, err
-	}
-
-	if _, err := tempFile.Seek(0, io.SeekStart); err != nil {
-		return nil, err
-	}
-
-	zr, err := zip.NewReader(tempFile, size)
-	if err != nil {
-		return nil, err
-	}
-
-	if len(zr.File) == 0 {
-		return nil, errors.New("empty zip file")
-	}
-
-	return zr.File[0].Open()
-}