Merge branch 'id-lsif-processing' into 'master'

Process LSIF document before sending it to GitLab

See merge request gitlab-org/gitlab-workhorse!492

changelogs/unreleased/id-lsif-processing.yml

+---
+title: Process LSIF document before sending it to GitLab
+merge_request: 492
+author:
+type: added

go.mod

@@ -5,6 +5,7 @@ go 1.12
 require (
 	github.com/BurntSushi/toml v0.3.1
 	github.com/FZambia/sentinel v1.0.0
+	github.com/alecthomas/chroma v0.7.3
 	github.com/dgrijalva/jwt-go v3.2.0+incompatible
 	github.com/getsentry/raven-go v0.1.2
 	github.com/golang/gddo v0.0.0-20190419222130-af0f2af80721
@@ -19,13 +20,13 @@ require (
 	github.com/rafaeljusto/redigomock v0.0.0-20190202135759-257e089e14a1
 	github.com/sebest/xff v0.0.0-20160910043805-6c115e0ffa35
 	github.com/sirupsen/logrus v1.3.0
-	github.com/stretchr/testify v1.4.0
+	github.com/stretchr/testify v1.5.1
 	gitlab.com/gitlab-org/gitaly v1.74.0
 	gitlab.com/gitlab-org/labkit v0.0.0-20200327153541-fac94cb428e6
 	golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f
 	golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa
-	golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 // indirect
 	golang.org/x/tools v0.0.0-20200117161641-43d50277825c
 	google.golang.org/grpc v1.24.0
+	gopkg.in/yaml.v2 v2.2.8 // indirect
 	honnef.co/go/tools v0.0.1-2019.2.3
 )

go.sum

@@ -26,6 +26,15 @@ github.com/Joker/hpp v0.0.0-20180418125244-6893e659854a/go.mod h1:MzD2WMdSxvbHw5
 github.com/Joker/jade v1.0.0/go.mod h1:efZIdO0py/LtcJRSa/j2WEklMSAw84WV0zZVMxNToB8=
 github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0=
 github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY=
+github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38 h1:smF2tmSOzy2Mm+0dGI2AIUHY+w0BUc+4tn40djz7+6U=
+github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38/go.mod h1:r7bzyVFMNntcxPZXK3/+KdruV1H5KSlyVY0gc+NgInI=
+github.com/alecthomas/chroma v0.7.3 h1:NfdAERMy+esYQs8OXk0I868/qDxxCEo7FMz1WIqMAeI=
+github.com/alecthomas/chroma v0.7.3/go.mod h1:sko8vR34/90zvl5QdcUdvzL3J8NKjAUx9va9jPuFNoM=
+github.com/alecthomas/colour v0.0.0-20160524082231-60882d9e2721 h1:JHZL0hZKJ1VENNfmXvHbgYlbUOvpzYzvy2aZU5gXVeo=
+github.com/alecthomas/colour v0.0.0-20160524082231-60882d9e2721/go.mod h1:QO9JBoKquHd+jz9nshCh40fOfO+JzsoXy8qTHF68zU0=
+github.com/alecthomas/kong v0.2.4/go.mod h1:kQOmtJgV+Lb4aj+I2LEn40cbtawdWJ9Y8QLq+lElKxE=
+github.com/alecthomas/repr v0.0.0-20180818092828-117648cd9897 h1:p9Sln00KOTlrYkxI1zYWl1QLnEqAqEARBEYa8FQnQcY=
+github.com/alecthomas/repr v0.0.0-20180818092828-117648cd9897/go.mod h1:xTS7Pm1pD1mvyM075QCDSRqH6qRLXylzS24ZTpRiSzQ=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/aymerick/raymond v2.0.2+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g=
@@ -44,11 +53,15 @@ github.com/cloudflare/tableflip v0.0.0-20190329062924-8392f1641731/go.mod h1:erh
 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w=
 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
 github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM=
+github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 h1:y5HC9v93H5EPKqaS1UYVg1uYah5Xf51mBfIoWehClUQ=
+github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964/go.mod h1:Xd9hchkHSWYkEqJwUGisez3G1QY8Ryz0sdWrLPMGjLk=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
+github.com/dlclark/regexp2 v1.2.0 h1:8sAhBGEM0dRWogWqWyQeIJnxjWO6oIjl8FKqREDsGfk=
+github.com/dlclark/regexp2 v1.2.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=
 github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM=
 github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
 github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4/go.mod h1:T9YF2M40nIgbVgp3rreNmTged+9HrbNTIQf1PsaIiTA=
@@ -167,9 +180,12 @@ github.com/libgit2/git2go v0.0.0-20190104134018-ecaeb7a21d47/go.mod h1:4bKN42efk
 github.com/lightstep/lightstep-tracer-go v0.15.6 h1:D0GGa7afJ7GcQvu5as6ssLEEKYXvRgKI5d5cevtz8r4=
 github.com/lightstep/lightstep-tracer-go v0.15.6/go.mod h1:6AMpwZpsyCFwSovxzM78e+AsYxE8sGwiM6C3TytaWeI=
 github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
+github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
 github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
+github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
+github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw=
 github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
@@ -201,6 +217,8 @@ github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
@@ -222,6 +240,7 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR
 github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
 github.com/sebest/xff v0.0.0-20160910043805-6c115e0ffa35 h1:eajwn6K3weW5cd1ZXLu2sJ4pvwlBiCWY4uDejOr73gM=
 github.com/sebest/xff v0.0.0-20160910043805-6c115e0ffa35/go.mod h1:wozgYq9WEBQBaIJe4YZ0qTSFAMxmcwBhQH0fO0R34Z0=
+github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
@@ -238,6 +257,8 @@ github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
 github.com/tinylib/msgp v1.1.0 h1:9fQd+ICuRIu/ue4vxJZu6/LzxN0HwMds2nq/0cFvxHU=
 github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
@@ -263,10 +284,6 @@ github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZ
 gitlab.com/gitlab-org/gitaly v1.74.0 h1:yVWvO2ulWMwyzAaehgGwGjv1lBPEPvXD9p4ovFhTkbc=
 gitlab.com/gitlab-org/gitaly v1.74.0/go.mod h1:AXxSyIMuGdv25WogKreZJRGSiWGkySYvDKwBaWD4bnc=
 gitlab.com/gitlab-org/labkit v0.0.0-20190221122536-0c3fc7cdd57c/go.mod h1:rYhLgfrbEcyfinG+R3EvKu6bZSsmwQqcXzLfHWSfUKM=
-gitlab.com/gitlab-org/labkit v0.0.0-20200219201213-4f4f5c1b0539 h1:I2xPr4WlOK3Gt2zhaDEMJpLv5t82/fN3JVuJbbmMn04=
-gitlab.com/gitlab-org/labkit v0.0.0-20200219201213-4f4f5c1b0539/go.mod h1:WyFbg+j26WJmH44qGeINba3QlIVbLxepQ/+qMA2s2+U=
-gitlab.com/gitlab-org/labkit v0.0.0-20200227174525-4f8041c014c2 h1:CCfQxzBy7He5oakCE/3e5yJWFA4yHJqLnGQVje4HiJc=
-gitlab.com/gitlab-org/labkit v0.0.0-20200227174525-4f8041c014c2/go.mod h1:WyFbg+j26WJmH44qGeINba3QlIVbLxepQ/+qMA2s2+U=
 gitlab.com/gitlab-org/labkit v0.0.0-20200327153541-fac94cb428e6 h1:FZ/+I7jHVKUq3ev8SmD1fD743Kb7qYkjDcsaNg887vg=
 gitlab.com/gitlab-org/labkit v0.0.0-20200327153541-fac94cb428e6/go.mod h1:WyFbg+j26WJmH44qGeINba3QlIVbLxepQ/+qMA2s2+U=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
@@ -360,8 +377,10 @@ golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a h1:aYOabOQFp6Vj6W1F80affTUvO
 golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 h1:LfCXLvNmTYH9kEmVgqbnsWfruoXZIrh4YBgqVHtDvw0=
-golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200413165638-669c56c373c4 h1:opSr2sbRXk5X5/givKrrKj9HXxFpW2sdCiP8MJSKLQY=
+golang.org/x/sys v0.0.0-20200413165638-669c56c373c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -447,6 +466,8 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWD
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

internal/api/api.go

@@ -131,6 +131,8 @@ type Response struct {
 	Repository gitalypb.Repository
 	// For git-http, does the requestor have the right to view all refs?
 	ShowAllRefs bool
+	// Detects whether an artifact is used for code intelligence
+	ProcessLsif bool
 }
 
 // singleJoiningSlash is taken from reverseproxy.go:NewSingleHostReverseProxy

internal/artifacts/artifacts_upload_test.go

@@ -118,11 +118,15 @@ type testServer struct {
 	cleanup    func()
 }
 
-func setupWithTmpPath(t *testing.T, filename string, bodyProcessor func(w http.ResponseWriter, r *http.Request)) *testServer {
+func setupWithTmpPath(t *testing.T, filename string, authResponse *api.Response, bodyProcessor func(w http.ResponseWriter, r *http.Request)) *testServer {
 	tempPath, err := ioutil.TempDir("", "uploads")
 	require.NoError(t, err)
 
-	ts := testArtifactsUploadServer(t, api.Response{TempPath: tempPath}, bodyProcessor)
+	if authResponse == nil {
+		authResponse = &api.Response{TempPath: tempPath}
+	}
+
+	ts := testArtifactsUploadServer(t, *authResponse, bodyProcessor)
 
 	var buffer bytes.Buffer
 	writer := multipart.NewWriter(&buffer)
@@ -155,7 +159,7 @@ func testUploadArtifacts(t *testing.T, contentType, url string, body io.Reader)
 }
 
 func TestUploadHandlerAddingMetadata(t *testing.T) {
-	s := setupWithTmpPath(t, "file",
+	s := setupWithTmpPath(t, "file", nil,
 		func(w http.ResponseWriter, r *http.Request) {
 			token, err := jwt.ParseWithClaims(r.Header.Get(upload.RewrittenFieldsHeader), &upload.MultipartClaims{}, testhelper.ParseJWT)
 			require.NoError(t, err)
@@ -185,7 +189,7 @@ func TestUploadHandlerAddingMetadata(t *testing.T) {
 }
 
 func TestUploadHandlerForUnsupportedArchive(t *testing.T) {
-	s := setupWithTmpPath(t, "file", nil)
+	s := setupWithTmpPath(t, "file", nil, nil)
 	defer s.cleanup()
 	require.NoError(t, s.writer.Close())
 
@@ -195,7 +199,7 @@ func TestUploadHandlerForUnsupportedArchive(t *testing.T) {
 }
 
 func TestUploadHandlerForMultipleFiles(t *testing.T) {
-	s := setupWithTmpPath(t, "file", nil)
+	s := setupWithTmpPath(t, "file", nil, nil)
 	defer s.cleanup()
 
 	file, err := s.writer.CreateFormFile("file", "my.file")
@@ -208,8 +212,47 @@ func TestUploadHandlerForMultipleFiles(t *testing.T) {
 }
 
 func TestUploadFormProcessing(t *testing.T) {
-	s := setupWithTmpPath(t, "metadata", nil)
+	s := setupWithTmpPath(t, "metadata", nil, nil)
+	defer s.cleanup()
+	require.NoError(t, s.writer.Close())
+
+	response := testUploadArtifacts(t, s.writer.FormDataContentType(), s.url, s.buffer)
+	testhelper.AssertResponseCode(t, response, http.StatusInternalServerError)
+}
+
+func TestLsifFileProcessing(t *testing.T) {
+	tempPath, err := ioutil.TempDir("", "uploads")
+	require.NoError(t, err)
+
+	s := setupWithTmpPath(t, "file", &api.Response{TempPath: tempPath, ProcessLsif: true}, nil)
 	defer s.cleanup()
+
+	file, err := os.Open("../../testdata/lsif/valid.lsif.zip")
+	require.NoError(t, err)
+
+	_, err = io.Copy(s.fileWriter, file)
+	require.NoError(t, err)
+	require.NoError(t, file.Close())
+	require.NoError(t, s.writer.Close())
+
+	response := testUploadArtifacts(t, s.writer.FormDataContentType(), s.url, s.buffer)
+	testhelper.AssertResponseCode(t, response, http.StatusOK)
+	testhelper.AssertResponseHeader(t, response, MetadataHeaderKey, MetadataHeaderPresent)
+}
+
+func TestInvalidLsifFileProcessing(t *testing.T) {
+	tempPath, err := ioutil.TempDir("", "uploads")
+	require.NoError(t, err)
+
+	s := setupWithTmpPath(t, "file", &api.Response{TempPath: tempPath, ProcessLsif: true}, nil)
+	defer s.cleanup()
+
+	file, err := os.Open("../../testdata/lsif/invalid.lsif.zip")
+	require.NoError(t, err)
+
+	_, err = io.Copy(s.fileWriter, file)
+	require.NoError(t, err)
+	require.NoError(t, file.Close())
 	require.NoError(t, s.writer.Close())
 
 	response := testUploadArtifacts(t, s.writer.FormDataContentType(), s.url, s.buffer)

internal/lsif_transformer/parser/code_hover.go

+package parser
+
+import (
+	"bytes"
+	"encoding/json"
+	"html/template"
+	"io"
+	"strings"
+
+	"github.com/alecthomas/chroma"
+	"github.com/alecthomas/chroma/lexers"
+)
+
+var (
+	languageTemplate = template.Must(template.New("lang").Parse(`<span class="line" lang="{{.}}">`))
+	valueTemplate    = template.Must(template.New("value").Parse(`<span class="{{.Class}}">{{.Value}}</span>`))
+)
+
+type CodeHover struct {
+	Value    string `json:"value"`
+	Language string `json:"language,omitempty"`
+}
+
+func NewCodeHover(content json.RawMessage) (*CodeHover, error) {
+	// Hover value can be either an object: { "value": "func main()", "language": "go" }
+	// Or a string with documentation
+	// We try to unmarshal the content into a string and if we fail, we unmarshal it into an object
+	var codeHover CodeHover
+	if err := json.Unmarshal(content, &codeHover.Value); err != nil {
+		if err := json.Unmarshal(content, &codeHover); err != nil {
+			return nil, err
+		}
+
+		codeHover.Highlight()
+	}
+
+	return &codeHover, nil
+}
+
+func (c *CodeHover) Highlight() {
+	var b bytes.Buffer
+
+	for i, line := range c.codeLines() {
+		if i > 0 {
+			if _, err := io.WriteString(&b, "\n"); err != nil {
+				return
+			}
+		}
+
+		languageTemplate.Execute(&b, c.Language)
+
+		for _, token := range line {
+			if err := writeTokenValue(&b, token); err != nil {
+				return
+			}
+		}
+
+		if _, err := io.WriteString(&b, "</span>"); err != nil {
+			return
+		}
+	}
+
+	c.Value = b.String()
+}
+
+func writeTokenValue(w io.Writer, token chroma.Token) error {
+	if strings.HasPrefix(token.Type.String(), "Keyword") || token.Type == chroma.String || token.Type == chroma.Comment {
+		data := struct {
+			Class string
+			Value string
+		}{
+			Class: chroma.StandardTypes[token.Type],
+			Value: replaceNewLines(token.Value),
+		}
+		return valueTemplate.Execute(w, data)
+	}
+
+	_, err := io.WriteString(w, template.HTMLEscapeString(replaceNewLines(token.Value)))
+	return err
+}
+
+func replaceNewLines(value string) string {
+	return strings.ReplaceAll(value, "\n", "")
+}
+
+func (c *CodeHover) codeLines() [][]chroma.Token {
+	lexer := lexers.Get(c.Language)
+	if lexer == nil {
+		return [][]chroma.Token{}
+	}
+
+	iterator, err := lexer.Tokenise(nil, c.Value)
+	if err != nil {
+		return [][]chroma.Token{}
+	}
+
+	return chroma.SplitTokensIntoLines(iterator.Tokens())
+}

internal/lsif_transformer/parser/code_hover_test.go

+package parser
+
+import (
+	"encoding/json"
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestHighlight(t *testing.T) {
+	tests := []struct {
+		name     string
+		language string
+		value    string
+		want     string
+	}{
+		{
+			name:     "go function definition",
+			language: "go",
+			value:    "func main()",
+			want:     "<span class=\"line\" lang=\"go\"><span class=\"kd\">func</span> main()</span>",
+		},
+		{
+			name:     "go struct definition",
+			language: "go",
+			value:    "type Command struct",
+			want:     "<span class=\"line\" lang=\"go\"><span class=\"kd\">type</span> Command <span class=\"kd\">struct</span></span>",
+		},
+		{
+			name:     "go struct multiline definition",
+			language: "go",
+			value:    `struct {\nConfig *Config\nReadWriter *ReadWriter\nEOFSent bool\n}`,
+			want:     "<span class=\"line\" lang=\"go\"><span class=\"kd\">struct</span> {</span>\n<span class=\"line\" lang=\"go\">Config *Config</span>\n<span class=\"line\" lang=\"go\">ReadWriter *ReadWriter</span>\n<span class=\"line\" lang=\"go\">EOFSent <span class=\"kt\">bool</span></span>\n<span class=\"line\" lang=\"go\">}</span>",
+		},
+		{
+			name:     "ruby method definition",
+			language: "ruby",
+			value:    "def read(line)",
+			want:     "<span class=\"line\" lang=\"ruby\"><span class=\"k\">def</span> read(line)</span>",
+		},
+		{
+			name:     "amp symbol is escaped",
+			language: "ruby",
+			value:    `def &(line)\nend`,
+			want:     "<span class=\"line\" lang=\"ruby\"><span class=\"k\">def</span> &amp;(line)</span>\n<span class=\"line\" lang=\"ruby\"><span class=\"k\">end</span></span>",
+		},
+		{
+			name:     "less symbol is escaped",
+			language: "ruby",
+			value:    "def <(line)",
+			want:     "<span class=\"line\" lang=\"ruby\"><span class=\"k\">def</span> &lt;(line)</span>",
+		},
+		{
+			name:     "more symbol is escaped",
+			language: "ruby",
+			value:    `def >(line)\nend`,
+			want:     "<span class=\"line\" lang=\"ruby\"><span class=\"k\">def</span> &gt;(line)</span>\n<span class=\"line\" lang=\"ruby\"><span class=\"k\">end</span></span>",
+		},
+		{
+			name:     "unknown/malicious language is passed",
+			language: "<lang> alert(1); </lang>",
+			value:    `def a;\nend`,
+			want:     "",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			raw := []byte(fmt.Sprintf(`{"language":"%s","value":"%s"}`, tt.language, tt.value))
+			c, err := NewCodeHover(json.RawMessage(raw))
+
+			require.NoError(t, err)
+			require.Equal(t, tt.want, c.Value)
+		})
+	}
+}
+
+func TestMarkdown(t *testing.T) {
+	value := `"This method reverses a string \n\n"`
+	c, err := NewCodeHover(json.RawMessage(value))
+
+	require.NoError(t, err)
+	require.Equal(t, "This method reverses a string \n\n", c.Value)
+}

internal/lsif_transformer/parser/docs.go

+package parser
+
+import (
+	"archive/zip"
+	"encoding/json"
+	"strings"
+)
+
+type Line struct {
+	Type string `json:"label"`
+}
+
+type Docs struct {
+	Root      string
+	Entries   map[string]string
+	DocRanges map[string][]string
+	Ranges    *Ranges
+}
+
+type Document struct {
+	Id  string `json:"id"`
+	Uri string `json:"uri"`
+}
+
+type DocumentRange struct {
+	OutV     string   `json:"outV"`
+	RangeIds []string `json:"inVs"`
+}
+
+type Metadata struct {
+	Root string `json:"projectRoot"`
+}
+
+func NewDocs(tempDir string) (*Docs, error) {
+	ranges, err := NewRanges(tempDir)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Docs{
+		Root:      "file:///",
+		Entries:   make(map[string]string),
+		DocRanges: make(map[string][]string),
+		Ranges:    ranges,
+	}, nil
+}
+
+func (d *Docs) Read(line []byte) error {
+	l := Line{}
+	if err := json.Unmarshal(line, &l); err != nil {
+		return err
+	}
+
+	switch l.Type {
+	case "metaData":
+		if err := d.addMetadata(line); err != nil {
+			return err
+		}
+	case "document":
+		if err := d.addDocument(line); err != nil {
+			return err
+		}
+	case "contains":
+		if err := d.addDocRanges(line); err != nil {
+			return err
+		}
+	default:
+		return d.Ranges.Read(l.Type, line)
+	}
+
+	return nil
+}
+
+func (d *Docs) Close() error {
+	return d.Ranges.Close()
+}
+
+func (d *Docs) SerializeEntries(w *zip.Writer) error {
+	for id, path := range d.Entries {
+		filePath := Lsif + "/" + path + ".json"
+
+		f, err := w.Create(filePath)
+		if err != nil {
+			return err
+		}
+
+		if err := d.Ranges.Serialize(f, d.DocRanges[id], d.Entries); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (d *Docs) addMetadata(line []byte) error {
+	var metadata Metadata
+	if err := json.Unmarshal(line, &metadata); err != nil {
+		return err
+	}
+
+	d.Root = strings.TrimSpace(metadata.Root) + "/"
+
+	return nil
+}
+
+func (d *Docs) addDocument(line []byte) error {
+	var doc Document
+	if err := json.Unmarshal(line, &doc); err != nil {
+		return err
+	}
+
+	d.Entries[doc.Id] = strings.TrimPrefix(doc.Uri, d.Root)
+
+	return nil
+}
+
+func (d *Docs) addDocRanges(line []byte) error {
+	var docRange DocumentRange
+	if err := json.Unmarshal(line, &docRange); err != nil {
+		return err
+	}
+
+	d.DocRanges[docRange.OutV] = docRange.RangeIds
+
+	return nil
+}

internal/lsif_transformer/parser/docs_test.go

+package parser
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func createLine(id, label, uri string) []byte {
+	return []byte(fmt.Sprintf(`{"id":"%s","label":"%s","uri":"%s"}`, id, label, uri))
+}
+
+func TestRead(t *testing.T) {
+	d, err := NewDocs("")
+	require.NoError(t, err)
+	defer d.Close()
+
+	metadataLine := []byte(`{"id":"1","label":"metaData","projectRoot":"file:///Users/nested"}`)
+
+	require.NoError(t, d.Read(metadataLine))
+	require.NoError(t, d.Read(createLine("2", "document", "file:///Users/nested/file.rb")))
+	require.NoError(t, d.Read(createLine("3", "document", "file:///Users/nested/folder/file.rb")))
+	require.NoError(t, d.Read(createLine("4", "document", "file:///Users/wrong/file.rb")))
+
+	require.Equal(t, d.Entries["2"], "file.rb")
+	require.Equal(t, d.Entries["3"], "folder/file.rb")
+	require.Equal(t, d.Entries["4"], "file:///Users/wrong/file.rb")
+}
+
+func TestReadContainsLine(t *testing.T) {
+	d, err := NewDocs("")
+	require.NoError(t, err)
+	defer d.Close()
+
+	line := []byte(`{"id":"5","label":"contains","outV":"1", "inVs": ["2", "3"]}`)
+
+	require.NoError(t, d.Read(line))
+
+	require.Equal(t, []string{"2", "3"}, d.DocRanges["1"])
+}

internal/lsif_transformer/parser/hovers.go

+package parser
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"os"
+)
+
+type Offset struct {
+	At  int
+	Len int
+}
+
+type Hovers struct {
+	Offsets       map[string]*Offset
+	File          *os.File
+	CurrentOffset int
+}
+
+type RawResult struct {
+	Contents []json.RawMessage `json:"contents"`
+}
+
+type RawData struct {
+	Id     string    `json:"id"`
+	Result RawResult `json:"result"`
+}
+
+type HoverRef struct {
+	ResultSetId string `json:"outV"`
+	HoverId     string `json:"inV"`
+}
+
+type ResultSetRef struct {
+	ResultSetId string `json:"outV"`
+	RefId       string `json:"inV"`
+}
+
+func NewHovers(tempDir string) (*Hovers, error) {
+	file, err := ioutil.TempFile(tempDir, "hovers")
+	if err != nil {
+		return nil, err
+	}
+
+	return &Hovers{
+		Offsets:       make(map[string]*Offset),
+		File:          file,
+		CurrentOffset: 0,
+	}, nil
+}
+
+func (h *Hovers) Read(label string, line []byte) error {
+	switch label {
+	case "hoverResult":
+		if err := h.addData(line); err != nil {
+			return err
+		}
+	case "textDocument/hover":
+		if err := h.addHoverRef(line); err != nil {
+			return err
+		}
+	case "textDocument/references":
+		if err := h.addResultSetRef(line); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (h *Hovers) For(refId string) json.RawMessage {
+	offset, ok := h.Offsets[refId]
+	if !ok || offset == nil {
+		return nil
+	}
+
+	hover := make([]byte, offset.Len)
+	_, err := h.File.ReadAt(hover, int64(offset.At))
+	if err != nil {
+		return nil
+	}
+
+	return json.RawMessage(hover)
+}
+
+func (h *Hovers) Close() error {
+	if err := h.File.Close(); err != nil {
+		return err
+	}
+
+	return os.Remove(h.File.Name())
+}
+
+func (h *Hovers) addData(line []byte) error {
+	var rawData RawData
+	if err := json.Unmarshal(line, &rawData); err != nil {
+		return err
+	}
+
+	codeHovers := []*CodeHover{}
+	for _, rawContent := range rawData.Result.Contents {
+		codeHover, err := NewCodeHover(rawContent)
+		if err != nil {
+			return err
+		}
+
+		codeHovers = append(codeHovers, codeHover)
+	}
+
+	codeHoversData, err := json.Marshal(codeHovers)
+	if err != nil {
+		return err
+	}
+
+	n, err := h.File.Write(codeHoversData)
+	if err != nil {
+		return err
+	}
+
+	h.Offsets[rawData.Id] = &Offset{At: h.CurrentOffset, Len: n}
+	h.CurrentOffset += n
+
+	return nil
+}
+
+func (h *Hovers) addHoverRef(line []byte) error {
+	var hoverRef HoverRef
+	if err := json.Unmarshal(line, &hoverRef); err != nil {
+		return err
+	}
+
+	h.Offsets[hoverRef.ResultSetId] = h.Offsets[hoverRef.HoverId]
+
+	return nil
+}
+
+func (h *Hovers) addResultSetRef(line []byte) error {
+	var ref ResultSetRef
+	if err := json.Unmarshal(line, &ref); err != nil {
+		return err
+	}
+
+	offset, ok := h.Offsets[ref.ResultSetId]
+	if !ok {
+		return nil
+	}
+
+	h.Offsets[ref.RefId] = offset
+	delete(h.Offsets, ref.ResultSetId)
+
+	return nil
+}

internal/lsif_transformer/parser/hovers_test.go

+package parser
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestHoversRead(t *testing.T) {
+	h := setupHovers(t)
+
+	require.Equal(t, `[{"value":"hello"}]`, string(h.For("1")))
+
+	require.NoError(t, h.Close())
+}
+
+func setupHovers(t *testing.T) *Hovers {
+	h, err := NewHovers("")
+	require.NoError(t, err)
+
+	require.NoError(t, h.Read("hoverResult", []byte(`{"id":"2","label":"hoverResult","result":{"contents": ["hello"]}}`)))
+	require.NoError(t, h.Read("textDocument/hover", []byte(`{"id":"4","label":"textDocument/hover","outV":"3","inV":"2"}`)))
+	require.NoError(t, h.Read("textDocument/references", []byte(`{"id":"3","label":"textDocument/references","outV":"3","inV":"1"}`)))
+
+	return h
+}

internal/lsif_transformer/parser/parser.go

+package parser
+
+import (
+	"archive/zip"
+	"bufio"
+	"bytes"
+	"errors"
+	"io"
+	"io/ioutil"
+	"os"
+)
+
+var (
+	Lsif = "lsif"
+)
+
+type Parser struct {
+	Docs *Docs
+}
+
+func NewParser(r io.Reader, tempDir string) (*Parser, error) {
+	docs, err := NewDocs(tempDir)
+	if err != nil {
+		return nil, err
+	}
+
+	zr, err := openZipReader(r, tempDir)
+	if err != nil {
+		return nil, err
+	}
+	reader := bufio.NewReader(zr)
+
+	for {
+		line, err := reader.ReadBytes('\n')
+		if err != nil {
+			break
+		}
+
+		if err := docs.Read(line); err != nil {
+			return nil, err
+		}
+	}
+
+	return &Parser{Docs: docs}, nil
+}
+
+func (p *Parser) ZipReader() (io.Reader, error) {
+	buf := new(bytes.Buffer)
+	w := zip.NewWriter(buf)
+
+	if err := p.Docs.SerializeEntries(w); err != nil {
+		return nil, err
+	}
+
+	if err := w.Close(); err != nil {
+		return nil, err
+	}
+
+	return buf, nil
+}
+
+func (p *Parser) Close() error {
+	return p.Docs.Close()
+}
+
+func openZipReader(reader io.Reader, tempDir string) (io.Reader, error) {
+	tempFile, err := ioutil.TempFile(tempDir, Lsif)
+	if err != nil {
+		return nil, err
+	}
+	defer os.Remove(tempFile.Name())
+
+	if _, err := io.Copy(tempFile, reader); err != nil {
+		return nil, err
+	}
+
+	zr, err := zip.OpenReader(tempFile.Name())
+	if err != nil {
+		return nil, err
+	}
+
+	f := zr.File[0]
+	if f == nil {
+		return nil, errors.New("invalid zip file")
+	}
+
+	return f.Open()
+}

internal/lsif_transformer/parser/parser_test.go

+package parser
+
+import (
+	"archive/zip"
+	"bytes"
+	"encoding/json"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestGenerate(t *testing.T) {
+	filePath := "testdata/dump.lsif.zip"
+	tmpDir := filePath + ".tmp"
+	defer os.RemoveAll(tmpDir)
+
+	createFiles(t, filePath, tmpDir)
+
+	verifyCorrectnessOf(t, tmpDir, "lsif/main.go.json")
+	verifyCorrectnessOf(t, tmpDir, "lsif/morestrings/reverse.go.json")
+}
+
+func verifyCorrectnessOf(t *testing.T, tmpDir, fileName string) {
+	file, err := ioutil.ReadFile(filepath.Join(tmpDir, fileName))
+	require.NoError(t, err)
+
+	var buf bytes.Buffer
+	require.NoError(t, json.Indent(&buf, file, "", "    "))
+
+	expected, err := ioutil.ReadFile(filepath.Join("testdata/expected/", fileName))
+	require.NoError(t, err)
+
+	require.Equal(t, string(expected), buf.String())
+}
+
+func createFiles(t *testing.T, filePath, tmpDir string) {
+	file, err := os.Open(filePath)
+	require.NoError(t, err)
+
+	p, err := NewParser(file, "")
+	require.NoError(t, err)
+
+	r, err := p.ZipReader()
+	require.NoError(t, err)
+
+	require.NoError(t, p.Close())
+
+	zipFileName := tmpDir + ".zip"
+	w, err := os.Create(zipFileName)
+	require.NoError(t, err)
+	defer os.RemoveAll(zipFileName)
+
+	_, err = io.Copy(w, r)
+	require.NoError(t, err)
+
+	extractZipFiles(t, tmpDir, zipFileName)
+}
+
+func extractZipFiles(t *testing.T, tmpDir, zipFileName string) {
+	zipReader, err := zip.OpenReader(zipFileName)
+	require.NoError(t, err)
+
+	for _, file := range zipReader.Reader.File {
+		zippedFile, err := file.Open()
+		require.NoError(t, err)
+		defer zippedFile.Close()
+
+		fileDir, fileName := filepath.Split(file.Name)
+		require.NoError(t, os.MkdirAll(filepath.Join(tmpDir, fileDir), os.ModePerm))
+
+		outputFile, err := os.Create(filepath.Join(tmpDir, fileDir, fileName))
+		require.NoError(t, err)
+		defer outputFile.Close()
+
+		_, err = io.Copy(outputFile, zippedFile)
+		require.NoError(t, err)
+	}
+}

internal/lsif_transformer/parser/ranges.go

+package parser
+
+import (
+	"encoding/json"
+	"io"
+	"strconv"
+)
+
+const Definitions = "definitions"
+const References = "references"
+
+type Ranges struct {
+	Entries map[string]*Range
+	DefRefs map[string]*DefRef
+	Hovers  *Hovers
+}
+
+type RawRange struct {
+	Id   string `json:"id"`
+	Data Range  `json:"start"`
+}
+
+type Range struct {
+	Line      int `json:"line"`
+	Character int `json:"character"`
+	RefId     string
+}
+
+type RawDefRef struct {
+	Property string   `json:"property"`
+	RefId    string   `json:"outV"`
+	RangeIds []string `json:"inVs"`
+	DocId    string   `json:"document"`
+}
+
+type DefRef struct {
+	Line  string
+	DocId string
+}
+
+type SerializedRange struct {
+	StartLine      int             `json:"start_line"`
+	StartChar      int             `json:"start_char"`
+	DefinitionPath string          `json:"definition_path,omitempty"`
+	Hover          json.RawMessage `json:"hover"`
+}
+
+func NewRanges(tempDir string) (*Ranges, error) {
+	hovers, err := NewHovers(tempDir)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Ranges{
+		Entries: make(map[string]*Range),
+		DefRefs: make(map[string]*DefRef),
+		Hovers:  hovers,
+	}, nil
+}
+
+func (r *Ranges) Read(label string, line []byte) error {
+	switch label {
+	case "range":
+		if err := r.addRange(line); err != nil {
+			return err
+		}
+	case "item":
+		if err := r.addItem(line); err != nil {
+			return err
+		}
+	default:
+		return r.Hovers.Read(label, line)
+	}
+
+	return nil
+}
+
+func (r *Ranges) Serialize(f io.Writer, rangeIds []string, docs map[string]string) error {
+	encoder := json.NewEncoder(f)
+	n := len(rangeIds)
+
+	if _, err := f.Write([]byte("[")); err != nil {
+		return err
+	}
+
+	for i, rangeId := range rangeIds {
+		entry := r.Entries[rangeId]
+		serializedRange := SerializedRange{
+			StartLine:      entry.Line,
+			StartChar:      entry.Character,
+			DefinitionPath: r.definitionPathFor(docs, entry.RefId),
+			Hover:          r.Hovers.For(entry.RefId),
+		}
+		if err := encoder.Encode(serializedRange); err != nil {
+			return err
+		}
+		if i+1 < n {
+			if _, err := f.Write([]byte(",")); err != nil {
+				return err
+			}
+		}
+	}
+
+	if _, err := f.Write([]byte("]")); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (r *Ranges) Close() error {
+	return r.Hovers.Close()
+}
+
+func (r *Ranges) definitionPathFor(docs map[string]string, refId string) string {
+	defRef, ok := r.DefRefs[refId]
+	if !ok {
+		return ""
+	}
+
+	defPath := docs[defRef.DocId] + "#L" + defRef.Line
+
+	return defPath
+}
+
+func (r *Ranges) addRange(line []byte) error {
+	var rg RawRange
+	if err := json.Unmarshal(line, &rg); err != nil {
+		return err
+	}
+
+	r.Entries[rg.Id] = &rg.Data
+
+	return nil
+}
+
+func (r *Ranges) addItem(line []byte) error {
+	var defRef RawDefRef
+	if err := json.Unmarshal(line, &defRef); err != nil {
+		return err
+	}
+
+	if defRef.Property != Definitions && defRef.Property != References {
+		return nil
+	}
+
+	for _, rangeId := range defRef.RangeIds {
+		if entry, ok := r.Entries[rangeId]; ok {
+			entry.RefId = defRef.RefId
+		}
+	}
+
+	if defRef.Property != Definitions {
+		return nil
+	}
+
+	defRange := r.Entries[defRef.RangeIds[0]]
+
+	r.DefRefs[defRef.RefId] = &DefRef{
+		Line:  strconv.Itoa(defRange.Line + 1),
+		DocId: defRef.DocId,
+	}
+
+	return nil
+}

internal/lsif_transformer/parser/ranges_test.go

+package parser
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestRangesRead(t *testing.T) {
+	r, cleanup := setup(t)
+	defer cleanup()
+
+	firstRange := Range{Line: 1, Character: 2, RefId: "3"}
+	require.Equal(t, &firstRange, r.Entries["1"])
+
+	secondRange := Range{Line: 5, Character: 4, RefId: "3"}
+	require.Equal(t, &secondRange, r.Entries["2"])
+}
+
+func TestSerialize(t *testing.T) {
+	r, cleanup := setup(t)
+	defer cleanup()
+
+	docs := map[string]string{"6": "def-path"}
+
+	var buf bytes.Buffer
+	err := r.Serialize(&buf, []string{"1"}, docs)
+	want := `[{"start_line":1,"start_char":2,"definition_path":"def-path#L2","hover":null}` + "\n]"
+
+	require.NoError(t, err)
+	require.Equal(t, want, buf.String())
+}
+
+func setup(t *testing.T) (*Ranges, func()) {
+	r, err := NewRanges("")
+	require.NoError(t, err)
+
+	require.NoError(t, r.Read("range", []byte(`{"id":"1","label":"range","start":{"line":1,"character":2}}`)))
+	require.NoError(t, r.Read("range", []byte(`{"id":"2","label":"range","start":{"line":5,"character":4}}`)))
+
+	require.NoError(t, r.Read("item", []byte(`{"id":"4","label":"item","property":"definitions","outV":"3","inVs":["1"],"document":"6"}`)))
+	require.NoError(t, r.Read("item", []byte(`{"id":"4","label":"item","property":"references","outV":"3","inVs":["2"]}`)))
+
+	cleanup := func() {
+		require.NoError(t, r.Close())
+	}
+
+	return r, cleanup
+}

internal/lsif_transformer/parser/testdata/expected/lsif/main.go.json

+[
+    {
+        "start_line": 7,
+        "start_char": 1,
+        "definition_path": "main.go#L4",
+        "hover": [
+            {
+                "value": "\u003cspan class=\"line\" lang=\"go\"\u003e\u003cspan class=\"kn\"\u003epackage\u003c/span\u003e \u003cspan class=\"s\"\u003e\u0026#34;github.com/user/hello/morestrings\u0026#34;\u003c/span\u003e\u003c/span\u003e",
+                "language": "go"
+            },
+            {
+                "value": "Package morestrings implements additional functions to manipulate UTF-8 encoded strings, beyond what is provided in the standard \"strings\" package. \n\n"
+            }
+        ]
+    },
+    {
+        "start_line": 7,
+        "start_char": 13,
+        "definition_path": "morestrings/reverse.go#L12",
+        "hover": [
+            {
+                "value": "\u003cspan class=\"line\" lang=\"go\"\u003e\u003cspan class=\"kd\"\u003efunc\u003c/span\u003e Reverse(s \u003cspan class=\"kt\"\u003estring\u003c/span\u003e) \u003cspan class=\"kt\"\u003estring\u003c/span\u003e\u003c/span\u003e",
+                "language": "go"
+            },
+            {
+                "value": "This method reverses a string \n\n"
+            }
+        ]
+    },
+    {
+        "start_line": 8,
+        "start_char": 1,
+        "definition_path": "main.go#L4",
+        "hover": [
+            {
+                "value": "\u003cspan class=\"line\" lang=\"go\"\u003e\u003cspan class=\"kn\"\u003epackage\u003c/span\u003e \u003cspan class=\"s\"\u003e\u0026#34;github.com/user/hello/morestrings\u0026#34;\u003c/span\u003e\u003c/span\u003e",
+                "language": "go"
+            },
+            {
+                "value": "Package morestrings implements additional functions to manipulate UTF-8 encoded strings, beyond what is provided in the standard \"strings\" package. \n\n"
+            }
+        ]
+    },
+    {
+        "start_line": 8,
+        "start_char": 13,
+        "definition_path": "morestrings/reverse.go#L5",
+        "hover": [
+            {
+                "value": "\u003cspan class=\"line\" lang=\"go\"\u003e\u003cspan class=\"kd\"\u003efunc\u003c/span\u003e Func2(i \u003cspan class=\"kt\"\u003eint\u003c/span\u003e) \u003cspan class=\"kt\"\u003estring\u003c/span\u003e\u003c/span\u003e",
+                "language": "go"
+            }
+        ]
+    },
+    {
+        "start_line": 6,
+        "start_char": 5,
+        "definition_path": "main.go#L7",
+        "hover": [
+            {
+                "value": "\u003cspan class=\"line\" lang=\"go\"\u003e\u003cspan class=\"kd\"\u003efunc\u003c/span\u003e main()\u003c/span\u003e",
+                "language": "go"
+            }
+        ]
+    },
+    {
+        "start_line": 3,
+        "start_char": 2,
+        "definition_path": "main.go#L4",
+        "hover": [
+            {
+                "value": "\u003cspan class=\"line\" lang=\"go\"\u003e\u003cspan class=\"kn\"\u003epackage\u003c/span\u003e \u003cspan class=\"s\"\u003e\u0026#34;github.com/user/hello/morestrings\u0026#34;\u003c/span\u003e\u003c/span\u003e",
+                "language": "go"
+            },
+            {
+                "value": "Package morestrings implements additional functions to manipulate UTF-8 encoded strings, beyond what is provided in the standard \"strings\" package. \n\n"
+            }
+        ]
+    }
+]
\ No newline at end of file

internal/lsif_transformer/parser/testdata/expected/lsif/morestrings/reverse.go.json

+[
+    {
+        "start_line": 11,
+        "start_char": 5,
+        "definition_path": "morestrings/reverse.go#L12",
+        "hover": [
+            {
+                "value": "\u003cspan class=\"line\" lang=\"go\"\u003e\u003cspan class=\"kd\"\u003efunc\u003c/span\u003e Reverse(s \u003cspan class=\"kt\"\u003estring\u003c/span\u003e) \u003cspan class=\"kt\"\u003estring\u003c/span\u003e\u003c/span\u003e",
+                "language": "go"
+            },
+            {
+                "value": "This method reverses a string \n\n"
+            }
+        ]
+    },
+    {
+        "start_line": 4,
+        "start_char": 11,
+        "definition_path": "morestrings/reverse.go#L5",
+        "hover": [
+            {
+                "value": "\u003cspan class=\"line\" lang=\"go\"\u003e\u003cspan class=\"kd\"\u003evar\u003c/span\u003e i \u003cspan class=\"kt\"\u003eint\u003c/span\u003e\u003c/span\u003e",
+                "language": "go"
+            }
+        ]
+    },
+    {
+        "start_line": 11,
+        "start_char": 13,
+        "definition_path": "morestrings/reverse.go#L12",
+        "hover": [
+            {
+                "value": "\u003cspan class=\"line\" lang=\"go\"\u003e\u003cspan class=\"kd\"\u003evar\u003c/span\u003e s \u003cspan class=\"kt\"\u003estring\u003c/span\u003e\u003c/span\u003e",
+                "language": "go"
+            }
+        ]
+    },
+    {
+        "start_line": 12,
+        "start_char": 1,
+        "definition_path": "morestrings/reverse.go#L13",
+        "hover": [
+            {
+                "value": "\u003cspan class=\"line\" lang=\"go\"\u003e\u003cspan class=\"kd\"\u003evar\u003c/span\u003e a \u003cspan class=\"kt\"\u003estring\u003c/span\u003e\u003c/span\u003e",
+                "language": "go"
+            }
+        ]
+    },
+    {
+        "start_line": 5,
+        "start_char": 1,
+        "definition_path": "morestrings/reverse.go#L6",
+        "hover": [
+            {
+                "value": "\u003cspan class=\"line\" lang=\"go\"\u003e\u003cspan class=\"kd\"\u003evar\u003c/span\u003e b \u003cspan class=\"kt\"\u003estring\u003c/span\u003e\u003c/span\u003e",
+                "language": "go"
+            }
+        ]
+    },
+    {
+        "start_line": 14,
+        "start_char": 8,
+        "definition_path": "morestrings/reverse.go#L13",
+        "hover": [
+            {
+                "value": "\u003cspan class=\"line\" lang=\"go\"\u003e\u003cspan class=\"kd\"\u003evar\u003c/span\u003e a \u003cspan class=\"kt\"\u003estring\u003c/span\u003e\u003c/span\u003e",
+                "language": "go"
+            }
+        ]
+    },
+    {
+        "start_line": 7,
+        "start_char": 8,
+        "definition_path": "morestrings/reverse.go#L6",
+        "hover": [
+            {
+                "value": "\u003cspan class=\"line\" lang=\"go\"\u003e\u003cspan class=\"kd\"\u003evar\u003c/span\u003e b \u003cspan class=\"kt\"\u003estring\u003c/span\u003e\u003c/span\u003e",
+                "language": "go"
+            }
+        ]
+    },
+    {
+        "start_line": 4,
+        "start_char": 5,
+        "definition_path": "morestrings/reverse.go#L5",
+        "hover": [
+            {
+                "value": "\u003cspan class=\"line\" lang=\"go\"\u003e\u003cspan class=\"kd\"\u003efunc\u003c/span\u003e Func2(i \u003cspan class=\"kt\"\u003eint\u003c/span\u003e) \u003cspan class=\"kt\"\u003estring\u003c/span\u003e\u003c/span\u003e",
+                "language": "go"
+            }
+        ]
+    }
+]
\ No newline at end of file

internal/upload/rewrite.go

@@ -14,6 +14,7 @@ import (
 
 	"gitlab.com/gitlab-org/gitlab-workhorse/internal/api"
 	"gitlab.com/gitlab-org/gitlab-workhorse/internal/filestore"
+	"gitlab.com/gitlab-org/gitlab-workhorse/internal/lsif_transformer/parser"
 	"gitlab.com/gitlab-org/gitlab-workhorse/internal/upload/exif"
 )
 
@@ -140,6 +141,11 @@ func (rew *rewriter) handleFilePart(ctx context.Context, name string, p *multipa
 		inputReader = p
 	}
 
+	inputReader, err := rew.handleLsifUpload(inputReader, opts.LocalTempPath)
+	if err != nil {
+		return err
+	}
+
 	fh, err := filestore.SaveFileFromReader(ctx, inputReader, -1, opts)
 	if err != nil {
 		switch err {
@@ -165,6 +171,28 @@ func (rew *rewriter) handleFilePart(ctx context.Context, name string, p *multipa
 	return rew.filter.ProcessFile(ctx, name, fh, rew.writer)
 }
 
+func (rew *rewriter) handleLsifUpload(reader io.Reader, tempPath string) (io.Reader, error) {
+	if rew.preauth.ProcessLsif {
+		p, err := parser.NewParser(reader, tempPath)
+		if err != nil {
+			return nil, err
+		}
+
+		z, err := p.ZipReader()
+		if err != nil {
+			return nil, err
+		}
+
+		if err := p.Close(); err != nil {
+			return nil, err
+		}
+
+		return z, nil
+	}
+
+	return reader, nil
+}
+
 func (rew *rewriter) copyPart(ctx context.Context, name string, p *multipart.Part) error {
 	np, err := rew.writer.CreatePart(p.Header)
 	if err != nil {