Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
G
gitlab-workhorse
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
1
Merge Requests
1
Analytics
Analytics
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Commits
Issue Boards
Open sidebar
nexedi
gitlab-workhorse
Commits
07f5e048
Commit
07f5e048
authored
Mar 01, 2018
by
Andrew Newdigate
Committed by
Nick Thomas
Mar 01, 2018
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Structured logging rebased
parent
271b5fb7
Changes
26
Show whitespace changes
Inline
Side-by-side
Showing
26 changed files
with
447 additions
and
155 deletions
+447
-155
_support/fake-auth-backend.go
_support/fake-auth-backend.go
+2
-1
internal/artifacts/artifacts_test.go
internal/artifacts/artifacts_test.go
+3
-2
internal/artifacts/entry.go
internal/artifacts/entry.go
+7
-2
internal/git/blob.go
internal/git/blob.go
+6
-2
internal/git/diff.go
internal/git/diff.go
+7
-2
internal/git/format-patch.go
internal/git/format-patch.go
+7
-2
internal/helper/accesslogformatter.go
internal/helper/accesslogformatter.go
+53
-0
internal/helper/accesslogformatter_test.go
internal/helper/accesslogformatter_test.go
+57
-0
internal/helper/clock.go
internal/helper/clock.go
+24
-0
internal/helper/helpers.go
internal/helper/helpers.go
+8
-4
internal/helper/logging.go
internal/helper/logging.go
+50
-30
internal/helper/logging_test.go
internal/helper/logging_test.go
+87
-0
internal/redis/keywatcher.go
internal/redis/keywatcher.go
+2
-1
internal/redis/redis.go
internal/redis/redis.go
+10
-3
internal/sendfile/sendfile.go
internal/sendfile/sendfile.go
+8
-2
internal/sendurl/sendurl.go
internal/sendurl/sendurl.go
+6
-2
internal/staticpages/servefile.go
internal/staticpages/servefile.go
+8
-2
internal/terminal/terminal.go
internal/terminal/terminal.go
+13
-6
internal/testhelper/gitaly.go
internal/testhelper/gitaly.go
+2
-1
internal/testhelper/testhelper.go
internal/testhelper/testhelper.go
+2
-1
internal/upstream/upstream.go
internal/upstream/upstream.go
+2
-2
logging.go
logging.go
+60
-17
main.go
main.go
+14
-6
main_test.go
main_test.go
+4
-64
sendfile_test.go
sendfile_test.go
+2
-1
terminal_test.go
terminal_test.go
+3
-2
No files found.
_support/fake-auth-backend.go
View file @
07f5e048
...
@@ -2,9 +2,10 @@ package main
...
@@ -2,9 +2,10 @@ package main
import
(
import
(
"fmt"
"fmt"
"log"
"net/http"
"net/http"
"os"
"os"
log
"github.com/sirupsen/logrus"
)
)
func
main
()
{
func
main
()
{
...
...
internal/artifacts/artifacts_test.go
View file @
07f5e048
package
artifacts
package
artifacts
import
(
import
(
"log"
"os"
"os"
"testing"
"testing"
log
"github.com/sirupsen/logrus"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/testhelper"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/testhelper"
)
)
func
TestMain
(
m
*
testing
.
M
)
{
func
TestMain
(
m
*
testing
.
M
)
{
cleanup
,
err
:=
testhelper
.
BuildExecutables
()
cleanup
,
err
:=
testhelper
.
BuildExecutables
()
if
err
!=
nil
{
if
err
!=
nil
{
log
.
Printf
(
"Test setup: failed to build executables: %v"
,
err
)
log
.
WithError
(
err
)
.
Print
(
"Test setup: failed to build executables"
)
os
.
Exit
(
1
)
os
.
Exit
(
1
)
}
}
...
...
internal/artifacts/entry.go
View file @
07f5e048
...
@@ -4,7 +4,6 @@ import (
...
@@ -4,7 +4,6 @@ import (
"bufio"
"bufio"
"fmt"
"fmt"
"io"
"io"
"log"
"mime"
"mime"
"net/http"
"net/http"
"os"
"os"
...
@@ -13,6 +12,8 @@ import (
...
@@ -13,6 +12,8 @@ import (
"strings"
"strings"
"syscall"
"syscall"
log
"github.com/sirupsen/logrus"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/senddata"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/senddata"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/zipartifacts"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/zipartifacts"
...
@@ -31,7 +32,11 @@ func (e *entry) Inject(w http.ResponseWriter, r *http.Request, sendData string)
...
@@ -31,7 +32,11 @@ func (e *entry) Inject(w http.ResponseWriter, r *http.Request, sendData string)
return
return
}
}
log
.
Printf
(
"SendEntry: sending %q from %q for %q"
,
params
.
Entry
,
params
.
Archive
,
r
.
URL
.
Path
)
log
.
WithFields
(
log
.
Fields
{
"entry"
:
params
.
Entry
,
"archive"
:
params
.
Archive
,
"path"
:
r
.
URL
.
Path
,
})
.
Print
(
"SendEntry: sending"
)
if
params
.
Archive
==
""
||
params
.
Entry
==
""
{
if
params
.
Archive
==
""
||
params
.
Entry
==
""
{
helper
.
Fail500
(
w
,
r
,
fmt
.
Errorf
(
"SendEntry: Archive or Entry is empty"
))
helper
.
Fail500
(
w
,
r
,
fmt
.
Errorf
(
"SendEntry: Archive or Entry is empty"
))
...
...
internal/git/blob.go
View file @
07f5e048
...
@@ -3,10 +3,11 @@ package git
...
@@ -3,10 +3,11 @@ package git
import
(
import
(
"fmt"
"fmt"
"io"
"io"
"log"
"net/http"
"net/http"
"strings"
"strings"
log
"github.com/sirupsen/logrus"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/gitaly"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/gitaly"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/senddata"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/senddata"
...
@@ -50,7 +51,10 @@ func handleSendBlobWithGitaly(w http.ResponseWriter, r *http.Request, params *bl
...
@@ -50,7 +51,10 @@ func handleSendBlobWithGitaly(w http.ResponseWriter, r *http.Request, params *bl
}
}
func
handleSendBlobLocally
(
w
http
.
ResponseWriter
,
r
*
http
.
Request
,
params
*
blobParams
)
{
func
handleSendBlobLocally
(
w
http
.
ResponseWriter
,
r
*
http
.
Request
,
params
*
blobParams
)
{
log
.
Printf
(
"SendBlob: sending %q for %q"
,
params
.
BlobId
,
r
.
URL
.
Path
)
log
.
WithFields
(
log
.
Fields
{
"blobId"
:
params
.
BlobId
,
"path"
:
r
.
URL
.
Path
,
})
.
Print
(
"SendBlob: sending"
)
sizeOutput
,
err
:=
gitCommand
(
"git"
,
"--git-dir="
+
params
.
RepoPath
,
"cat-file"
,
"-s"
,
params
.
BlobId
)
.
Output
()
sizeOutput
,
err
:=
gitCommand
(
"git"
,
"--git-dir="
+
params
.
RepoPath
,
"cat-file"
,
"-s"
,
params
.
BlobId
)
.
Output
()
if
err
!=
nil
{
if
err
!=
nil
{
...
...
internal/git/diff.go
View file @
07f5e048
...
@@ -3,9 +3,10 @@ package git
...
@@ -3,9 +3,10 @@ package git
import
(
import
(
"fmt"
"fmt"
"io"
"io"
"log"
"net/http"
"net/http"
log
"github.com/sirupsen/logrus"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/gitaly"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/gitaly"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/senddata"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/senddata"
...
@@ -60,7 +61,11 @@ func handleSendDiffWithGitaly(w http.ResponseWriter, r *http.Request, params *di
...
@@ -60,7 +61,11 @@ func handleSendDiffWithGitaly(w http.ResponseWriter, r *http.Request, params *di
}
}
func
handleSendDiffLocally
(
w
http
.
ResponseWriter
,
r
*
http
.
Request
,
params
*
diffParams
)
{
func
handleSendDiffLocally
(
w
http
.
ResponseWriter
,
r
*
http
.
Request
,
params
*
diffParams
)
{
log
.
Printf
(
"SendDiff: sending diff between %q and %q for %q"
,
params
.
ShaFrom
,
params
.
ShaTo
,
r
.
URL
.
Path
)
log
.
WithFields
(
log
.
Fields
{
"shaFrom"
:
params
.
ShaFrom
,
"shaTo"
:
params
.
ShaTo
,
"path"
:
r
.
URL
.
Path
,
})
.
Print
(
"SendDiff: sending diff"
)
gitDiffCmd
:=
gitCommand
(
"git"
,
"--git-dir="
+
params
.
RepoPath
,
"diff"
,
params
.
ShaFrom
,
params
.
ShaTo
)
gitDiffCmd
:=
gitCommand
(
"git"
,
"--git-dir="
+
params
.
RepoPath
,
"diff"
,
params
.
ShaFrom
,
params
.
ShaTo
)
stdout
,
err
:=
gitDiffCmd
.
StdoutPipe
()
stdout
,
err
:=
gitDiffCmd
.
StdoutPipe
()
...
...
internal/git/format-patch.go
View file @
07f5e048
...
@@ -3,9 +3,10 @@ package git
...
@@ -3,9 +3,10 @@ package git
import
(
import
(
"fmt"
"fmt"
"io"
"io"
"log"
"net/http"
"net/http"
log
"github.com/sirupsen/logrus"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/gitaly"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/gitaly"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/senddata"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/senddata"
...
@@ -59,7 +60,11 @@ func handleSendPatchWithGitaly(w http.ResponseWriter, r *http.Request, params *p
...
@@ -59,7 +60,11 @@ func handleSendPatchWithGitaly(w http.ResponseWriter, r *http.Request, params *p
}
}
func
handleSendPatchLocally
(
w
http
.
ResponseWriter
,
r
*
http
.
Request
,
params
*
patchParams
)
{
func
handleSendPatchLocally
(
w
http
.
ResponseWriter
,
r
*
http
.
Request
,
params
*
patchParams
)
{
log
.
Printf
(
"SendPatch: sending patch between %q and %q for %q"
,
params
.
ShaFrom
,
params
.
ShaTo
,
r
.
URL
.
Path
)
log
.
WithFields
(
log
.
Fields
{
"shaFrom"
:
params
.
ShaFrom
,
"shaTo"
:
params
.
ShaTo
,
"path"
:
r
.
URL
.
Path
,
})
.
Print
(
"SendPatch: sending patch"
)
gitRange
:=
fmt
.
Sprintf
(
"%s..%s"
,
params
.
ShaFrom
,
params
.
ShaTo
)
gitRange
:=
fmt
.
Sprintf
(
"%s..%s"
,
params
.
ShaFrom
,
params
.
ShaTo
)
gitPatchCmd
:=
gitCommand
(
"git"
,
"--git-dir="
+
params
.
RepoPath
,
"format-patch"
,
gitRange
,
"--stdout"
)
gitPatchCmd
:=
gitCommand
(
"git"
,
"--git-dir="
+
params
.
RepoPath
,
"format-patch"
,
gitRange
,
"--stdout"
)
...
...
internal/helper/accesslogformatter.go
0 → 100644
View file @
07f5e048
package
helper
import
(
"bytes"
"fmt"
log
"github.com/sirupsen/logrus"
)
func
forNil
(
v
interface
{},
otherwise
interface
{})
interface
{}
{
if
v
==
nil
{
return
otherwise
}
return
v
}
// accessLogFormatter formats logs into a format similar to the combined access log format
// See https://httpd.apache.org/docs/1.3/logs.html#combined
type
accessLogFormatter
struct
{
clock
Clock
}
// Format renders a single log entry
func
(
f
*
accessLogFormatter
)
Format
(
entry
*
log
.
Entry
)
([]
byte
,
error
)
{
host
:=
forNil
(
entry
.
Data
[
"host"
],
"-"
)
remoteAddr
:=
forNil
(
entry
.
Data
[
"remoteAddr"
],
""
)
method
:=
forNil
(
entry
.
Data
[
"method"
],
""
)
uri
:=
forNil
(
entry
.
Data
[
"uri"
],
""
)
proto
:=
forNil
(
entry
.
Data
[
"proto"
],
""
)
status
:=
forNil
(
entry
.
Data
[
"status"
],
0
)
written
:=
forNil
(
entry
.
Data
[
"written"
],
0
)
referer
:=
forNil
(
entry
.
Data
[
"referer"
],
""
)
userAgent
:=
forNil
(
entry
.
Data
[
"userAgent"
],
""
)
duration
:=
forNil
(
entry
.
Data
[
"duration"
],
0.0
)
now
:=
f
.
clock
.
Now
()
.
Format
(
"2006/01/02:15:04:05 -0700"
)
requestLine
:=
fmt
.
Sprintf
(
"%s %s %s"
,
method
,
uri
,
proto
)
buf
:=
&
bytes
.
Buffer
{}
_
,
err
:=
fmt
.
Fprintf
(
buf
,
"%s %s - - [%s] %q %d %d %q %q %.3f
\n
"
,
host
,
remoteAddr
,
now
,
requestLine
,
status
,
written
,
referer
,
userAgent
,
duration
,
)
return
buf
.
Bytes
(),
err
}
// NewAccessLogFormatter returns a new formatter for combined access logs
func
NewAccessLogFormatter
()
log
.
Formatter
{
return
&
accessLogFormatter
{
clock
:
&
RealClock
{}}
}
internal/helper/accesslogformatter_test.go
0 → 100644
View file @
07f5e048
package
helper
import
(
"io/ioutil"
"testing"
"time"
log
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
)
func
TestAccessLogFormatter_Format
(
t
*
testing
.
T
)
{
discardLogger
:=
log
.
New
()
discardLogger
.
Out
=
ioutil
.
Discard
tests
:=
[]
struct
{
name
string
entry
*
log
.
Entry
want
string
}{
{
"blank"
,
discardLogger
.
WithField
(
"blank"
,
""
),
"- - - [2018/01/07:00:00:00 +0000]
\"
\"
0 0
\"\"
\"\"
0.000
\n
"
,
},
{
"full"
,
discardLogger
.
WithFields
(
log
.
Fields
{
"host"
:
"gitlab.com"
,
"remoteAddr"
:
"127.0.0.1"
,
"method"
:
"GET"
,
"uri"
:
"/"
,
"proto"
:
"HTTP/1.1"
,
"status"
:
200
,
"written"
:
100
,
"referer"
:
"http://localhost"
,
"userAgent"
:
"Mozilla/1.0"
,
"duration"
:
5.0
,
}),
"gitlab.com 127.0.0.1 - - [2018/01/07:00:00:00 +0000]
\"
GET / HTTP/1.1
\"
200 100
\"
http://localhost
\"
\"
Mozilla/1.0
\"
5.000
\n
"
,
},
}
for
_
,
tt
:=
range
tests
{
t
.
Run
(
tt
.
name
,
func
(
t
*
testing
.
T
)
{
f
:=
&
accessLogFormatter
{
clock
:
&
StubClock
{
time
.
Unix
(
1515283200
,
0
)}}
got
,
err
:=
f
.
Format
(
tt
.
entry
)
if
err
!=
nil
{
t
.
Errorf
(
"AccessLogFormatter.Format() error = %v"
,
err
)
return
}
assert
.
Equal
(
t
,
tt
.
want
,
string
(
got
))
})
}
}
internal/helper/clock.go
0 → 100644
View file @
07f5e048
package
helper
import
(
"time"
)
// Clock interface provides the time
type
Clock
interface
{
Now
()
time
.
Time
}
// RealClock is the default time implementation
type
RealClock
struct
{}
// Now returns the time
func
(
RealClock
)
Now
()
time
.
Time
{
return
time
.
Now
()
}
// StubClock is the default time implementation
type
StubClock
struct
{
StubTime
time
.
Time
}
// Now returns a stub time
func
(
c
*
StubClock
)
Now
()
time
.
Time
{
return
c
.
StubTime
}
internal/helper/helpers.go
View file @
07f5e048
...
@@ -4,7 +4,6 @@ import (
...
@@ -4,7 +4,6 @@ import (
"bytes"
"bytes"
"errors"
"errors"
"io/ioutil"
"io/ioutil"
"log"
"mime"
"mime"
"net"
"net"
"net/http"
"net/http"
...
@@ -14,6 +13,8 @@ import (
...
@@ -14,6 +13,8 @@ import (
"regexp"
"regexp"
"strings"
"strings"
"syscall"
"syscall"
log
"github.com/sirupsen/logrus"
)
)
const
NginxResponseBufferHeader
=
"X-Accel-Buffering"
const
NginxResponseBufferHeader
=
"X-Accel-Buffering"
...
@@ -45,9 +46,12 @@ func RequestEntityTooLarge(w http.ResponseWriter, r *http.Request, err error) {
...
@@ -45,9 +46,12 @@ func RequestEntityTooLarge(w http.ResponseWriter, r *http.Request, err error) {
func
printError
(
r
*
http
.
Request
,
err
error
)
{
func
printError
(
r
*
http
.
Request
,
err
error
)
{
if
r
!=
nil
{
if
r
!=
nil
{
log
.
Printf
(
"error: %s %q: %v"
,
r
.
Method
,
ScrubURLParams
(
r
.
RequestURI
),
err
)
log
.
WithFields
(
log
.
Fields
{
"method"
:
r
.
Method
,
"uri"
:
ScrubURLParams
(
r
.
RequestURI
),
})
.
WithError
(
err
)
.
Error
(
"error"
)
}
else
{
}
else
{
log
.
Printf
(
"error: %v"
,
err
)
log
.
WithError
(
err
)
.
Error
(
"unknown error"
)
}
}
}
}
...
@@ -90,7 +94,7 @@ func OpenFile(path string) (file *os.File, fi os.FileInfo, err error) {
...
@@ -90,7 +94,7 @@ func OpenFile(path string) (file *os.File, fi os.FileInfo, err error) {
func
URLMustParse
(
s
string
)
*
url
.
URL
{
func
URLMustParse
(
s
string
)
*
url
.
URL
{
u
,
err
:=
url
.
Parse
(
s
)
u
,
err
:=
url
.
Parse
(
s
)
if
err
!=
nil
{
if
err
!=
nil
{
log
.
Fatalf
(
"urlMustParse: %q %v"
,
s
,
err
)
log
.
WithField
(
"url"
,
s
)
.
WithError
(
err
)
.
Fatal
(
"urlMustParse"
)
}
}
return
u
return
u
}
}
...
...
internal/helper/logging.go
View file @
07f5e048
...
@@ -2,20 +2,17 @@ package helper
...
@@ -2,20 +2,17 @@ package helper
import
(
import
(
"bufio"
"bufio"
"fmt"
"io"
"log"
"net"
"net"
"net/http"
"net/http"
"os"
"strconv"
"strconv"
"time"
"time"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus"
log
"github.com/sirupsen/logrus"
)
)
var
(
var
(
responseLogger
*
log
.
Logger
accessLogEntry
*
log
.
Entry
sessionsActive
=
prometheus
.
NewGauge
(
prometheus
.
GaugeOpts
{
sessionsActive
=
prometheus
.
NewGauge
(
prometheus
.
GaugeOpts
{
Name
:
"gitlab_workhorse_http_sessions_active"
,
Name
:
"gitlab_workhorse_http_sessions_active"
,
...
@@ -32,12 +29,12 @@ var (
...
@@ -32,12 +29,12 @@ var (
)
)
func
init
()
{
func
init
()
{
SetCustomResponseLogger
(
os
.
Stderr
)
registerPrometheusMetrics
()
registerPrometheusMetrics
()
}
}
func
SetCustomResponseLogger
(
writer
io
.
Writer
)
{
// SetAccessLoggerEntry sets the access logger used in the system
responseLogger
=
log
.
New
(
writer
,
""
,
0
)
func
SetAccessLoggerEntry
(
logEntry
*
log
.
Entry
)
{
accessLogEntry
=
logEntry
}
}
func
registerPrometheusMetrics
()
{
func
registerPrometheusMetrics
()
{
...
@@ -48,23 +45,24 @@ func registerPrometheusMetrics() {
...
@@ -48,23 +45,24 @@ func registerPrometheusMetrics() {
type
LoggingResponseWriter
interface
{
type
LoggingResponseWriter
interface
{
http
.
ResponseWriter
http
.
ResponseWriter
Log
(
r
*
http
.
Request
)
RequestFinished
(
r
*
http
.
Request
)
}
}
type
logg
ingResponseWriter
struct
{
type
statsCollect
ingResponseWriter
struct
{
rw
http
.
ResponseWriter
rw
http
.
ResponseWriter
status
int
status
int
wroteHeader
bool
written
int64
written
int64
started
time
.
Time
started
time
.
Time
}
}
type
hijackingResponseWriter
struct
{
type
hijackingResponseWriter
struct
{
logg
ingResponseWriter
statsCollect
ingResponseWriter
}
}
func
New
Logg
ingResponseWriter
(
rw
http
.
ResponseWriter
)
LoggingResponseWriter
{
func
New
StatsCollect
ingResponseWriter
(
rw
http
.
ResponseWriter
)
LoggingResponseWriter
{
sessionsActive
.
Inc
()
sessionsActive
.
Inc
()
out
:=
logg
ingResponseWriter
{
out
:=
statsCollect
ingResponseWriter
{
rw
:
rw
,
rw
:
rw
,
started
:
time
.
Now
(),
started
:
time
.
Now
(),
}
}
...
@@ -77,41 +75,63 @@ func NewLoggingResponseWriter(rw http.ResponseWriter) LoggingResponseWriter {
...
@@ -77,41 +75,63 @@ func NewLoggingResponseWriter(rw http.ResponseWriter) LoggingResponseWriter {
}
}
func
(
l
*
hijackingResponseWriter
)
Hijack
()
(
net
.
Conn
,
*
bufio
.
ReadWriter
,
error
)
{
func
(
l
*
hijackingResponseWriter
)
Hijack
()
(
net
.
Conn
,
*
bufio
.
ReadWriter
,
error
)
{
// The only way to get
here is through NewLogg
ingResponseWriter(), which
// The only way to get
here is through NewStatsCollect
ingResponseWriter(), which
// checks that this cast will be valid.
// checks that this cast will be valid.
hijacker
:=
l
.
rw
.
(
http
.
Hijacker
)
hijacker
:=
l
.
rw
.
(
http
.
Hijacker
)
return
hijacker
.
Hijack
()
return
hijacker
.
Hijack
()
}
}
func
(
l
*
logg
ingResponseWriter
)
Header
()
http
.
Header
{
func
(
l
*
statsCollect
ingResponseWriter
)
Header
()
http
.
Header
{
return
l
.
rw
.
Header
()
return
l
.
rw
.
Header
()
}
}
func
(
l
*
logg
ingResponseWriter
)
Write
(
data
[]
byte
)
(
n
int
,
err
error
)
{
func
(
l
*
statsCollect
ingResponseWriter
)
Write
(
data
[]
byte
)
(
n
int
,
err
error
)
{
if
l
.
status
==
0
{
if
!
l
.
wroteHeader
{
l
.
WriteHeader
(
http
.
StatusOK
)
l
.
WriteHeader
(
http
.
StatusOK
)
}
}
n
,
err
=
l
.
rw
.
Write
(
data
)
n
,
err
=
l
.
rw
.
Write
(
data
)
l
.
written
+=
int64
(
n
)
l
.
written
+=
int64
(
n
)
return
n
,
err
return
n
,
err
}
}
func
(
l
*
logg
ingResponseWriter
)
WriteHeader
(
status
int
)
{
func
(
l
*
statsCollect
ingResponseWriter
)
WriteHeader
(
status
int
)
{
if
l
.
status
!=
0
{
if
l
.
wroteHeader
{
return
return
}
}
l
.
wroteHeader
=
true
l
.
status
=
status
l
.
status
=
status
l
.
rw
.
WriteHeader
(
status
)
l
.
rw
.
WriteHeader
(
status
)
}
}
func
(
l
*
loggingResponseWriter
)
Log
(
r
*
http
.
Request
)
{
func
(
l
*
statsCollectingResponseWriter
)
writeAccessLog
(
r
*
http
.
Request
)
{
if
accessLogEntry
==
nil
{
return
}
accessLogEntry
.
WithFields
(
l
.
accessLogFields
(
r
))
.
Info
(
"access"
)
}
func
(
l
*
statsCollectingResponseWriter
)
accessLogFields
(
r
*
http
.
Request
)
log
.
Fields
{
duration
:=
time
.
Since
(
l
.
started
)
duration
:=
time
.
Since
(
l
.
started
)
responseLogger
.
Printf
(
"%s %s - - [%s] %q %d %d %q %q %f
\n
"
,
r
.
Host
,
r
.
RemoteAddr
,
l
.
started
,
return
log
.
Fields
{
fmt
.
Sprintf
(
"%s %s %s"
,
r
.
Method
,
ScrubURLParams
(
r
.
RequestURI
),
r
.
Proto
),
"host"
:
r
.
Host
,
l
.
status
,
l
.
written
,
ScrubURLParams
(
r
.
Referer
()),
r
.
UserAgent
(),
duration
.
Seconds
(),
"remoteAddr"
:
r
.
RemoteAddr
,
)
"method"
:
r
.
Method
,
"uri"
:
ScrubURLParams
(
r
.
RequestURI
),
"proto"
:
r
.
Proto
,
"status"
:
l
.
status
,
"written"
:
l
.
written
,
"referer"
:
ScrubURLParams
(
r
.
Referer
()),
"userAgent"
:
r
.
UserAgent
(),
"duration"
:
duration
.
Seconds
(),
}
}
func
(
l
*
statsCollectingResponseWriter
)
RequestFinished
(
r
*
http
.
Request
)
{
l
.
writeAccessLog
(
r
)
sessionsActive
.
Dec
()
sessionsActive
.
Dec
()
requestsTotal
.
WithLabelValues
(
strconv
.
Itoa
(
l
.
status
),
r
.
Method
)
.
Inc
()
requestsTotal
.
WithLabelValues
(
strconv
.
Itoa
(
l
.
status
),
r
.
Method
)
.
Inc
()
...
...
internal/helper/logging_test.go
0 → 100644
View file @
07f5e048
package
helper
import
(
"fmt"
"net/http"
"strings"
"testing"
"time"
"github.com/stretchr/testify/assert"
)
func
Test_statsCollectingResponseWriter_accessLogFields
(
t
*
testing
.
T
)
{
passwords
:=
[]
string
{
"should_be_filtered"
,
// Basic case
"should+++filtered"
,
// Password contains +
"this/is/a/to--ken"
,
// Password contains on / and -
"%E9%A9%AE%E9%A9%AC"
,
// Password contains URI Encoded chars
"should_be_%252E%252E%252F"
,
// Password is mixed
}
queryStrings
:=
[]
string
{
"private-token=%s"
,
"authenticity-token=%s"
,
"rss-token=%s"
,
"private_token=%s"
,
"authenticity_token=%s"
,
"rss-token=%s"
,
"private-token=%s&authenticity-token=%s"
,
"private_token=%s&authenticity_token=%s"
,
"param=not_private&private-token=%s"
,
// Non-private fields prefixed
"private-token=%s¶m=not_private"
,
// Non-private fields suffixed
}
paths
:=
[]
string
{
""
,
"/"
,
"/groups/private_token/"
,
}
for
i
,
password
:=
range
passwords
{
for
j
,
path
:=
range
paths
{
for
k
,
qs
:=
range
queryStrings
{
queryString
:=
strings
.
Replace
(
qs
,
"%s"
,
password
,
-
1
)
t
.
Run
(
fmt
.
Sprintf
(
"Test #%v %v %v"
,
i
,
j
,
k
),
func
(
t
*
testing
.
T
)
{
resource
:=
path
+
"?"
+
queryString
// Ensure the Referer is scrubbed too
req
,
err
:=
http
.
NewRequest
(
"GET"
,
"/blah"
+
resource
,
nil
)
assert
.
NoError
(
t
,
err
,
"GET %q: %v"
,
resource
,
err
)
req
.
Header
.
Set
(
"Referer"
,
"http://referer.example.com"
+
resource
)
req
.
RequestURI
=
resource
l
:=
&
statsCollectingResponseWriter
{
rw
:
nil
,
status
:
200
,
wroteHeader
:
true
,
written
:
50
,
started
:
time
.
Now
(),
}
fields
:=
l
.
accessLogFields
(
req
)
uri
:=
fields
[
"uri"
]
.
(
string
)
assert
.
NotEmpty
(
t
,
uri
,
"uri is empty"
)
assert
.
Contains
(
t
,
uri
,
path
,
"GET %q: path not logged"
,
resource
)
assert
.
NotContains
(
t
,
uri
,
password
,
"GET %q: log not filtered correctly"
,
resource
)
referer
:=
fields
[
"referer"
]
.
(
string
)
assert
.
NotEmpty
(
t
,
referer
,
"referer is empty"
)
assert
.
Contains
(
t
,
referer
,
path
,
"GET %q: path not logged"
,
resource
)
assert
.
NotContains
(
t
,
referer
,
password
,
"GET %q: log not filtered correctly"
,
resource
)
assert
.
NotContains
(
t
,
fmt
.
Sprintf
(
"%#v"
,
fields
),
password
,
"password leaked into fields"
,
fields
)
if
strings
.
Contains
(
queryString
,
"param=not_private"
)
{
assert
.
Contains
(
t
,
uri
,
"param=not_private"
,
"Missing non-private parameters in uri"
,
uri
)
assert
.
Contains
(
t
,
referer
,
"param=not_private"
,
"Missing non-private parameters in referer"
,
referer
)
}
})
}
}
}
}
internal/redis/keywatcher.go
View file @
07f5e048
...
@@ -2,11 +2,12 @@ package redis
...
@@ -2,11 +2,12 @@ package redis
import
(
import
(
"fmt"
"fmt"
"log"
"strings"
"strings"
"sync"
"sync"
"time"
"time"
log
"github.com/sirupsen/logrus"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
"github.com/garyburd/redigo/redis"
"github.com/garyburd/redigo/redis"
...
...
internal/redis/redis.go
View file @
07f5e048
...
@@ -3,11 +3,12 @@ package redis
...
@@ -3,11 +3,12 @@ package redis
import
(
import
(
"errors"
"errors"
"fmt"
"fmt"
"log"
"net"
"net"
"net/url"
"net/url"
"time"
"time"
log
"github.com/sirupsen/logrus"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/config"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/config"
sentinel
"github.com/FZambia/go-sentinel"
sentinel
"github.com/FZambia/go-sentinel"
...
@@ -74,7 +75,9 @@ func sentinelConn(master string, urls []config.TomlURL) *sentinel.Sentinel {
...
@@ -74,7 +75,9 @@ func sentinelConn(master string, urls []config.TomlURL) *sentinel.Sentinel {
var
addrs
[]
string
var
addrs
[]
string
for
_
,
url
:=
range
urls
{
for
_
,
url
:=
range
urls
{
h
:=
url
.
URL
.
Host
h
:=
url
.
URL
.
Host
log
.
Printf
(
"redis: using sentinel %q"
,
h
)
log
.
WithFields
(
log
.
Fields
{
"host"
:
h
,
})
.
Printf
(
"redis: using sentinel"
)
addrs
=
append
(
addrs
,
h
)
addrs
=
append
(
addrs
,
h
)
}
}
return
&
sentinel
.
Sentinel
{
return
&
sentinel
.
Sentinel
{
...
@@ -180,7 +183,11 @@ func defaultDialer(dopts []redis.DialOption, keepAlivePeriod time.Duration, url
...
@@ -180,7 +183,11 @@ func defaultDialer(dopts []redis.DialOption, keepAlivePeriod time.Duration, url
}
}
func
redisDial
(
network
,
address
string
,
options
...
redis
.
DialOption
)
(
redis
.
Conn
,
error
)
{
func
redisDial
(
network
,
address
string
,
options
...
redis
.
DialOption
)
(
redis
.
Conn
,
error
)
{
log
.
Printf
(
"redis: dialing %q, %q"
,
network
,
address
)
log
.
WithFields
(
log
.
Fields
{
"network"
:
network
,
"address"
:
address
,
})
.
Printf
(
"redis: dialing"
)
return
redis
.
Dial
(
network
,
address
,
options
...
)
return
redis
.
Dial
(
network
,
address
,
options
...
)
}
}
...
...
internal/sendfile/sendfile.go
View file @
07f5e048
...
@@ -7,10 +7,11 @@ via the X-Sendfile mechanism. All that is needed in the Rails code is the
...
@@ -7,10 +7,11 @@ via the X-Sendfile mechanism. All that is needed in the Rails code is the
package
sendfile
package
sendfile
import
(
import
(
"log"
"net/http"
"net/http"
"regexp"
"regexp"
log
"github.com/sirupsen/logrus"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
...
@@ -104,7 +105,12 @@ func (s *sendFileResponseWriter) WriteHeader(status int) {
...
@@ -104,7 +105,12 @@ func (s *sendFileResponseWriter) WriteHeader(status int) {
}
}
func
sendFileFromDisk
(
w
http
.
ResponseWriter
,
r
*
http
.
Request
,
file
string
)
{
func
sendFileFromDisk
(
w
http
.
ResponseWriter
,
r
*
http
.
Request
,
file
string
)
{
log
.
Printf
(
"Send file %q for %s %q"
,
file
,
r
.
Method
,
helper
.
ScrubURLParams
(
r
.
RequestURI
))
log
.
WithFields
(
log
.
Fields
{
"file"
:
file
,
"method"
:
r
.
Method
,
"uri"
:
helper
.
ScrubURLParams
(
r
.
RequestURI
),
})
.
Print
(
"Send file"
)
content
,
fi
,
err
:=
helper
.
OpenFile
(
file
)
content
,
fi
,
err
:=
helper
.
OpenFile
(
file
)
if
err
!=
nil
{
if
err
!=
nil
{
http
.
NotFound
(
w
,
r
)
http
.
NotFound
(
w
,
r
)
...
...
internal/sendurl/sendurl.go
View file @
07f5e048
...
@@ -3,11 +3,12 @@ package sendurl
...
@@ -3,11 +3,12 @@ package sendurl
import
(
import
(
"fmt"
"fmt"
"io"
"io"
"log"
"net"
"net"
"net/http"
"net/http"
"time"
"time"
log
"github.com/sirupsen/logrus"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
...
@@ -97,7 +98,10 @@ func (e *entry) Inject(w http.ResponseWriter, r *http.Request, sendData string)
...
@@ -97,7 +98,10 @@ func (e *entry) Inject(w http.ResponseWriter, r *http.Request, sendData string)
return
return
}
}
log
.
Printf
(
"SendURL: sending %q for %q"
,
helper
.
ScrubURLParams
(
params
.
URL
),
r
.
URL
.
Path
)
log
.
WithFields
(
log
.
Fields
{
"url"
:
helper
.
ScrubURLParams
(
params
.
URL
),
"path"
:
r
.
URL
.
Path
,
})
.
Print
(
"SendURL: sending"
)
if
params
.
URL
==
""
{
if
params
.
URL
==
""
{
sendURLRequestsInvalidData
.
Inc
()
sendURLRequestsInvalidData
.
Inc
()
...
...
internal/staticpages/servefile.go
View file @
07f5e048
package
staticpages
package
staticpages
import
(
import
(
"log"
"net/http"
"net/http"
"os"
"os"
"path/filepath"
"path/filepath"
"strings"
"strings"
"time"
"time"
log
"github.com/sirupsen/logrus"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/urlprefix"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/urlprefix"
)
)
...
@@ -70,7 +71,12 @@ func (s *Static) ServeExisting(prefix urlprefix.Prefix, cache CacheMode, notFoun
...
@@ -70,7 +71,12 @@ func (s *Static) ServeExisting(prefix urlprefix.Prefix, cache CacheMode, notFoun
w
.
Header
()
.
Set
(
"Expires"
,
cacheUntil
)
w
.
Header
()
.
Set
(
"Expires"
,
cacheUntil
)
}
}
log
.
Printf
(
"Send static file %q (%q) for %s %q"
,
file
,
w
.
Header
()
.
Get
(
"Content-Encoding"
),
r
.
Method
,
helper
.
ScrubURLParams
(
r
.
RequestURI
))
log
.
WithFields
(
log
.
Fields
{
"file"
:
file
,
"encoding"
:
w
.
Header
()
.
Get
(
"Content-Encoding"
),
"method"
:
r
.
Method
,
"uri"
:
helper
.
ScrubURLParams
(
r
.
RequestURI
),
})
.
Printf
(
"Send static file"
)
http
.
ServeContent
(
w
,
r
,
filepath
.
Base
(
file
),
fi
.
ModTime
(),
content
)
http
.
ServeContent
(
w
,
r
,
filepath
.
Base
(
file
),
fi
.
ModTime
(),
content
)
})
})
}
}
internal/terminal/terminal.go
View file @
07f5e048
...
@@ -3,10 +3,11 @@ package terminal
...
@@ -3,10 +3,11 @@ package terminal
import
(
import
(
"errors"
"errors"
"fmt"
"fmt"
"log"
"net/http"
"net/http"
"time"
"time"
log
"github.com/sirupsen/logrus"
"github.com/gorilla/websocket"
"github.com/gorilla/websocket"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/api"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/api"
...
@@ -46,7 +47,7 @@ func ProxyTerminal(w http.ResponseWriter, r *http.Request, terminal *api.Termina
...
@@ -46,7 +47,7 @@ func ProxyTerminal(w http.ResponseWriter, r *http.Request, terminal *api.Termina
server
,
err
:=
connectToServer
(
terminal
,
r
)
server
,
err
:=
connectToServer
(
terminal
,
r
)
if
err
!=
nil
{
if
err
!=
nil
{
helper
.
Fail500
(
w
,
r
,
err
)
helper
.
Fail500
(
w
,
r
,
err
)
log
.
Printf
(
"Terminal: connecting to server failed: %s"
,
err
)
log
.
WithError
(
err
)
.
Print
(
"Terminal: connecting to server failed"
)
return
return
}
}
defer
server
.
UnderlyingConn
()
.
Close
()
defer
server
.
UnderlyingConn
()
.
Close
()
...
@@ -54,7 +55,7 @@ func ProxyTerminal(w http.ResponseWriter, r *http.Request, terminal *api.Termina
...
@@ -54,7 +55,7 @@ func ProxyTerminal(w http.ResponseWriter, r *http.Request, terminal *api.Termina
client
,
err
:=
upgradeClient
(
w
,
r
)
client
,
err
:=
upgradeClient
(
w
,
r
)
if
err
!=
nil
{
if
err
!=
nil
{
log
.
Printf
(
"Terminal: upgrading client to websocket failed: %s"
,
err
)
log
.
WithError
(
err
)
.
Print
(
"Terminal: upgrading client to websocket failed"
)
return
return
}
}
...
@@ -65,11 +66,17 @@ func ProxyTerminal(w http.ResponseWriter, r *http.Request, terminal *api.Termina
...
@@ -65,11 +66,17 @@ func ProxyTerminal(w http.ResponseWriter, r *http.Request, terminal *api.Termina
defer
client
.
UnderlyingConn
()
.
Close
()
defer
client
.
UnderlyingConn
()
.
Close
()
clientAddr
:=
getClientAddr
(
r
)
// We can't know the port with confidence
clientAddr
:=
getClientAddr
(
r
)
// We can't know the port with confidence
log
.
Printf
(
"Terminal: started proxying from %s to %s"
,
clientAddr
,
serverAddr
)
logEntry
:=
log
.
WithFields
(
log
.
Fields
{
defer
log
.
Printf
(
"Terminal: finished proxying from %s to %s"
,
clientAddr
,
serverAddr
)
"clientAddr"
:
clientAddr
,
"serverAddr"
:
serverAddr
,
})
logEntry
.
Print
(
"Terminal: started proxying"
)
defer
logEntry
.
Print
(
"Terminal: finished proxying"
)
if
err
:=
proxy
.
Serve
(
server
,
client
,
serverAddr
,
clientAddr
);
err
!=
nil
{
if
err
:=
proxy
.
Serve
(
server
,
client
,
serverAddr
,
clientAddr
);
err
!=
nil
{
log
.
Printf
(
"Terminal: error proxying from %s to %s: %s"
,
clientAddr
,
serverAddr
,
err
)
log
Entry
.
WithError
(
err
)
.
Print
(
"Terminal: error proxying"
)
}
}
}
}
...
...
internal/testhelper/gitaly.go
View file @
07f5e048
...
@@ -4,11 +4,12 @@ import (
...
@@ -4,11 +4,12 @@ import (
"fmt"
"fmt"
"io"
"io"
"io/ioutil"
"io/ioutil"
"log"
"path"
"path"
"strings"
"strings"
"sync"
"sync"
log
"github.com/sirupsen/logrus"
pb
"gitlab.com/gitlab-org/gitaly-proto/go"
pb
"gitlab.com/gitlab-org/gitaly-proto/go"
"golang.org/x/net/context"
"golang.org/x/net/context"
...
...
internal/testhelper/testhelper.go
View file @
07f5e048
...
@@ -6,7 +6,6 @@ import (
...
@@ -6,7 +6,6 @@ import (
"errors"
"errors"
"fmt"
"fmt"
"io/ioutil"
"io/ioutil"
"log"
"net/http"
"net/http"
"net/http/httptest"
"net/http/httptest"
"os"
"os"
...
@@ -17,6 +16,8 @@ import (
...
@@ -17,6 +16,8 @@ import (
"strings"
"strings"
"testing"
"testing"
log
"github.com/sirupsen/logrus"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/secret"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/secret"
)
)
...
...
internal/upstream/upstream.go
View file @
07f5e048
...
@@ -54,8 +54,8 @@ func (u *Upstream) configureURLPrefix() {
...
@@ -54,8 +54,8 @@ func (u *Upstream) configureURLPrefix() {
}
}
func
(
u
*
Upstream
)
ServeHTTP
(
ow
http
.
ResponseWriter
,
r
*
http
.
Request
)
{
func
(
u
*
Upstream
)
ServeHTTP
(
ow
http
.
ResponseWriter
,
r
*
http
.
Request
)
{
w
:=
helper
.
New
Logg
ingResponseWriter
(
ow
)
w
:=
helper
.
New
StatsCollect
ingResponseWriter
(
ow
)
defer
w
.
Log
(
r
)
defer
w
.
RequestFinished
(
r
)
helper
.
DisableResponseBuffering
(
w
)
helper
.
DisableResponseBuffering
(
w
)
...
...
logging.go
View file @
07f5e048
package
main
package
main
import
(
import
(
"log"
"io"
"io/ioutil"
goLog
"log"
"os"
"os"
"os/signal"
"os/signal"
"syscall"
"syscall"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
"github.com/client9/reopen"
"github.com/client9/reopen"
log
"github.com/sirupsen/logrus"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
)
)
func
reopenLogWriter
(
l
reopen
.
WriteCloser
,
sighup
chan
os
.
Signal
)
{
func
reopenLogWriter
(
l
reopen
.
WriteCloser
,
sighup
chan
os
.
Signal
)
{
for
_
=
range
sighup
{
for
_
=
range
sighup
{
log
.
Print
f
(
"Reopening log file"
)
log
.
Print
(
"Reopening log file"
)
l
.
Reopen
()
l
.
Reopen
()
}
}
}
}
func
startLogging
(
logFile
string
)
{
func
prepareLoggingFile
(
logFile
string
)
*
reopen
.
FileWriter
{
var
logWriter
=
reopen
.
Stderr
if
logFile
!=
""
{
file
,
err
:=
reopen
.
NewFileWriter
(
logFile
)
file
,
err
:=
reopen
.
NewFileWriter
(
logFile
)
if
err
!=
nil
{
if
err
!=
nil
{
log
.
Fatalf
(
"Unable to set output log: %s"
,
err
)
goLog
.
Fatalf
(
"Unable to set output log: %s"
,
err
)
}
logWriter
=
file
}
}
log
.
SetOutput
(
logWriter
)
helper
.
SetCustomResponseLogger
(
logWriter
)
sighup
:=
make
(
chan
os
.
Signal
,
1
)
sighup
:=
make
(
chan
os
.
Signal
,
1
)
signal
.
Notify
(
sighup
,
syscall
.
SIGHUP
)
signal
.
Notify
(
sighup
,
syscall
.
SIGHUP
)
go
reopenLogWriter
(
logWriter
,
sighup
)
go
reopenLogWriter
(
file
,
sighup
)
return
file
}
const
(
jsonLogFormat
=
"json"
textLogFormat
=
"text"
noneLogType
=
"none"
)
type
logConfiguration
struct
{
logFile
string
logFormat
string
}
func
startLogging
(
config
logConfiguration
)
{
var
accessLogEntry
*
log
.
Entry
var
logOutputWriter
io
.
Writer
if
config
.
logFile
!=
""
{
logOutputWriter
=
prepareLoggingFile
(
config
.
logFile
)
}
else
{
logOutputWriter
=
os
.
Stderr
}
switch
config
.
logFormat
{
case
noneLogType
:
accessLogEntry
=
nil
logOutputWriter
=
ioutil
.
Discard
case
jsonLogFormat
:
accessLogEntry
=
log
.
WithField
(
"system"
,
"http"
)
log
.
SetFormatter
(
&
log
.
JSONFormatter
{})
case
textLogFormat
:
accessLogger
:=
log
.
New
()
accessLogger
.
Formatter
=
helper
.
NewAccessLogFormatter
()
accessLogger
.
Out
=
logOutputWriter
accessLogger
.
SetLevel
(
log
.
InfoLevel
)
accessLogEntry
=
accessLogger
.
WithField
(
"system"
,
"http"
)
log
.
SetFormatter
(
&
log
.
TextFormatter
{})
default
:
log
.
WithField
(
"logFormat"
,
config
.
logFormat
)
.
Fatal
(
"Unknown logFormat configured"
)
}
helper
.
SetAccessLoggerEntry
(
accessLogEntry
)
log
.
SetOutput
(
logOutputWriter
)
// Golog always goes to stderr
goLog
.
SetOutput
(
os
.
Stderr
)
}
}
main.go
View file @
07f5e048
...
@@ -16,7 +16,6 @@ package main
...
@@ -16,7 +16,6 @@ package main
import
(
import
(
"flag"
"flag"
"fmt"
"fmt"
"log"
"net"
"net"
"net/http"
"net/http"
_
"net/http/pprof"
_
"net/http/pprof"
...
@@ -24,6 +23,8 @@ import (
...
@@ -24,6 +23,8 @@ import (
"syscall"
"syscall"
"time"
"time"
log
"github.com/sirupsen/logrus"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/config"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/config"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/queueing"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/queueing"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/redis"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/redis"
...
@@ -52,9 +53,16 @@ var apiLimit = flag.Uint("apiLimit", 0, "Number of API requests allowed at singl
...
@@ -52,9 +53,16 @@ var apiLimit = flag.Uint("apiLimit", 0, "Number of API requests allowed at singl
var
apiQueueLimit
=
flag
.
Uint
(
"apiQueueLimit"
,
0
,
"Number of API requests allowed to be queued"
)
var
apiQueueLimit
=
flag
.
Uint
(
"apiQueueLimit"
,
0
,
"Number of API requests allowed to be queued"
)
var
apiQueueTimeout
=
flag
.
Duration
(
"apiQueueDuration"
,
queueing
.
DefaultTimeout
,
"Maximum queueing duration of requests"
)
var
apiQueueTimeout
=
flag
.
Duration
(
"apiQueueDuration"
,
queueing
.
DefaultTimeout
,
"Maximum queueing duration of requests"
)
var
apiCiLongPollingDuration
=
flag
.
Duration
(
"apiCiLongPollingDuration"
,
50
,
"Long polling duration for job requesting for runners (default 50s - enabled)"
)
var
apiCiLongPollingDuration
=
flag
.
Duration
(
"apiCiLongPollingDuration"
,
50
,
"Long polling duration for job requesting for runners (default 50s - enabled)"
)
var
logFile
=
flag
.
String
(
"logFile"
,
""
,
"Log file to be used"
)
var
prometheusListenAddr
=
flag
.
String
(
"prometheusListenAddr"
,
""
,
"Prometheus listening address, e.g. 'localhost:9229'"
)
var
prometheusListenAddr
=
flag
.
String
(
"prometheusListenAddr"
,
""
,
"Prometheus listening address, e.g. 'localhost:9229'"
)
var
logConfig
=
logConfiguration
{}
func
init
()
{
flag
.
StringVar
(
&
logConfig
.
logFile
,
"logFile"
,
""
,
"Log file location"
)
flag
.
StringVar
(
&
logConfig
.
logFormat
,
"logFormat"
,
"text"
,
"Log format to use defaults to text (text, json, none)"
)
}
func
main
()
{
func
main
()
{
flag
.
Usage
=
func
()
{
flag
.
Usage
=
func
()
{
fmt
.
Fprintf
(
os
.
Stderr
,
"Usage of %s:
\n
"
,
os
.
Args
[
0
])
fmt
.
Fprintf
(
os
.
Stderr
,
"Usage of %s:
\n
"
,
os
.
Args
[
0
])
...
@@ -69,14 +77,14 @@ func main() {
...
@@ -69,14 +77,14 @@ func main() {
os
.
Exit
(
0
)
os
.
Exit
(
0
)
}
}
startLogging
(
*
logFile
)
startLogging
(
logConfig
)
backendURL
,
err
:=
parseAuthBackend
(
*
authBackend
)
backendURL
,
err
:=
parseAuthBackend
(
*
authBackend
)
if
err
!=
nil
{
if
err
!=
nil
{
log
.
Fatalf
(
"invalid authBackend: %v"
,
err
)
log
.
WithError
(
err
)
.
Fatal
(
"invalid authBackend"
)
}
}
log
.
Printf
(
"Starting %s"
,
version
)
log
.
WithField
(
"version"
,
version
)
.
Print
(
"Starting"
)
// Good housekeeping for Unix sockets: unlink before binding
// Good housekeeping for Unix sockets: unlink before binding
if
*
listenNetwork
==
"unix"
{
if
*
listenNetwork
==
"unix"
{
...
@@ -128,7 +136,7 @@ func main() {
...
@@ -128,7 +136,7 @@ func main() {
if
*
configFile
!=
""
{
if
*
configFile
!=
""
{
cfgFromFile
,
err
:=
config
.
LoadConfig
(
*
configFile
)
cfgFromFile
,
err
:=
config
.
LoadConfig
(
*
configFile
)
if
err
!=
nil
{
if
err
!=
nil
{
log
.
Fatalf
(
"Can not load config file %q: %v"
,
*
configFile
,
err
)
log
.
WithField
(
"configFile"
,
*
configFile
)
.
WithError
(
err
)
.
Fatal
(
"Can not load config file"
)
}
}
cfg
.
Redis
=
cfgFromFile
.
Redis
cfg
.
Redis
=
cfgFromFile
.
Redis
...
...
main_test.go
View file @
07f5e048
...
@@ -7,7 +7,6 @@ import (
...
@@ -7,7 +7,6 @@ import (
"encoding/json"
"encoding/json"
"fmt"
"fmt"
"io/ioutil"
"io/ioutil"
"log"
"net/http"
"net/http"
"net/http/httptest"
"net/http/httptest"
"os"
"os"
...
@@ -18,6 +17,8 @@ import (
...
@@ -18,6 +17,8 @@ import (
"testing"
"testing"
"time"
"time"
log
"github.com/sirupsen/logrus"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/api"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/api"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/config"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/config"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/git"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/git"
...
@@ -52,14 +53,14 @@ func TestMain(m *testing.M) {
...
@@ -52,14 +53,14 @@ func TestMain(m *testing.M) {
testCmd
.
Stderr
=
os
.
Stderr
testCmd
.
Stderr
=
os
.
Stderr
if
err
:=
testCmd
.
Run
();
err
!=
nil
{
if
err
:=
testCmd
.
Run
();
err
!=
nil
{
log
.
Printf
(
"Test setup: failed to run %v"
,
testCmd
)
log
.
WithField
(
"testCmd"
,
testCmd
)
.
Print
(
"Test setup: failed to run"
)
os
.
Exit
(
-
1
)
os
.
Exit
(
-
1
)
}
}
}
}
cleanup
,
err
:=
testhelper
.
BuildExecutables
()
cleanup
,
err
:=
testhelper
.
BuildExecutables
()
if
err
!=
nil
{
if
err
!=
nil
{
log
.
Printf
(
"Test setup: failed to build executables: %v"
,
err
)
log
.
WithError
(
err
)
.
Print
(
"Test setup: failed to build executables"
)
os
.
Exit
(
1
)
os
.
Exit
(
1
)
}
}
...
@@ -572,67 +573,6 @@ func TestAPIFalsePositivesAreProxied(t *testing.T) {
...
@@ -572,67 +573,6 @@ func TestAPIFalsePositivesAreProxied(t *testing.T) {
}
}
}
}
func
TestQueryStringLogFiltering
(
t
*
testing
.
T
)
{
// capture the log output
buf
:=
bytes
.
NewBuffer
(
nil
)
log
.
SetOutput
(
buf
)
helper
.
SetCustomResponseLogger
(
buf
)
defer
log
.
SetOutput
(
os
.
Stderr
)
defer
helper
.
SetCustomResponseLogger
(
os
.
Stderr
)
ts
:=
testhelper
.
TestServerWithHandler
(
regexp
.
MustCompile
(
`.`
),
func
(
w
http
.
ResponseWriter
,
_
*
http
.
Request
)
{
w
.
WriteHeader
(
200
)
})
defer
ts
.
Close
()
ws
:=
startWorkhorseServer
(
ts
.
URL
)
defer
ws
.
Close
()
for
_
,
path
:=
range
[]
string
{
""
,
"/"
,
}
{
for
_
,
queryString
:=
range
[]
string
{
"private-token=should_be_filtered"
,
"authenticity-token=should_be_filtered"
,
"rss-token=should_be_filtered"
,
"private_token=should_be_filtered"
,
"authenticity_token=should_be_filtered"
,
"rss-token=should_be_filtered"
,
"private-token=should_be_filtered&authenticity-token=should_be_filtered"
,
"private_token=should_be_filtered&authenticity_token=should_be_filtered"
,
}
{
resource
:=
path
+
"?"
+
queryString
// Ensure the Referer is scrubbed too
req
,
err
:=
http
.
NewRequest
(
"GET"
,
ws
.
URL
+
resource
,
nil
)
if
!
assert
.
NoError
(
t
,
err
,
"GET %q: %v"
,
resource
,
err
)
{
continue
}
req
.
Header
.
Set
(
"Referer"
,
"http://referer.example.com"
+
resource
)
resp
,
err
:=
http
.
DefaultClient
.
Do
(
req
)
if
!
assert
.
NoError
(
t
,
err
,
"GET %q: %v"
,
resource
,
err
)
{
continue
}
resp
.
Body
.
Close
()
assert
.
Equal
(
t
,
200
,
resp
.
StatusCode
,
"GET %q: status code"
,
resource
)
logged
:=
buf
.
String
()
buf
.
Reset
()
assert
.
NotEqual
(
t
,
0
,
len
(
logged
),
"GET %q: log is empty"
,
resource
)
assert
.
Contains
(
t
,
logged
,
path
,
"GET %q: path not logged"
,
resource
)
assert
.
Contains
(
t
,
logged
,
"referer"
,
"GET %q: referer not logged"
,
resource
)
assert
.
NotContains
(
t
,
logged
,
"should_be_filtered"
,
"GET %q: log not filtered correctly"
,
resource
)
}
}
}
func
setupStaticFile
(
fpath
,
content
string
)
error
{
func
setupStaticFile
(
fpath
,
content
string
)
error
{
cwd
,
err
:=
os
.
Getwd
()
cwd
,
err
:=
os
.
Getwd
()
if
err
!=
nil
{
if
err
!=
nil
{
...
...
sendfile_test.go
View file @
07f5e048
...
@@ -4,13 +4,14 @@ import (
...
@@ -4,13 +4,14 @@ import (
"bytes"
"bytes"
"fmt"
"fmt"
"io/ioutil"
"io/ioutil"
"log"
"net/http"
"net/http"
"net/http/httptest"
"net/http/httptest"
"os"
"os"
"os/exec"
"os/exec"
"path"
"path"
"testing"
"testing"
log
"github.com/sirupsen/logrus"
)
)
func
TestDeniedLfsDownload
(
t
*
testing
.
T
)
{
func
TestDeniedLfsDownload
(
t
*
testing
.
T
)
{
...
...
terminal_test.go
View file @
07f5e048
...
@@ -4,7 +4,6 @@ import (
...
@@ -4,7 +4,6 @@ import (
"bytes"
"bytes"
"encoding/pem"
"encoding/pem"
"fmt"
"fmt"
"log"
"net"
"net"
"net/http"
"net/http"
"net/http/httptest"
"net/http/httptest"
...
@@ -14,6 +13,8 @@ import (
...
@@ -14,6 +13,8 @@ import (
"testing"
"testing"
"time"
"time"
log
"github.com/sirupsen/logrus"
"github.com/gorilla/websocket"
"github.com/gorilla/websocket"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/api"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/api"
...
@@ -67,7 +68,7 @@ func TestTerminalBadTLS(t *testing.T) {
...
@@ -67,7 +68,7 @@ func TestTerminalBadTLS(t *testing.T) {
t
.
Fatalf
(
"Expected connection to fail ErrBadHandshake, got: %v"
,
err
)
t
.
Fatalf
(
"Expected connection to fail ErrBadHandshake, got: %v"
,
err
)
}
}
if
err
==
nil
{
if
err
==
nil
{
log
.
Println
(
"TLS negotiation should have failed!"
)
log
.
Info
(
"TLS negotiation should have failed!"
)
defer
client
.
Close
()
defer
client
.
Close
()
}
}
}
}
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment