Commit 078770a5 authored by Matthew Holt's avatar Matthew Holt

telemetry: Record TLS ClientHellos by hash of key of structured data

Also improve handling of disabled metrics, and record TLS ClientHello
in association with User-Agent
parent 518edd3c
......@@ -91,7 +91,10 @@ func Run() {
// initialize telemetry client
if enableTelemetry {
initTelemetry()
err := initTelemetry()
if err != nil {
mustLogFatalf("[ERROR] Initializing telemetry: %v", err)
}
} else if disabledMetrics != "" {
mustLogFatalf("[ERROR] Cannot disable specific metrics because telemetry is disabled")
}
......@@ -293,7 +296,7 @@ func setCPU(cpu string) error {
}
// initTelemetry initializes the telemetry engine.
func initTelemetry() {
func initTelemetry() error {
uuidFilename := filepath.Join(caddy.AssetsPath(), "uuid")
newUUID := func() uuid.UUID {
......@@ -329,7 +332,34 @@ func initTelemetry() {
}
}
telemetry.Init(id, strings.Split(disabledMetrics, ","))
// parse and check the list of disabled metrics
var disabledMetricsSlice []string
if len(disabledMetrics) > 0 {
if len(disabledMetrics) > 1024 {
// mitigate disk space exhaustion at the collection endpoint
return fmt.Errorf("too many metrics to disable")
}
disabledMetricsSlice = strings.Split(disabledMetrics, ",")
for i, metric := range disabledMetricsSlice {
if metric == "instance_id" || metric == "timestamp" || metric == "disabled_metrics" {
return fmt.Errorf("instance_id, timestamp, and disabled_metrics cannot be disabled")
}
if metric == "" {
disabledMetricsSlice = append(disabledMetricsSlice[:i], disabledMetricsSlice[i+1:]...)
}
}
}
// initialize telemetry
telemetry.Init(id, disabledMetricsSlice)
// if any metrics were disabled, report it
if len(disabledMetricsSlice) > 0 {
telemetry.Set("disabled_metrics", disabledMetricsSlice)
log.Printf("[NOTICE] The following telemetry metrics are disabled: %s", disabledMetrics)
}
return nil
}
const appName = "Caddy"
......
This diff is collapsed.
......@@ -67,6 +67,12 @@ func init() {
caddy.RegisterParsingCallback(serverType, "root", hideCaddyfile)
caddy.RegisterParsingCallback(serverType, "tls", activateHTTPS)
caddytls.RegisterConfigGetter(serverType, func(c *caddy.Controller) *caddytls.Config { return GetConfig(c).TLS })
// disable the caddytls package reporting ClientHellos
// to telemetry, since our MITM detector does this but
// with more information than the standard lib provides
// (as of May 2018)
caddytls.ClientHelloTelemetry = false
}
// hideCaddyfile hides the source/origin Caddyfile if it is within the
......
......@@ -349,8 +349,12 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
}
}()
// TODO: Somehow report UA string in conjunction with TLS handshake, if any (and just once per connection)
go telemetry.AppendUnique("http_user_agent", r.Header.Get("User-Agent"))
// record the User-Agent string (with a cap on its length to mitigate attacks)
ua := r.Header.Get("User-Agent")
if len(ua) > 512 {
ua = ua[:512]
}
go telemetry.AppendUnique("http_user_agent", ua)
go telemetry.Increment("http_request_count")
// copy the original, unchanged URL into the context
......
......@@ -341,7 +341,7 @@ func standaloneTLSTicketKeyRotation(c *tls.Config, ticker *time.Ticker, exitChan
// Do not use this for cryptographic purposes.
func fastHash(input []byte) string {
h := fnv.New32a()
h.Write([]byte(input))
h.Write(input)
return fmt.Sprintf("%x", h.Sum32())
}
......
......@@ -99,25 +99,23 @@ func (cg configGroup) GetConfigForClient(clientHello *tls.ClientHelloInfo) (*tls
//
// This method is safe for use as a tls.Config.GetCertificate callback.
func (cfg *Config) GetCertificate(clientHello *tls.ClientHelloInfo) (*tls.Certificate, error) {
// TODO: We need to collect this in a heavily de-duplicating way
// It would also be nice to associate a handshake with the UA string (but that is only for HTTP server type)
// go telemetry.Append("tls_client_hello", struct {
// NoSNI bool `json:"no_sni,omitempty"`
// CipherSuites []uint16 `json:"cipher_suites,omitempty"`
// SupportedCurves []tls.CurveID `json:"curves,omitempty"`
// SupportedPoints []uint8 `json:"points,omitempty"`
// SignatureSchemes []tls.SignatureScheme `json:"sig_scheme,omitempty"`
// ALPN []string `json:"alpn,omitempty"`
// SupportedVersions []uint16 `json:"versions,omitempty"`
// }{
// NoSNI: clientHello.ServerName == "",
// CipherSuites: clientHello.CipherSuites,
// SupportedCurves: clientHello.SupportedCurves,
// SupportedPoints: clientHello.SupportedPoints,
// SignatureSchemes: clientHello.SignatureSchemes,
// ALPN: clientHello.SupportedProtos,
// SupportedVersions: clientHello.SupportedVersions,
// })
if ClientHelloTelemetry {
// If no other plugin (such as the HTTP server type) is implementing ClientHello telemetry, we do it.
// NOTE: The values in the Go standard lib's ClientHelloInfo aren't guaranteed to be in order.
info := ClientHelloInfo{
Version: clientHello.SupportedVersions[0], // report the highest
CipherSuites: clientHello.CipherSuites,
ExtensionsUnknown: true, // no extension info... :(
CompressionMethodsUnknown: true, // no compression methods... :(
Curves: clientHello.SupportedCurves,
Points: clientHello.SupportedPoints,
// We also have, but do not yet use: SignatureSchemes, ServerName, and SupportedProtos (ALPN)
// because the standard lib parses some extensions, but our MITM detector generally doesn't.
}
go telemetry.SetNested("tls_client_hello", info.Key(), info)
}
// get the certificate and serve it up
cert, err := cfg.getCertDuringHandshake(strings.ToLower(clientHello.ServerName), true, true)
if err == nil {
go telemetry.Increment("tls_handshake_count") // TODO: This is a "best guess" for now, we need something listener-level
......@@ -487,6 +485,42 @@ func (cfg *Config) renewDynamicCertificate(name string, currentCert Certificate)
return cfg.getCertDuringHandshake(name, true, false)
}
// ClientHelloInfo is our own version of the standard lib's
// tls.ClientHelloInfo. As of May 2018, any fields populated
// by the Go standard library are not guaranteed to have their
// values in the original order as on the wire.
type ClientHelloInfo struct {
Version uint16 `json:"version,omitempty"`
CipherSuites []uint16 `json:"cipher_suites,omitempty"`
Extensions []uint16 `json:"extensions,omitempty"`
CompressionMethods []byte `json:"compression,omitempty"`
Curves []tls.CurveID `json:"curves,omitempty"`
Points []uint8 `json:"points,omitempty"`
// Whether a couple of fields are unknown; if not, the key will encode
// differently to reflect that, as opposed to being known empty values.
// (some fields may be unknown depending on what package is being used;
// i.e. the Go standard lib doesn't expose some things)
// (very important to NOT encode these to JSON)
ExtensionsUnknown bool `json:"-"`
CompressionMethodsUnknown bool `json:"-"`
}
// Key returns a standardized string form of the data in info,
// useful for identifying duplicates.
func (info ClientHelloInfo) Key() string {
extensions, compressionMethods := "?", "?"
if !info.ExtensionsUnknown {
extensions = fmt.Sprintf("%x", info.Extensions)
}
if !info.CompressionMethodsUnknown {
compressionMethods = fmt.Sprintf("%x", info.CompressionMethods)
}
return fastHash([]byte(fmt.Sprintf("%x-%x-%s-%s-%x-%x",
info.Version, info.CipherSuites, extensions,
compressionMethods, info.Curves, info.Points)))
}
// obtainCertWaitChans is used to coordinate obtaining certs for each hostname.
var obtainCertWaitChans = make(map[string]chan struct{})
var obtainCertWaitChansMu sync.Mutex
......@@ -501,3 +535,8 @@ var failedIssuanceMu sync.RWMutex
// If this value is recent, do not make any on-demand certificate requests.
var lastIssueTime time.Time
var lastIssueTimeMu sync.Mutex
// ClientHelloTelemetry determines whether to report
// TLS ClientHellos to telemetry. Disable if doing
// it from a different package.
var ClientHelloTelemetry = true
......@@ -16,6 +16,7 @@ package telemetry
import (
"log"
"strings"
"github.com/google/uuid"
)
......@@ -117,17 +118,58 @@ func Set(key string, val interface{}) {
return
}
bufferMu.Lock()
if _, ok := buffer[key]; !ok {
if bufferItemCount >= maxBufferItems {
bufferMu.Unlock()
return
}
if _, ok := buffer[key]; !ok {
bufferItemCount++
}
buffer[key] = val
bufferMu.Unlock()
}
// SetNested puts a value in the buffer to be included
// in the next emission, nested under the top-level key
// as subkey. It overwrites any previous value.
//
// This function is safe for multiple goroutines,
// and it is recommended to call this using the
// go keyword after the call to SendHello so it
// doesn't block crucial code.
func SetNested(key, subkey string, val interface{}) {
if !enabled || isDisabled(key) {
return
}
bufferMu.Lock()
if topLevel, ok1 := buffer[key]; ok1 {
topLevelMap, ok2 := topLevel.(map[string]interface{})
if !ok2 {
bufferMu.Unlock()
log.Printf("[PANIC] Telemetry: key %s is already used for non-nested-map value", key)
return
}
if _, ok3 := topLevelMap[subkey]; !ok3 {
// don't exceed max buffer size
if bufferItemCount >= maxBufferItems {
bufferMu.Unlock()
return
}
bufferItemCount++
}
topLevelMap[subkey] = val
} else {
// don't exceed max buffer size
if bufferItemCount >= maxBufferItems {
bufferMu.Unlock()
return
}
bufferItemCount++
buffer[key] = map[string]interface{}{subkey: val}
}
bufferMu.Unlock()
}
// Append appends value to a list named key.
// If key is new, a new list will be created.
// If key maps to a type that is not a list,
......@@ -161,7 +203,8 @@ func Append(key string, value interface{}) {
// AppendUnique adds value to a set named key.
// Set items are unordered. Values in the set
// are unique, but how many times they are
// appended is counted.
// appended is counted. The value must be
// hashable.
//
// If key is new, a new set will be created for
// values with that key. If key maps to a type
......@@ -238,8 +281,16 @@ func atomicAdd(key string, amount int) {
// functions should call this and not
// save the value if this returns true.
func isDisabled(key string) bool {
// for keys that are augmented with data, such as
// "tls_client_hello_ua:<hash>", just
// check the prefix "tls_client_hello_ua"
checkKey := key
if idx := strings.Index(key, ":"); idx > -1 {
checkKey = key[:idx]
}
disabledMetricsMu.RLock()
_, ok := disabledMetrics[key]
_, ok := disabledMetrics[checkKey]
disabledMetricsMu.RUnlock()
return ok
}
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment