telemetry: Record TLS ClientHellos by hash of key of structured data

Also improve handling of disabled metrics, and record TLS ClientHello
in association with User-Agent

caddy/caddymain/run.go

@@ -91,7 +91,10 @@ func Run() {
 
 	// initialize telemetry client
 	if enableTelemetry {
-		initTelemetry()
+		err := initTelemetry()
+		if err != nil {
+			mustLogFatalf("[ERROR] Initializing telemetry: %v", err)
+		}
 	} else if disabledMetrics != "" {
 		mustLogFatalf("[ERROR] Cannot disable specific metrics because telemetry is disabled")
 	}
@@ -293,7 +296,7 @@ func setCPU(cpu string) error {
 }
 
 // initTelemetry initializes the telemetry engine.
-func initTelemetry() {
+func initTelemetry() error {
 	uuidFilename := filepath.Join(caddy.AssetsPath(), "uuid")
 
 	newUUID := func() uuid.UUID {
@@ -329,7 +332,34 @@ func initTelemetry() {
 		}
 	}
 
-	telemetry.Init(id, strings.Split(disabledMetrics, ","))
+	// parse and check the list of disabled metrics
+	var disabledMetricsSlice []string
+	if len(disabledMetrics) > 0 {
+		if len(disabledMetrics) > 1024 {
+			// mitigate disk space exhaustion at the collection endpoint
+			return fmt.Errorf("too many metrics to disable")
+		}
+		disabledMetricsSlice = strings.Split(disabledMetrics, ",")
+		for i, metric := range disabledMetricsSlice {
+			if metric == "instance_id" || metric == "timestamp" || metric == "disabled_metrics" {
+				return fmt.Errorf("instance_id, timestamp, and disabled_metrics cannot be disabled")
+			}
+			if metric == "" {
+				disabledMetricsSlice = append(disabledMetricsSlice[:i], disabledMetricsSlice[i+1:]...)
+			}
+		}
+	}
+
+	// initialize telemetry
+	telemetry.Init(id, disabledMetricsSlice)
+
+	// if any metrics were disabled, report it
+	if len(disabledMetricsSlice) > 0 {
+		telemetry.Set("disabled_metrics", disabledMetricsSlice)
+		log.Printf("[NOTICE] The following telemetry metrics are disabled: %s", disabledMetrics)
+	}
+
+	return nil
 }
 
 const appName = "Caddy"

caddyhttp/httpserver/plugin.go

@@ -67,6 +67,12 @@ func init() {
 	caddy.RegisterParsingCallback(serverType, "root", hideCaddyfile)
 	caddy.RegisterParsingCallback(serverType, "tls", activateHTTPS)
 	caddytls.RegisterConfigGetter(serverType, func(c *caddy.Controller) *caddytls.Config { return GetConfig(c).TLS })
+
+	// disable the caddytls package reporting ClientHellos
+	// to telemetry, since our MITM detector does this but
+	// with more information than the standard lib provides
+	// (as of May 2018)
+	caddytls.ClientHelloTelemetry = false
 }
 
 // hideCaddyfile hides the source/origin Caddyfile if it is within the

caddyhttp/httpserver/server.go

@@ -349,8 +349,12 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		}
 	}()
 
-	// TODO: Somehow report UA string in conjunction with TLS handshake, if any (and just once per connection)
-	go telemetry.AppendUnique("http_user_agent", r.Header.Get("User-Agent"))
+	// record the User-Agent string (with a cap on its length to mitigate attacks)
+	ua := r.Header.Get("User-Agent")
+	if len(ua) > 512 {
+		ua = ua[:512]
+	}
+	go telemetry.AppendUnique("http_user_agent", ua)
 	go telemetry.Increment("http_request_count")
 
 	// copy the original, unchanged URL into the context

caddytls/crypto.go

@@ -341,7 +341,7 @@ func standaloneTLSTicketKeyRotation(c *tls.Config, ticker *time.Ticker, exitChan
 // Do not use this for cryptographic purposes.
 func fastHash(input []byte) string {
 	h := fnv.New32a()
-	h.Write([]byte(input))
+	h.Write(input)
 	return fmt.Sprintf("%x", h.Sum32())
 }
 

caddytls/handshake.go

@@ -99,25 +99,23 @@ func (cg configGroup) GetConfigForClient(clientHello *tls.ClientHelloInfo) (*tls
 //
 // This method is safe for use as a tls.Config.GetCertificate callback.
 func (cfg *Config) GetCertificate(clientHello *tls.ClientHelloInfo) (*tls.Certificate, error) {
-	// TODO: We need to collect this in a heavily de-duplicating way
-	// It would also be nice to associate a handshake with the UA string (but that is only for HTTP server type)
-	// go telemetry.Append("tls_client_hello", struct {
-	// 	NoSNI             bool                  `json:"no_sni,omitempty"`
-	// 	CipherSuites      []uint16              `json:"cipher_suites,omitempty"`
-	// 	SupportedCurves   []tls.CurveID         `json:"curves,omitempty"`
-	// 	SupportedPoints   []uint8               `json:"points,omitempty"`
-	// 	SignatureSchemes  []tls.SignatureScheme `json:"sig_scheme,omitempty"`
-	// 	ALPN              []string              `json:"alpn,omitempty"`
-	// 	SupportedVersions []uint16              `json:"versions,omitempty"`
-	// }{
-	// 	NoSNI:             clientHello.ServerName == "",
-	// 	CipherSuites:      clientHello.CipherSuites,
-	// 	SupportedCurves:   clientHello.SupportedCurves,
-	// 	SupportedPoints:   clientHello.SupportedPoints,
-	// 	SignatureSchemes:  clientHello.SignatureSchemes,
-	// 	ALPN:              clientHello.SupportedProtos,
-	// 	SupportedVersions: clientHello.SupportedVersions,
-	// })
+	if ClientHelloTelemetry {
+		// If no other plugin (such as the HTTP server type) is implementing ClientHello telemetry, we do it.
+		// NOTE: The values in the Go standard lib's ClientHelloInfo aren't guaranteed to be in order.
+		info := ClientHelloInfo{
+			Version:                   clientHello.SupportedVersions[0], // report the highest
+			CipherSuites:              clientHello.CipherSuites,
+			ExtensionsUnknown:         true, // no extension info... :(
+			CompressionMethodsUnknown: true, // no compression methods... :(
+			Curves: clientHello.SupportedCurves,
+			Points: clientHello.SupportedPoints,
+			// We also have, but do not yet use: SignatureSchemes, ServerName, and SupportedProtos (ALPN)
+			// because the standard lib parses some extensions, but our MITM detector generally doesn't.
+		}
+		go telemetry.SetNested("tls_client_hello", info.Key(), info)
+	}
+
+	// get the certificate and serve it up
 	cert, err := cfg.getCertDuringHandshake(strings.ToLower(clientHello.ServerName), true, true)
 	if err == nil {
 		go telemetry.Increment("tls_handshake_count") // TODO: This is a "best guess" for now, we need something listener-level
@@ -487,6 +485,42 @@ func (cfg *Config) renewDynamicCertificate(name string, currentCert Certificate)
 	return cfg.getCertDuringHandshake(name, true, false)
 }
 
+// ClientHelloInfo is our own version of the standard lib's
+// tls.ClientHelloInfo. As of May 2018, any fields populated
+// by the Go standard library are not guaranteed to have their
+// values in the original order as on the wire.
+type ClientHelloInfo struct {
+	Version            uint16        `json:"version,omitempty"`
+	CipherSuites       []uint16      `json:"cipher_suites,omitempty"`
+	Extensions         []uint16      `json:"extensions,omitempty"`
+	CompressionMethods []byte        `json:"compression,omitempty"`
+	Curves             []tls.CurveID `json:"curves,omitempty"`
+	Points             []uint8       `json:"points,omitempty"`
+
+	// Whether a couple of fields are unknown; if not, the key will encode
+	// differently to reflect that, as opposed to being known empty values.
+	// (some fields may be unknown depending on what package is being used;
+	// i.e. the Go standard lib doesn't expose some things)
+	// (very important to NOT encode these to JSON)
+	ExtensionsUnknown         bool `json:"-"`
+	CompressionMethodsUnknown bool `json:"-"`
+}
+
+// Key returns a standardized string form of the data in info,
+// useful for identifying duplicates.
+func (info ClientHelloInfo) Key() string {
+	extensions, compressionMethods := "?", "?"
+	if !info.ExtensionsUnknown {
+		extensions = fmt.Sprintf("%x", info.Extensions)
+	}
+	if !info.CompressionMethodsUnknown {
+		compressionMethods = fmt.Sprintf("%x", info.CompressionMethods)
+	}
+	return fastHash([]byte(fmt.Sprintf("%x-%x-%s-%s-%x-%x",
+		info.Version, info.CipherSuites, extensions,
+		compressionMethods, info.Curves, info.Points)))
+}
+
 // obtainCertWaitChans is used to coordinate obtaining certs for each hostname.
 var obtainCertWaitChans = make(map[string]chan struct{})
 var obtainCertWaitChansMu sync.Mutex
@@ -501,3 +535,8 @@ var failedIssuanceMu sync.RWMutex
 // If this value is recent, do not make any on-demand certificate requests.
 var lastIssueTime time.Time
 var lastIssueTimeMu sync.Mutex
+
+// ClientHelloTelemetry determines whether to report
+// TLS ClientHellos to telemetry. Disable if doing
+// it from a different package.
+var ClientHelloTelemetry = true

telemetry/collection.go

@@ -16,6 +16,7 @@ package telemetry
 
 import (
 	"log"
+	"strings"
 
 	"github.com/google/uuid"
 )
@@ -117,17 +118,58 @@ func Set(key string, val interface{}) {
 		return
 	}
 	bufferMu.Lock()
-	if bufferItemCount >= maxBufferItems {
-		bufferMu.Unlock()
-		return
-	}
 	if _, ok := buffer[key]; !ok {
+		if bufferItemCount >= maxBufferItems {
+			bufferMu.Unlock()
+			return
+		}
 		bufferItemCount++
 	}
 	buffer[key] = val
 	bufferMu.Unlock()
 }
 
+// SetNested puts a value in the buffer to be included
+// in the next emission, nested under the top-level key
+// as subkey. It overwrites any previous value.
+//
+// This function is safe for multiple goroutines,
+// and it is recommended to call this using the
+// go keyword after the call to SendHello so it
+// doesn't block crucial code.
+func SetNested(key, subkey string, val interface{}) {
+	if !enabled || isDisabled(key) {
+		return
+	}
+	bufferMu.Lock()
+	if topLevel, ok1 := buffer[key]; ok1 {
+		topLevelMap, ok2 := topLevel.(map[string]interface{})
+		if !ok2 {
+			bufferMu.Unlock()
+			log.Printf("[PANIC] Telemetry: key %s is already used for non-nested-map value", key)
+			return
+		}
+		if _, ok3 := topLevelMap[subkey]; !ok3 {
+			// don't exceed max buffer size
+			if bufferItemCount >= maxBufferItems {
+				bufferMu.Unlock()
+				return
+			}
+			bufferItemCount++
+		}
+		topLevelMap[subkey] = val
+	} else {
+		// don't exceed max buffer size
+		if bufferItemCount >= maxBufferItems {
+			bufferMu.Unlock()
+			return
+		}
+		bufferItemCount++
+		buffer[key] = map[string]interface{}{subkey: val}
+	}
+	bufferMu.Unlock()
+}
+
 // Append appends value to a list named key.
 // If key is new, a new list will be created.
 // If key maps to a type that is not a list,
@@ -161,7 +203,8 @@ func Append(key string, value interface{}) {
 // AppendUnique adds value to a set named key.
 // Set items are unordered. Values in the set
 // are unique, but how many times they are
-// appended is counted.
+// appended is counted. The value must be
+// hashable.
 //
 // If key is new, a new set will be created for
 // values with that key. If key maps to a type
@@ -238,8 +281,16 @@ func atomicAdd(key string, amount int) {
 // functions should call this and not
 // save the value if this returns true.
 func isDisabled(key string) bool {
+	// for keys that are augmented with data, such as
+	// "tls_client_hello_ua:<hash>", just
+	// check the prefix "tls_client_hello_ua"
+	checkKey := key
+	if idx := strings.Index(key, ":"); idx > -1 {
+		checkKey = key[:idx]
+	}
+
 	disabledMetricsMu.RLock()
-	_, ok := disabledMetrics[key]
+	_, ok := disabledMetrics[checkKey]
 	disabledMetricsMu.RUnlock()
 	return ok
 }