proxy: Add new fallback_delay sub-directive (#2309)

* Updates the existing proxy and reverse proxy tests to include a new fallback delay value

* Adds a new fallback_delay sub-directive to the proxy directive and uses it in the creation of single host reverse proxies

caddyhttp/proxy/proxy.go

@@ -47,6 +47,12 @@ type Upstream interface {
 	// Checks if subpath is not an ignored path
 	AllowedPath(string) bool
 
+	// Gets the duration of the headstart the first
+	// connection is given in the Go standard library's
+	// implementation of "Happy Eyeballs" when DualStack
+	// is enabled in net.Dialer.
+	GetFallbackDelay() time.Duration
+
 	// Gets how long to try selecting upstream hosts
 	// in the case of cascading failures.
 	GetTryDuration() time.Duration
@@ -195,6 +201,7 @@ func (p Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, error) {
 					host.WithoutPathPrefix,
 					http.DefaultMaxIdleConnsPerHost,
 					upstream.GetTimeout(),
+					upstream.GetFallbackDelay(),
 				)
 			}
 

caddyhttp/proxy/proxy_test.go

@@ -122,7 +122,7 @@ func TestReverseProxy(t *testing.T) {
 	// set up proxy
 	p := &Proxy{
 		Next:      httpserver.EmptyNext, // prevents panic in some cases when test fails
-		Upstreams: []Upstream{newFakeUpstream(backend.URL, false, 30*time.Second)},
+		Upstreams: []Upstream{newFakeUpstream(backend.URL, false, 30*time.Second, 300*time.Millisecond)},
 	}
 
 	// Create the fake request body.
@@ -202,7 +202,7 @@ func TestReverseProxyInsecureSkipVerify(t *testing.T) {
 	// set up proxy
 	p := &Proxy{
 		Next:      httpserver.EmptyNext, // prevents panic in some cases when test fails
-		Upstreams: []Upstream{newFakeUpstream(backend.URL, true, 30*time.Second)},
+		Upstreams: []Upstream{newFakeUpstream(backend.URL, true, 30*time.Second, 300*time.Millisecond)},
 	}
 
 	// create request and response recorder
@@ -289,6 +289,7 @@ func TestReverseProxyMaxConnLimit(t *testing.T) {
 
 func TestReverseProxyTimeout(t *testing.T) {
 	timeout := 2 * time.Second
+	fallbackDelay := 300 * time.Millisecond
 	errorMargin := 100 * time.Millisecond
 	log.SetOutput(ioutil.Discard)
 	defer log.SetOutput(os.Stderr)
@@ -296,7 +297,7 @@ func TestReverseProxyTimeout(t *testing.T) {
 	// set up proxy
 	p := &Proxy{
 		Next:      httpserver.EmptyNext, // prevents panic in some cases when test fails
-		Upstreams: []Upstream{newFakeUpstream("https://8.8.8.8", true, timeout)},
+		Upstreams: []Upstream{newFakeUpstream("https://8.8.8.8", true, timeout, fallbackDelay)},
 	}
 
 	// create request and response recorder
@@ -711,7 +712,7 @@ func TestUpstreamHeadersUpdate(t *testing.T) {
 	}))
 	defer backend.Close()
 
-	upstream := newFakeUpstream(backend.URL, false, 30*time.Second)
+	upstream := newFakeUpstream(backend.URL, false, 30*time.Second, 300*time.Millisecond)
 	upstream.host.UpstreamHeaders = http.Header{
 		"Connection": {"{>Connection}"},
 		"Upgrade":    {"{>Upgrade}"},
@@ -778,7 +779,7 @@ func TestDownstreamHeadersUpdate(t *testing.T) {
 	}))
 	defer backend.Close()
 
-	upstream := newFakeUpstream(backend.URL, false, 30*time.Second)
+	upstream := newFakeUpstream(backend.URL, false, 30*time.Second, 300*time.Millisecond)
 	upstream.host.DownstreamHeaders = http.Header{
 		"+Merge-Me":  {"Merge-Value"},
 		"+Add-Me":    {"Add-Value"},
@@ -918,7 +919,7 @@ func TestHostSimpleProxyNoHeaderForward(t *testing.T) {
 	// set up proxy
 	p := &Proxy{
 		Next:      httpserver.EmptyNext, // prevents panic in some cases when test fails
-		Upstreams: []Upstream{newFakeUpstream(backend.URL, false, 30*time.Second)},
+		Upstreams: []Upstream{newFakeUpstream(backend.URL, false, 30*time.Second, 300*time.Millisecond)},
 	}
 
 	r := httptest.NewRequest("GET", "/", nil)
@@ -1007,7 +1008,7 @@ func TestHostHeaderReplacedUsingForward(t *testing.T) {
 	}))
 	defer backend.Close()
 
-	upstream := newFakeUpstream(backend.URL, false, 30*time.Second)
+	upstream := newFakeUpstream(backend.URL, false, 30*time.Second, 300*time.Millisecond)
 	proxyHostHeader := "test2.com"
 	upstream.host.UpstreamHeaders = http.Header{"Host": []string{proxyHostHeader}}
 	// set up proxy
@@ -1069,7 +1070,7 @@ func basicAuthTestcase(t *testing.T, upstreamUser, clientUser *url.Userinfo) {
 
 	p := &Proxy{
 		Next:      httpserver.EmptyNext,
-		Upstreams: []Upstream{newFakeUpstream(backURL.String(), false, 30*time.Second)},
+		Upstreams: []Upstream{newFakeUpstream(backURL.String(), false, 30*time.Second, 300*time.Millisecond)},
 	}
 	r, err := http.NewRequest("GET", "/foo", nil)
 	if err != nil {
@@ -1204,7 +1205,7 @@ func TestProxyDirectorURL(t *testing.T) {
 			continue
 		}
 
-		NewSingleHostReverseProxy(targetURL, c.without, 0, 30*time.Second).Director(req)
+		NewSingleHostReverseProxy(targetURL, c.without, 0, 30*time.Second, 300*time.Millisecond).Director(req)
 		if expect, got := c.expectURL, req.URL.String(); expect != got {
 			t.Errorf("case %d url not equal: expect %q, but got %q",
 				i, expect, got)
@@ -1351,7 +1352,7 @@ func TestCancelRequest(t *testing.T) {
 	// set up proxy
 	p := &Proxy{
 		Next:      httpserver.EmptyNext, // prevents panic in some cases when test fails
-		Upstreams: []Upstream{newFakeUpstream(backend.URL, false, 30*time.Second)},
+		Upstreams: []Upstream{newFakeUpstream(backend.URL, false, 30*time.Second, 300*time.Millisecond)},
 	}
 
 	// setup request with cancel ctx
@@ -1400,15 +1401,16 @@ func (r *noopReader) Read(b []byte) (int, error) {
 	return n, nil
 }
 
-func newFakeUpstream(name string, insecure bool, timeout time.Duration) *fakeUpstream {
+func newFakeUpstream(name string, insecure bool, timeout, fallbackDelay time.Duration) *fakeUpstream {
 	uri, _ := url.Parse(name)
 	u := &fakeUpstream{
 		name:          name,
 		from:          "/",
 		timeout:       timeout,
+		fallbackDelay: fallbackDelay,
 		host: &UpstreamHost{
 			Name:         name,
-			ReverseProxy: NewSingleHostReverseProxy(uri, "", http.DefaultMaxIdleConnsPerHost, timeout),
+			ReverseProxy: NewSingleHostReverseProxy(uri, "", http.DefaultMaxIdleConnsPerHost, timeout, fallbackDelay),
 		},
 	}
 	if insecure {
@@ -1423,6 +1425,7 @@ type fakeUpstream struct {
 	from          string
 	without       string
 	timeout       time.Duration
+	fallbackDelay time.Duration
 }
 
 func (u *fakeUpstream) From() string {
@@ -1437,13 +1440,14 @@ func (u *fakeUpstream) Select(r *http.Request) *UpstreamHost {
 		}
 		u.host = &UpstreamHost{
 			Name:         u.name,
-			ReverseProxy: NewSingleHostReverseProxy(uri, u.without, http.DefaultMaxIdleConnsPerHost, u.GetTimeout()),
+			ReverseProxy: NewSingleHostReverseProxy(uri, u.without, http.DefaultMaxIdleConnsPerHost, u.GetTimeout(), u.GetFallbackDelay()),
 		}
 	}
 	return u.host
 }
 
 func (u *fakeUpstream) AllowedPath(requestPath string) bool { return true }
+func (u *fakeUpstream) GetFallbackDelay() time.Duration     { return 300 * time.Millisecond }
 func (u *fakeUpstream) GetTryDuration() time.Duration       { return 1 * time.Second }
 func (u *fakeUpstream) GetTryInterval() time.Duration       { return 250 * time.Millisecond }
 func (u *fakeUpstream) GetTimeout() time.Duration           { return u.timeout }
@@ -1478,6 +1482,7 @@ type fakeWsUpstream struct {
 	without       string
 	insecure      bool
 	timeout       time.Duration
+	fallbackDelay time.Duration
 }
 
 func (u *fakeWsUpstream) From() string {
@@ -1488,7 +1493,7 @@ func (u *fakeWsUpstream) Select(r *http.Request) *UpstreamHost {
 	uri, _ := url.Parse(u.name)
 	host := &UpstreamHost{
 		Name:         u.name,
-		ReverseProxy: NewSingleHostReverseProxy(uri, u.without, http.DefaultMaxIdleConnsPerHost, u.GetTimeout()),
+		ReverseProxy: NewSingleHostReverseProxy(uri, u.without, http.DefaultMaxIdleConnsPerHost, u.GetTimeout(), u.GetFallbackDelay()),
 		UpstreamHeaders: http.Header{
 			"Connection": {"{>Connection}"},
 			"Upgrade":    {"{>Upgrade}"}},
@@ -1500,6 +1505,7 @@ func (u *fakeWsUpstream) Select(r *http.Request) *UpstreamHost {
 }
 
 func (u *fakeWsUpstream) AllowedPath(requestPath string) bool { return true }
+func (u *fakeWsUpstream) GetFallbackDelay() time.Duration     { return 300 * time.Millisecond }
 func (u *fakeWsUpstream) GetTryDuration() time.Duration       { return 1 * time.Second }
 func (u *fakeWsUpstream) GetTryInterval() time.Duration       { return 250 * time.Millisecond }
 func (u *fakeWsUpstream) GetTimeout() time.Duration           { return u.timeout }
@@ -1548,7 +1554,7 @@ func BenchmarkProxy(b *testing.B) {
 	}))
 	defer backend.Close()
 
-	upstream := newFakeUpstream(backend.URL, false, 30*time.Second)
+	upstream := newFakeUpstream(backend.URL, false, 30*time.Second, 300*time.Millisecond)
 	upstream.host.UpstreamHeaders = http.Header{
 		"Hostname":          {"{hostname}"},
 		"Host":              {"{host}"},

caddyhttp/proxy/reverseproxy.go

@@ -148,7 +148,7 @@ func singleJoiningSlash(a, b string) string {
 // the target request will be for /base/dir.
 // Without logic: target's path is "/", incoming is "/api/messages",
 // without is "/api", then the target request will be for /messages.
-func NewSingleHostReverseProxy(target *url.URL, without string, keepalive int, timeout time.Duration) *ReverseProxy {
+func NewSingleHostReverseProxy(target *url.URL, without string, keepalive int, timeout, fallbackDelay time.Duration) *ReverseProxy {
 	targetQuery := target.RawQuery
 	director := func(req *http.Request) {
 		if target.Scheme == "unix" {
@@ -234,6 +234,9 @@ func NewSingleHostReverseProxy(target *url.URL, without string, keepalive int, t
 	if timeout != defaultDialer.Timeout {
 		dialer.Timeout = timeout
 	}
+	if fallbackDelay != defaultDialer.FallbackDelay {
+		dialer.FallbackDelay = fallbackDelay
+	}
 
 	rp := &ReverseProxy{
 		Director:      director,

caddyhttp/proxy/reverseproxy_test.go

@@ -67,7 +67,7 @@ func TestSingleSRVHostReverseProxy(t *testing.T) {
 	}
 	port := uint16(pp)
 
-	rp := NewSingleHostReverseProxy(target, "", http.DefaultMaxIdleConnsPerHost, 30*time.Second)
+	rp := NewSingleHostReverseProxy(target, "", http.DefaultMaxIdleConnsPerHost, 30*time.Second, 300*time.Millisecond)
 	rp.srvResolver = testResolver{
 		result: []*net.SRV{
 			{Target: upstream.Hostname(), Port: port, Priority: 1, Weight: 1},

caddyhttp/proxy/upstream.go

@@ -49,6 +49,7 @@ type staticUpstream struct {
 	Hosts             HostPool
 	Policy            Policy
 	KeepAlive         int
+	FallbackDelay     time.Duration
 	Timeout           time.Duration
 	FailTimeout       time.Duration
 	TryDuration       time.Duration
@@ -227,7 +228,7 @@ func (u *staticUpstream) NewHost(host string) (*UpstreamHost, error) {
 		return nil, err
 	}
 
-	uh.ReverseProxy = NewSingleHostReverseProxy(baseURL, uh.WithoutPathPrefix, u.KeepAlive, u.Timeout)
+	uh.ReverseProxy = NewSingleHostReverseProxy(baseURL, uh.WithoutPathPrefix, u.KeepAlive, u.Timeout, u.FallbackDelay)
 	if u.insecureSkipVerify {
 		uh.ReverseProxy.UseInsecureTransport()
 	}
@@ -309,6 +310,15 @@ func parseBlock(c *caddyfile.Dispenser, u *staticUpstream, hasSrv bool) error {
 			arg = c.Val()
 		}
 		u.Policy = policyCreateFunc(arg)
+	case "fallback_delay":
+		if !c.NextArg() {
+			return c.ArgErr()
+		}
+		dur, err := time.ParseDuration(c.Val())
+		if err != nil {
+			return err
+		}
+		u.FallbackDelay = dur
 	case "fail_timeout":
 		if !c.NextArg() {
 			return c.ArgErr()
@@ -620,6 +630,11 @@ func (u *staticUpstream) AllowedPath(requestPath string) bool {
 	return true
 }
 
+// GetFallbackDelay returns u.FallbackDelay.
+func (u *staticUpstream) GetFallbackDelay() time.Duration {
+	return u.FallbackDelay
+}
+
 // GetTryDuration returns u.TryDuration.
 func (u *staticUpstream) GetTryDuration() time.Duration {
 	return u.TryDuration