Major refactor of all HTTPS/TLS/ACME code

Biggest change is no longer using standard library's tls.Config.getCertificate function to get a certificate during TLS handshake. Implemented our own cache which can be changed dynamically at runtime, even during TLS handshakes. As such, restarts are no longer required after certificate renewals or OCSP updates.

We also allow loading multiple certificates and keys per host, even by specifying a directory (tls got a new 'load' command for that).

Renamed the letsencrypt package to https in a gradual effort to become more generic; and https is more fitting for what the package does now.

There are still some known bugs, e.g. reloading where a new certificate is required but port 80 isn't currently listening, will cause the challenge to fail. There's still plenty of cleanup to do and tests to write. It is especially confusing right now how we enable "on-demand" TLS during setup and keep track of that. But this change should basically work so far.

caddy/caddy.go

@@ -28,7 +28,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/mholt/caddy/caddy/letsencrypt"
+	"github.com/mholt/caddy/caddy/https"
 	"github.com/mholt/caddy/server"
 )
 
@@ -44,7 +44,7 @@ var (
 	Quiet bool
 
 	// HTTP2 indicates whether HTTP2 is enabled or not.
-	HTTP2 bool // TODO: temporary flag until http2 is standard
+	HTTP2 bool
 
 	// PidFile is the path to the pidfile to create.
 	PidFile string
@@ -191,9 +191,13 @@ func startServers(groupings bindingGroup) error {
 		if err != nil {
 			return err
 		}
-		s.HTTP2 = HTTP2                                           // TODO: This setting is temporary
-		s.ReqCallback = letsencrypt.RequestCallback               // ensures we can solve ACME challenges while running
-		s.SNICallback = letsencrypt.GetCertificateDuringHandshake // TLS on demand -- awesome!
+		s.HTTP2 = HTTP2
+		s.ReqCallback = https.RequestCallback // ensures we can solve ACME challenges while running
+		if s.OnDemandTLS {
+			s.TLSConfig.GetCertificate = https.GetOrObtainCertificate // TLS on demand -- awesome!
+		} else {
+			s.TLSConfig.GetCertificate = https.GetCertificate
+		}
 
 		var ln server.ListenerFile
 		if IsRestart() {
@@ -278,7 +282,7 @@ func startServers(groupings bindingGroup) error {
 // It does NOT execute shutdown callbacks that may have been
 // configured by middleware (they must be executed separately).
 func Stop() error {
-	letsencrypt.Deactivate()
+	https.Deactivate()
 
 	serversMu.Lock()
 	for _, s := range servers {

caddy/caddy_test.go

@@ -4,10 +4,21 @@ import (
 	"net/http"
 	"testing"
 	"time"
+
+	"github.com/mholt/caddy/caddy/https"
+	"github.com/xenolf/lego/acme"
 )
 
 func TestCaddyStartStop(t *testing.T) {
-	caddyfile := "localhost:1984\ntls off"
+	// Use fake ACME clients for testing
+	https.NewACMEClient = func(email string, allowPrompts bool) (*https.ACMEClient, error) {
+		return &https.ACMEClient{
+			Client:       new(acme.Client),
+			AllowPrompts: allowPrompts,
+		}, nil
+	}
+
+	caddyfile := "localhost:1984"
 
 	for i := 0; i < 2; i++ {
 		err := Start(CaddyfileInput{Contents: []byte(caddyfile)})

caddy/config.go

@@ -8,7 +8,7 @@ import (
 	"net"
 	"sync"
 
-	"github.com/mholt/caddy/caddy/letsencrypt"
+	"github.com/mholt/caddy/caddy/https"
 	"github.com/mholt/caddy/caddy/parse"
 	"github.com/mholt/caddy/caddy/setup"
 	"github.com/mholt/caddy/middleware"
@@ -128,7 +128,7 @@ func loadConfigs(filename string, input io.Reader) ([]server.Config, error) {
 	if !IsRestart() && !Quiet {
 		fmt.Print("Activating privacy features...")
 	}
-	configs, err = letsencrypt.Activate(configs)
+	configs, err = https.Activate(configs)
 	if err != nil {
 		return nil, err
 	} else if !IsRestart() && !Quiet {
@@ -318,7 +318,7 @@ func validDirective(d string) bool {
 // root.
 func DefaultInput() CaddyfileInput {
 	port := Port
-	if letsencrypt.HostQualifies(Host) && port == DefaultPort {
+	if https.HostQualifies(Host) && port == DefaultPort {
 		port = "443"
 	}
 	return CaddyfileInput{

caddy/directives.go

 package caddy
 
 import (
+	"github.com/mholt/caddy/caddy/https"
 	"github.com/mholt/caddy/caddy/parse"
 	"github.com/mholt/caddy/caddy/setup"
 	"github.com/mholt/caddy/middleware"
@@ -43,7 +44,7 @@ var directiveOrder = []directive{
 	// Essential directives that initialize vital configuration settings
 	{"root", setup.Root},
 	{"bind", setup.BindHost},
-	{"tls", setup.TLS}, // letsencrypt is set up just after tls
+	{"tls", https.Setup},
 
 	// Other directives that don't create HTTP handlers
 	{"startup", setup.Startup},

caddy/helpers.go

@@ -11,14 +11,8 @@ import (
 	"strconv"
 	"strings"
 	"sync"
-
-	"github.com/mholt/caddy/caddy/letsencrypt"
 )
 
-func init() {
-	letsencrypt.OnChange = func() error { return Restart(nil) }
-}
-
 // isLocalhost returns true if host looks explicitly like a localhost address.
 func isLocalhost(host string) bool {
 	return host == "localhost" || host == "::1" || strings.HasPrefix(host, "127.")

caddy/https/certificates.go

+package https
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"errors"
+	"io/ioutil"
+	"log"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/xenolf/lego/acme"
+	"golang.org/x/crypto/ocsp"
+)
+
+// certCache stores certificates in memory,
+// keying certificates by name.
+var certCache = make(map[string]Certificate)
+var certCacheMu sync.RWMutex
+
+// Certificate is a tls.Certificate with associated metadata tacked on.
+// Even if the metadata can be obtained by parsing the certificate,
+// we can be more efficient by extracting the metadata once so it's
+// just there, ready to use.
+type Certificate struct {
+	*tls.Certificate
+
+	// Names is the list of names this certificate is written for.
+	// The first is the CommonName (if any), the rest are SAN.
+	Names []string
+
+	// NotAfter is when the certificate expires.
+	NotAfter time.Time
+
+	// Managed certificates are certificates that Caddy is managing,
+	// as opposed to the user specifying a certificate and key file
+	// or directory and managing the certificate resources themselves.
+	Managed bool
+
+	// OnDemand certificates are obtained or loaded on-demand during TLS
+	// handshakes (as opposed to preloaded certificates, which are loaded
+	// at startup). If OnDemand is true, Managed must necessarily be true.
+	// OnDemand certificates are maintained in the background just like
+	// preloaded ones, however, if an OnDemand certificate fails to renew,
+	// it is removed from the in-memory cache.
+	OnDemand bool
+
+	// OCSP contains the certificate's parsed OCSP response.
+	OCSP *ocsp.Response
+}
+
+// getCertificate gets a certificate from the in-memory cache that
+// matches name (a certificate name). Note that if name does not have
+// an exact match, it will be checked against names of the form
+// '*.example.com' (wildcard certificates) according to RFC 6125.
+//
+// If cert was found by matching name, matched will be returned true.
+// If no match is found, the default certificate will be returned and
+// matched will be returned as false. (The default certificate is the
+// first one that entered the cache.) If the cache is empty (or there
+// is no default certificate for some reason), matched will still be
+// false, but cert.Certificate will be nil.
+//
+// The logic in this function is adapted from the Go standard library,
+// which is by the Go Authors.
+//
+// This function is safe for concurrent use.
+func getCertificate(name string) (cert Certificate, matched bool) {
+	// Not going to trim trailing dots here since RFC 3546 says,
+	// "The hostname is represented ... without a trailing dot."
+	// Just normalize to lowercase.
+	name = strings.ToLower(name)
+
+	certCacheMu.RLock()
+	defer certCacheMu.RUnlock()
+
+	// exact match? great, let's use it
+	if cert, ok := certCache[name]; ok {
+		return cert, true
+	}
+
+	// try replacing labels in the name with wildcards until we get a match
+	labels := strings.Split(name, ".")
+	for i := range labels {
+		labels[i] = "*"
+		candidate := strings.Join(labels, ".")
+		if cert, ok := certCache[candidate]; ok {
+			return cert, true
+		}
+	}
+
+	// if nothing matches, return the default certificate
+	cert = certCache[""]
+	return cert, false
+}
+
+// cacheManagedCertificate loads the certificate for domain into the
+// cache, flagging it as Managed and, if onDemand is true, as OnDemand
+// (meaning that it was obtained or loaded during a TLS handshake).
+//
+// This function is safe for concurrent use.
+func cacheManagedCertificate(domain string, onDemand bool) (Certificate, error) {
+	cert, err := makeCertificateFromDisk(storage.SiteCertFile(domain), storage.SiteKeyFile(domain))
+	if err != nil {
+		return cert, err
+	}
+	cert.Managed = true
+	cert.OnDemand = onDemand
+	cacheCertificate(cert)
+	return cert, nil
+}
+
+// cacheUnmanagedCertificatePEMFile loads a certificate for host using certFile
+// and keyFile, which must be in PEM format. It stores the certificate in
+// memory. The Managed and OnDemand flags of the certificate will be set to
+// false.
+//
+// This function is safe for concurrent use.
+func cacheUnmanagedCertificatePEMFile(certFile, keyFile string) error {
+	cert, err := makeCertificateFromDisk(certFile, keyFile)
+	if err != nil {
+		return err
+	}
+	cacheCertificate(cert)
+	return nil
+}
+
+// cacheUnmanagedCertificatePEMBytes makes a certificate out of the PEM bytes
+// of the certificate and key, then caches it in memory.
+//
+// This function is safe for concurrent use.
+func cacheUnmanagedCertificatePEMBytes(certBytes, keyBytes []byte) error {
+	cert, err := makeCertificate(certBytes, keyBytes)
+	if err != nil {
+		return err
+	}
+	cacheCertificate(cert)
+	return nil
+}
+
+// makeCertificateFromDisk makes a Certificate by loading the
+// certificate and key files. It fills out all the fields in
+// the certificate except for the Managed and OnDemand flags.
+// (It is up to the caller to set those.)
+func makeCertificateFromDisk(certFile, keyFile string) (Certificate, error) {
+	certPEMBlock, err := ioutil.ReadFile(certFile)
+	if err != nil {
+		return Certificate{}, err
+	}
+	keyPEMBlock, err := ioutil.ReadFile(keyFile)
+	if err != nil {
+		return Certificate{}, err
+	}
+	return makeCertificate(certPEMBlock, keyPEMBlock)
+}
+
+// makeCertificate turns a certificate PEM bundle and a key PEM block into
+// a Certificate, with OCSP and other relevant metadata tagged with it,
+// except for the OnDemand and Managed flags. It is up to the caller to
+// set those properties.
+func makeCertificate(certPEMBlock, keyPEMBlock []byte) (Certificate, error) {
+	var cert Certificate
+
+	// Convert to a tls.Certificate
+	tlsCert, err := tls.X509KeyPair(certPEMBlock, keyPEMBlock)
+	if err != nil {
+		return cert, err
+	}
+	if len(tlsCert.Certificate) == 0 {
+		return cert, errors.New("certificate is empty")
+	}
+	cert.Certificate = &tlsCert
+
+	// Parse leaf certificate and extract relevant metadata
+	leaf, err := x509.ParseCertificate(tlsCert.Certificate[0])
+	if err != nil {
+		return cert, err
+	}
+	if leaf.Subject.CommonName != "" {
+		cert.Names = []string{strings.ToLower(leaf.Subject.CommonName)}
+	}
+	for _, name := range leaf.DNSNames {
+		if name != leaf.Subject.CommonName {
+			cert.Names = append(cert.Names, strings.ToLower(name))
+		}
+	}
+	cert.NotAfter = leaf.NotAfter
+
+	// Staple OCSP
+	ocspBytes, ocspResp, err := acme.GetOCSPForCert(certPEMBlock)
+	if err != nil {
+		// An error here is not a problem because a certificate may simply
+		// not contain a link to an OCSP server. But we should log it anyway.
+		log.Printf("[WARNING] No OCSP stapling for %v: %v", cert.Names, err)
+	} else if ocspResp.Status == ocsp.Good {
+		tlsCert.OCSPStaple = ocspBytes
+		cert.OCSP = ocspResp
+	}
+
+	return cert, nil
+}
+
+// cacheCertificate adds cert to the in-memory cache. If the cache is
+// empty, cert will be used as the default certificate. If the cache is
+// full, random entries are deleted until there is room to map all the
+// names on the certificate.
+//
+// This certificate will be keyed to the names in cert.Names. Any name
+// that is already a key in the cache will be replaced with this cert.
+//
+// This function is safe for concurrent use.
+func cacheCertificate(cert Certificate) {
+	certCacheMu.Lock()
+	if _, ok := certCache[""]; !ok {
+		certCache[""] = cert // use as default
+	}
+	for len(certCache)+len(cert.Names) > 10000 {
+		// for simplicity, just remove random elements
+		for key := range certCache {
+			if key == "" { // ... but not the default cert
+				continue
+			}
+			delete(certCache, key)
+			break
+		}
+	}
+	for _, name := range cert.Names {
+		certCache[name] = cert
+	}
+	certCacheMu.Unlock()
+}

caddy/https/client.go

+package https
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"net"
+	"sync"
+	"time"
+
+	"github.com/mholt/caddy/server"
+	"github.com/xenolf/lego/acme"
+)
+
+// acmeMu ensures that only one ACME challenge occurs at a time.
+var acmeMu sync.Mutex
+
+// ACMEClient is an acme.Client with custom state attached.
+type ACMEClient struct {
+	*acme.Client
+	AllowPrompts bool // if false, we assume AlternatePort must be used
+}
+
+// NewACMEClient creates a new ACMEClient given an email and whether
+// prompting the user is allowed. Clients should not be kept and
+// re-used over long periods of time, but immediate re-use is more
+// efficient than re-creating on every iteration.
+var NewACMEClient = func(email string, allowPrompts bool) (*ACMEClient, error) {
+	// Look up or create the LE user account
+	leUser, err := getUser(email)
+	if err != nil {
+		return nil, err
+	}
+
+	// The client facilitates our communication with the CA server.
+	client, err := acme.NewClient(CAUrl, &leUser, rsaKeySizeToUse)
+	if err != nil {
+		return nil, err
+	}
+
+	// If not registered, the user must register an account with the CA
+	// and agree to terms
+	if leUser.Registration == nil {
+		reg, err := client.Register()
+		if err != nil {
+			return nil, errors.New("registration error: " + err.Error())
+		}
+		leUser.Registration = reg
+
+		if allowPrompts { // can't prompt a user who isn't there
+			if !Agreed && reg.TosURL == "" {
+				Agreed = promptUserAgreement(saURL, false) // TODO - latest URL
+			}
+			if !Agreed && reg.TosURL == "" {
+				return nil, errors.New("user must agree to terms")
+			}
+		}
+
+		err = client.AgreeToTOS()
+		if err != nil {
+			saveUser(leUser) // Might as well try, right?
+			return nil, errors.New("error agreeing to terms: " + err.Error())
+		}
+
+		// save user to the file system
+		err = saveUser(leUser)
+		if err != nil {
+			return nil, errors.New("could not save user: " + err.Error())
+		}
+	}
+
+	return &ACMEClient{
+		Client:       client,
+		AllowPrompts: allowPrompts,
+	}, nil
+}
+
+// NewACMEClientGetEmail creates a new ACMEClient and gets an email
+// address at the same time (a server config is required, since it
+// may contain an email address in it).
+func NewACMEClientGetEmail(config server.Config, allowPrompts bool) (*ACMEClient, error) {
+	return NewACMEClient(getEmail(config, allowPrompts), allowPrompts)
+}
+
+// Configure configures c according to bindHost, which is the host (not
+// whole address) to bind the listener to in solving the http and tls-sni
+// challenges.
+func (c *ACMEClient) Configure(bindHost string) {
+	// If we allow prompts, operator must be present. In our case,
+	// that is synonymous with saying the server is not already
+	// started. So if the user is still there, we don't use
+	// AlternatePort because we don't need to proxy the challenges.
+	// Conversely, if the operator is not there, the server has
+	// already started and we need to proxy the challenge.
+	if c.AllowPrompts {
+		// Operator is present; server is not already listening
+		c.SetHTTPAddress(net.JoinHostPort(bindHost, ""))
+		c.SetTLSAddress(net.JoinHostPort(bindHost, ""))
+		//c.ExcludeChallenges([]acme.Challenge{acme.DNS01})
+	} else {
+		// Operator is not present; server is started, so proxy challenges
+		c.SetHTTPAddress(net.JoinHostPort(bindHost, AlternatePort))
+		c.SetTLSAddress(net.JoinHostPort(bindHost, AlternatePort))
+		//c.ExcludeChallenges([]acme.Challenge{acme.TLSSNI01, acme.DNS01})
+	}
+	c.ExcludeChallenges([]acme.Challenge{acme.TLSSNI01, acme.DNS01}) // TODO: can we proxy TLS challenges? and we should support DNS...
+}
+
+// Obtain obtains a single certificate for names. It stores the certificate
+// on the disk if successful.
+func (c *ACMEClient) Obtain(names []string) error {
+Attempts:
+	for attempts := 0; attempts < 2; attempts++ {
+		acmeMu.Lock()
+		certificate, failures := c.ObtainCertificate(names, true, nil)
+		acmeMu.Unlock()
+		if len(failures) > 0 {
+			// Error - try to fix it or report it to the user and abort
+			var errMsg string             // we'll combine all the failures into a single error message
+			var promptedForAgreement bool // only prompt user for agreement at most once
+
+			for errDomain, obtainErr := range failures {
+				// TODO: Double-check, will obtainErr ever be nil?
+				if tosErr, ok := obtainErr.(acme.TOSError); ok {
+					// Terms of Service agreement error; we can probably deal with this
+					if !Agreed && !promptedForAgreement && c.AllowPrompts {
+						Agreed = promptUserAgreement(tosErr.Detail, true) // TODO: Use latest URL
+						promptedForAgreement = true
+					}
+					if Agreed || !c.AllowPrompts {
+						err := c.AgreeToTOS()
+						if err != nil {
+							return errors.New("error agreeing to updated terms: " + err.Error())
+						}
+						continue Attempts
+					}
+				}
+
+				// If user did not agree or it was any other kind of error, just append to the list of errors
+				errMsg += "[" + errDomain + "] failed to get certificate: " + obtainErr.Error() + "\n"
+			}
+			return errors.New(errMsg)
+		}
+
+		// Success - immediately save the certificate resource
+		err := saveCertResource(certificate)
+		if err != nil {
+			return fmt.Errorf("error saving assets for %v: %v", names, err)
+		}
+
+		break
+	}
+
+	return nil
+}
+
+// Renew renews the managed certificate for name. Right now our storage
+// mechanism only supports one name per certificate, so this function only
+// accepts one domain as input. It can be easily modified to support SAN
+// certificates if, one day, they become desperately needed enough that our
+// storage mechanism is upgraded to be more complex to support SAN certs.
+//
+// Anyway, this function is safe for concurrent use.
+func (c *ACMEClient) Renew(name string) error {
+	// Prepare for renewal (load PEM cert, key, and meta)
+	certBytes, err := ioutil.ReadFile(storage.SiteCertFile(name))
+	if err != nil {
+		return err
+	}
+	keyBytes, err := ioutil.ReadFile(storage.SiteKeyFile(name))
+	if err != nil {
+		return err
+	}
+	metaBytes, err := ioutil.ReadFile(storage.SiteMetaFile(name))
+	if err != nil {
+		return err
+	}
+	var certMeta acme.CertificateResource
+	err = json.Unmarshal(metaBytes, &certMeta)
+	certMeta.Certificate = certBytes
+	certMeta.PrivateKey = keyBytes
+
+	// Perform renewal and retry if necessary, but not too many times.
+	var newCertMeta acme.CertificateResource
+	var success bool
+	for attempts := 0; attempts < 2; attempts++ {
+		acmeMu.Lock()
+		newCertMeta, err = c.RenewCertificate(certMeta, true)
+		acmeMu.Unlock()
+		if err == nil {
+			success = true
+			break
+		}
+
+		// If the legal terms changed and need to be agreed to again,
+		// we can handle that.
+		if _, ok := err.(acme.TOSError); ok {
+			err := c.AgreeToTOS()
+			if err != nil {
+				return err
+			}
+			continue
+		}
+
+		// For any other kind of error, wait 10s and try again.
+		time.Sleep(10 * time.Second)
+	}
+
+	if !success {
+		return errors.New("too many renewal attempts; last error: " + err.Error())
+	}
+
+	return saveCertResource(newCertMeta)
+}

caddy/letsencrypt/crypto.go → caddy/https/crypto.go

-package letsencrypt
+package https
 
 import (
 	"crypto/rsa"

caddy/letsencrypt/crypto_test.go → caddy/https/crypto_test.go

-package letsencrypt
+package https
 
 import (
 	"bytes"

caddy/letsencrypt/handler.go → caddy/https/handler.go

-package letsencrypt
+package https
 
 import (
 	"crypto/tls"

caddy/letsencrypt/handler_test.go → caddy/https/handler_test.go

-package letsencrypt
+package https
 
 import (
 	"net"

caddy/https/handshake.go

+package https
+
+import (
+	"bytes"
+	"crypto/tls"
+	"encoding/pem"
+	"errors"
+	"fmt"
+	"log"
+	"sync"
+	"time"
+
+	"github.com/mholt/caddy/server"
+	"github.com/xenolf/lego/acme"
+)
+
+// GetCertificate gets a certificate to satisfy clientHello as long as
+// the certificate is already cached in memory.
+//
+// This function is safe for use as a tls.Config.GetCertificate callback.
+func GetCertificate(clientHello *tls.ClientHelloInfo) (*tls.Certificate, error) {
+	cert, err := getCertDuringHandshake(clientHello.ServerName, false)
+	return cert.Certificate, err
+}
+
+// GetOrObtainCertificate will get a certificate to satisfy clientHello, even
+// if that means obtaining a new certificate from a CA during the handshake.
+// It first checks the in-memory cache, then accesses disk, then accesses the
+// network if it must. An obtained certificate will be stored on disk and
+// cached in memory.
+//
+// This function is safe for use as a tls.Config.GetCertificate callback.
+func GetOrObtainCertificate(clientHello *tls.ClientHelloInfo) (*tls.Certificate, error) {
+	cert, err := getCertDuringHandshake(clientHello.ServerName, true)
+	return cert.Certificate, err
+}
+
+// getCertDuringHandshake will get a certificate for name. It first tries
+// the in-memory cache, then, if obtainIfNecessary is true, it goes to disk,
+// then asks the CA for a certificate if necessary.
+//
+// This function is safe for concurrent use.
+func getCertDuringHandshake(name string, obtainIfNecessary bool) (Certificate, error) {
+	// First check our in-memory cache to see if we've already loaded it
+	cert, ok := getCertificate(name)
+	if ok {
+		return cert, nil
+	}
+
+	if obtainIfNecessary {
+		// TODO: Mitigate abuse!
+		var err error
+
+		// Then check to see if we have one on disk
+		cert, err := cacheManagedCertificate(name, true)
+		if err != nil {
+			return cert, err
+		} else if cert.Certificate != nil {
+			cert, err := handshakeMaintenance(name, cert)
+			if err != nil {
+				log.Printf("[ERROR] Maintaining newly-loaded certificate for %s: %v", name, err)
+			}
+			return cert, err
+		}
+
+		// Only option left is to get one from LE, but the name has to qualify first
+		if !HostQualifies(name) {
+			return cert, errors.New("hostname '" + name + "' does not qualify for certificate")
+		}
+
+		// By this point, we need to obtain one from the CA.
+		return obtainOnDemandCertificate(name)
+	}
+
+	return Certificate{}, nil
+}
+
+// obtainOnDemandCertificate obtains a certificate for name for the given
+// clientHello. If another goroutine has already started obtaining a cert
+// for name, it will wait and use what the other goroutine obtained.
+//
+// This function is safe for use by multiple concurrent goroutines.
+func obtainOnDemandCertificate(name string) (Certificate, error) {
+	// We must protect this process from happening concurrently, so synchronize.
+	obtainCertWaitChansMu.Lock()
+	wait, ok := obtainCertWaitChans[name]
+	if ok {
+		// lucky us -- another goroutine is already obtaining the certificate.
+		// wait for it to finish obtaining the cert and then we'll use it.
+		obtainCertWaitChansMu.Unlock()
+		<-wait
+		return getCertDuringHandshake(name, false) // passing in true might result in infinite loop if obtain failed
+	}
+
+	// looks like it's up to us to do all the work and obtain the cert
+	wait = make(chan struct{})
+	obtainCertWaitChans[name] = wait
+	obtainCertWaitChansMu.Unlock()
+
+	// Unblock waiters and delete waitgroup when we return
+	defer func() {
+		obtainCertWaitChansMu.Lock()
+		close(wait)
+		delete(obtainCertWaitChans, name)
+		obtainCertWaitChansMu.Unlock()
+	}()
+
+	log.Printf("[INFO] Obtaining new certificate for %s", name)
+
+	// obtain cert
+	client, err := NewACMEClientGetEmail(server.Config{}, false)
+	if err != nil {
+		return Certificate{}, errors.New("error creating client: " + err.Error())
+	}
+	client.Configure("") // TODO: which BindHost?
+	err = client.Obtain([]string{name})
+	if err != nil {
+		return Certificate{}, err
+	}
+
+	// The certificate is on disk; now just start over to load it and serve it
+	return getCertDuringHandshake(name, false) // pass in false as a fail-safe from infinite-looping
+}
+
+// handshakeMaintenance performs a check on cert for expiration and OCSP
+// validity.
+//
+// This function is safe for use by multiple concurrent goroutines.
+func handshakeMaintenance(name string, cert Certificate) (Certificate, error) {
+	// fmt.Println("ON-DEMAND CERT?", cert.OnDemand)
+	// if !cert.OnDemand {
+	// 	return cert, nil
+	// }
+	fmt.Println("Checking expiration of cert; on-demand:", cert.OnDemand)
+
+	// Check cert expiration
+	timeLeft := cert.NotAfter.Sub(time.Now().UTC())
+	if timeLeft < renewDurationBefore {
+		log.Printf("[INFO] Certificate for %v expires in %v; attempting renewal", cert.Names, timeLeft)
+		return renewDynamicCertificate(name)
+	}
+
+	// Check OCSP staple validity
+	if cert.OCSP != nil {
+		refreshTime := cert.OCSP.ThisUpdate.Add(cert.OCSP.NextUpdate.Sub(cert.OCSP.ThisUpdate) / 2)
+		if time.Now().After(refreshTime) {
+			err := stapleOCSP(&cert, nil)
+			if err != nil {
+				// An error with OCSP stapling is not the end of the world, and in fact, is
+				// quite common considering not all certs have issuer URLs that support it.
+				log.Printf("[ERROR] Getting OCSP for %s: %v", name, err)
+			}
+			certCacheMu.Lock()
+			certCache[name] = cert
+			certCacheMu.Unlock()
+		}
+	}
+
+	return cert, nil
+}
+
+// renewDynamicCertificate renews currentCert using the clientHello. It returns the
+// certificate to use and an error, if any. currentCert may be returned even if an
+// error occurs, since we perform renewals before they expire and it may still be
+// usable. name should already be lower-cased before calling this function.
+//
+// This function is safe for use by multiple concurrent goroutines.
+func renewDynamicCertificate(name string) (Certificate, error) {
+	obtainCertWaitChansMu.Lock()
+	wait, ok := obtainCertWaitChans[name]
+	if ok {
+		// lucky us -- another goroutine is already renewing the certificate.
+		// wait for it to finish, then we'll use the new one.
+		obtainCertWaitChansMu.Unlock()
+		<-wait
+		return getCertDuringHandshake(name, false)
+	}
+
+	// looks like it's up to us to do all the work and renew the cert
+	wait = make(chan struct{})
+	obtainCertWaitChans[name] = wait
+	obtainCertWaitChansMu.Unlock()
+
+	// unblock waiters and delete waitgroup when we return
+	defer func() {
+		obtainCertWaitChansMu.Lock()
+		close(wait)
+		delete(obtainCertWaitChans, name)
+		obtainCertWaitChansMu.Unlock()
+	}()
+
+	log.Printf("[INFO] Renewing certificate for %s", name)
+
+	client, err := NewACMEClient("", false) // renewals don't use email
+	if err != nil {
+		return Certificate{}, err
+	}
+	client.Configure("") // TODO: Bind address of relevant listener, yuck
+	err = client.Renew(name)
+	if err != nil {
+		return Certificate{}, err
+	}
+
+	return getCertDuringHandshake(name, false)
+}
+
+// stapleOCSP staples OCSP information to cert for hostname name.
+// If you have it handy, you should pass in the PEM-encoded certificate
+// bundle; otherwise the DER-encoded cert will have to be PEM-encoded.
+// If you don't have the PEM blocks handy, just pass in nil.
+//
+// Errors here are not necessarily fatal, it could just be that the
+// certificate doesn't have an issuer URL.
+func stapleOCSP(cert *Certificate, pemBundle []byte) error {
+	if pemBundle == nil {
+		// The function in the acme package that gets OCSP requires a PEM-encoded cert
+		bundle := new(bytes.Buffer)
+		for _, derBytes := range cert.Certificate.Certificate {
+			pem.Encode(bundle, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes})
+		}
+		pemBundle = bundle.Bytes()
+	}
+
+	ocspBytes, ocspResp, err := acme.GetOCSPForCert(pemBundle)
+	if err != nil {
+		return err
+	}
+
+	cert.Certificate.OCSPStaple = ocspBytes
+	cert.OCSP = ocspResp
+
+	return nil
+}
+
+// obtainCertWaitChans is used to coordinate obtaining certs for each hostname.
+var obtainCertWaitChans = make(map[string]chan struct{})
+var obtainCertWaitChansMu sync.Mutex

caddy/letsencrypt/letsencrypt.go → caddy/https/https.go

-// Package letsencrypt integrates Let's Encrypt functionality into Caddy
-// with first-class support for creating and renewing certificates
-// automatically. It is designed to configure sites for HTTPS by default.
-package letsencrypt
+// Package https facilitates the management of TLS assets and integrates
+// Let's Encrypt functionality into Caddy with first-class support for
+// creating and renewing certificates automatically. It is designed to
+// configure sites for HTTPS by default.
+package https
 
 import (
 	"encoding/json"
 	"errors"
-	"fmt"
 	"io/ioutil"
 	"net"
 	"net/http"
@@ -14,9 +14,6 @@ import (
 	"strings"
 	"time"
 
-	"golang.org/x/crypto/ocsp"
-
-	"github.com/mholt/caddy/caddy/setup"
 	"github.com/mholt/caddy/middleware"
 	"github.com/mholt/caddy/middleware/redirect"
 	"github.com/mholt/caddy/server"
@@ -38,34 +35,27 @@ import (
 //
 // Also note that calling this function activates asset
 // management automatically, which keeps certificates
-// renewed and OCSP stapling updated. This has the effect
-// of causing restarts when assets are updated.
+// renewed and OCSP stapling updated.
 //
 // Activate returns the updated list of configs, since
 // some may have been appended, for example, to redirect
 // plaintext HTTP requests to their HTTPS counterpart.
-// This function only appends; it does not prepend or splice.
+// This function only appends; it does not splice.
 func Activate(configs []server.Config) ([]server.Config, error) {
 	// just in case previous caller forgot...
 	Deactivate()
 
-	// reset cached ocsp from any previous activations
-	ocspCache = make(map[*[]byte]*ocsp.Response)
-
 	// pre-screen each config and earmark the ones that qualify for managed TLS
 	MarkQualified(configs)
 
 	// place certificates and keys on disk
-	err := ObtainCerts(configs, "")
+	err := ObtainCerts(configs, true)
 	if err != nil {
 		return configs, err
 	}
 
 	// update TLS configurations
-	EnableTLS(configs)
-
-	// enable OCSP stapling (this affects all TLS-enabled configs)
-	err = StapleOCSP(configs)
+	err = EnableTLS(configs, true)
 	if err != nil {
 		return configs, err
 	}
@@ -78,18 +68,19 @@ func Activate(configs []server.Config) ([]server.Config, error) {
 	// the renewal ticker is reset, so if restarts happen more often than
 	// the ticker interval, renewals would never happen. but doing
 	// it right away at start guarantees that renewals aren't missed.
-	renewCertificates(configs, false)
-
-	// keep certificates renewed and OCSP stapling updated
-	go maintainAssets(configs, stopChan)
-
-	// TODO - experimental dynamic TLS!
-	for i := range configs {
-		if configs[i].Host == "" && configs[i].Port == "443" {
-			configs[i].TLS.Enabled = true
+	client, err := NewACMEClient("", true) // renewals don't use email
+	if err != nil {
+		return configs, err
 	}
+	client.Configure("")
+	err = renewManagedCertificates(client)
+	if err != nil {
+		return configs, err
 	}
 
+	// keep certificates renewed and OCSP stapling updated
+	go maintainAssets(stopChan)
+
 	return configs, nil
 }
 
@@ -121,11 +112,16 @@ func MarkQualified(configs []server.Config) {
 // ObtainCerts obtains certificates for all these configs as long as a certificate does not
 // already exist on disk. It does not modify the configs at all; it only obtains and stores
 // certificates and keys to the disk.
-func ObtainCerts(configs []server.Config, altPort string) error {
-	groupedConfigs := groupConfigsByEmail(configs, altPort != "") // don't prompt user if server already running
+func ObtainCerts(configs []server.Config, allowPrompts bool) error {
+	// We group configs by email so we don't make the same clients over and
+	// over. This has the potential to prompt the user for an email, but we
+	// prevent that by assuming that if we already have a listener that can
+	// proxy ACME challenge requests, then the server is already running and
+	// the operator is no longer present.
+	groupedConfigs := groupConfigsByEmail(configs, allowPrompts)
 
 	for email, group := range groupedConfigs {
-		client, err := newClientPort(email, altPort)
+		client, err := NewACMEClient(email, allowPrompts)
 		if err != nil {
 			return errors.New("error creating client: " + err.Error())
 		}
@@ -135,7 +131,9 @@ func ObtainCerts(configs []server.Config, altPort string) error {
 				continue
 			}
 
-			err := clientObtain(client, []string{cfg.Host}, altPort == "")
+			client.Configure(cfg.BindHost)
+
+			err := client.Obtain([]string{cfg.Host})
 			if err != nil {
 				return err
 			}
@@ -147,15 +145,14 @@ func ObtainCerts(configs []server.Config, altPort string) error {
 
 // groupConfigsByEmail groups configs by the email address to be used by its
 // ACME client. It only includes configs that are marked as fully managed.
-// This is the function that may prompt for an email address, unless skipPrompt
-// is true, in which case it will assume an empty email address.
-func groupConfigsByEmail(configs []server.Config, skipPrompt bool) map[string][]server.Config {
+// If userPresent is true, the operator MAY be prompted for an email address.
+func groupConfigsByEmail(configs []server.Config, userPresent bool) map[string][]server.Config {
 	initMap := make(map[string][]server.Config)
 	for _, cfg := range configs {
 		if !cfg.TLS.Managed {
 			continue
 		}
-		leEmail := getEmail(cfg, skipPrompt)
+		leEmail := getEmail(cfg, userPresent)
 		initMap[leEmail] = append(initMap[leEmail], cfg)
 	}
 	return initMap
@@ -163,50 +160,24 @@ func groupConfigsByEmail(configs []server.Config, skipPrompt bool) map[string][]
 
 // EnableTLS configures each config to use TLS according to default settings.
 // It will only change configs that are marked as managed, and assumes that
-// certificates and keys are already on disk.
-func EnableTLS(configs []server.Config) {
+// certificates and keys are already on disk. If loadCertificates is true,
+// the certificates will be loaded from disk into the cache for this process
+// to use. If false, TLS will still be enabled and configured with default
+// settings, but no certificates will be parsed loaded into the cache, and
+// the returned error value will always be nil.
+func EnableTLS(configs []server.Config, loadCertificates bool) error {
 	for i := 0; i < len(configs); i++ {
 		if !configs[i].TLS.Managed {
 			continue
 		}
 		configs[i].TLS.Enabled = true
-		if configs[i].Host != "" {
-			configs[i].TLS.Certificate = storage.SiteCertFile(configs[i].Host)
-			configs[i].TLS.Key = storage.SiteKeyFile(configs[i].Host)
-		}
-		setup.SetDefaultTLSParams(&configs[i])
-	}
-}
-
-// StapleOCSP staples OCSP responses to each config according to their certificate.
-// This should work for any TLS-enabled config, not just Let's Encrypt ones.
-func StapleOCSP(configs []server.Config) error {
-	for i := 0; i < len(configs); i++ {
-		if configs[i].TLS.Certificate == "" {
-			continue
-		}
-
-		bundleBytes, err := ioutil.ReadFile(configs[i].TLS.Certificate)
+		if loadCertificates && configs[i].Host != "" {
+			_, err := cacheManagedCertificate(configs[i].Host, false)
 			if err != nil {
-			return errors.New("load certificate to staple ocsp: " + err.Error())
-		}
-
-		ocspBytes, ocspResp, err := acme.GetOCSPForCert(bundleBytes)
-		if err == nil {
-			// TODO: We ignore the error if it exists because some certificates
-			// may not have an issuer URL which we should ignore anyway, and
-			// sometimes we get syntax errors in the responses. To reproduce this
-			// behavior, start Caddy with an empty Caddyfile and -log stderr. Then
-			// add a host to the Caddyfile which requires a new LE certificate.
-			// Reload Caddy's config with SIGUSR1, and see the log report that it
-			// obtains the certificate, but then an error:
-			// getting ocsp: asn1: syntax error: sequence truncated
-			// But retrying the reload again sometimes solves the problem. It's flaky...
-			ocspCache[&bundleBytes] = ocspResp
-			if ocspResp.Status == ocsp.Good {
-				configs[i].TLS.OCSPStaple = ocspBytes
+				return err
 			}
 		}
+		setDefaultTLSParams(&configs[i])
 	}
 	return nil
 }
@@ -251,8 +222,7 @@ func MakePlaintextRedirects(allConfigs []server.Config) []server.Config {
 // setting up the config may make it look like it
 // doesn't qualify even though it originally did.
 func ConfigQualifies(cfg server.Config) bool {
-	return cfg.TLS.Certificate == "" && // user could provide their own cert and key
-		cfg.TLS.Key == "" &&
+	return !cfg.TLS.Manual && // user can provide own cert and key
 
 		// user can force-disable automatic HTTPS for this host
 		cfg.Scheme != "http" &&
@@ -297,71 +267,6 @@ func existingCertAndKey(host string) bool {
 	return true
 }
 
-// newClient creates a new ACME client to facilitate communication
-// with the Let's Encrypt CA server on behalf of the user specified
-// by leEmail. As part of this process, a user will be loaded from
-// disk (if already exists) or created new and registered via ACME
-// and saved to the file system for next time.
-func newClient(leEmail string) (*acme.Client, error) {
-	return newClientPort(leEmail, "")
-}
-
-// newClientPort does the same thing as newClient, except it creates a
-// new client with a custom port used for ACME transactions instead of
-// the default port. This is important if the default port is already in
-// use or is not exposed to the public, etc.
-func newClientPort(leEmail, port string) (*acme.Client, error) {
-	// Look up or create the LE user account
-	leUser, err := getUser(leEmail)
-	if err != nil {
-		return nil, err
-	}
-
-	// The client facilitates our communication with the CA server.
-	client, err := acme.NewClient(CAUrl, &leUser, rsaKeySizeToUse)
-	if err != nil {
-		return nil, err
-	}
-	if port != "" {
-		client.SetHTTPAddress(":" + port)
-		client.SetTLSAddress(":" + port)
-	}
-	client.ExcludeChallenges([]acme.Challenge{acme.TLSSNI01, acme.DNS01}) // We can only guarantee http-01 at this time, but tls-01 should work if port is not custom!
-
-	// If not registered, the user must register an account with the CA
-	// and agree to terms
-	if leUser.Registration == nil {
-		reg, err := client.Register()
-		if err != nil {
-			return nil, errors.New("registration error: " + err.Error())
-		}
-		leUser.Registration = reg
-
-		if port == "" { // can't prompt a user who isn't there
-			if !Agreed && reg.TosURL == "" {
-				Agreed = promptUserAgreement(saURL, false) // TODO - latest URL
-			}
-			if !Agreed && reg.TosURL == "" {
-				return nil, errors.New("user must agree to terms")
-			}
-		}
-
-		err = client.AgreeToTOS()
-		if err != nil {
-			saveUser(leUser) // TODO: Might as well try, right? Error check?
-			return nil, errors.New("error agreeing to terms: " + err.Error())
-		}
-
-		// save user to the file system
-		err = saveUser(leUser)
-		if err != nil {
-			return nil, errors.New("could not save user: " + err.Error())
-		}
-	}
-
-	return client, nil
-}
-
 // saveCertResource saves the certificate resource to disk. This
 // includes the certificate file itself, the private key, and the
 // metadata file.
@@ -427,61 +332,18 @@ func redirPlaintextHost(cfg server.Config) server.Config {
 	}
 }
 
-// clientObtain uses client to obtain a single certificate for domains in names. If
-// the user is present to provide an email address, pass in true for allowPrompt,
-// otherwise pass in false. If err == nil, the certificate (and key) will be saved
-// to disk in the storage folder.
-func clientObtain(client *acme.Client, names []string, allowPrompt bool) error {
-	certificate, failures := client.ObtainCertificate(names, true, nil)
-	if len(failures) > 0 {
-		// Error - either try to fix it or report them it to the user and abort
-		var errMsg string             // we'll combine all the failures into a single error message
-		var promptedForAgreement bool // only prompt user for agreement at most once
-
-		for errDomain, obtainErr := range failures {
-			// TODO: Double-check, will obtainErr ever be nil?
-			if tosErr, ok := obtainErr.(acme.TOSError); ok {
-				// Terms of Service agreement error; we can probably deal with this
-				if !Agreed && !promptedForAgreement && allowPrompt { // don't prompt if server is already running
-					Agreed = promptUserAgreement(tosErr.Detail, true) // TODO: Use latest URL
-					promptedForAgreement = true
-				}
-				if Agreed || !allowPrompt {
-					err := client.AgreeToTOS()
-					if err != nil {
-						return errors.New("error agreeing to updated terms: " + err.Error())
-					}
-					return clientObtain(client, names, allowPrompt)
-				}
-			}
-
-			// If user did not agree or it was any other kind of error, just append to the list of errors
-			errMsg += "[" + errDomain + "] failed to get certificate: " + obtainErr.Error() + "\n"
-		}
-		return errors.New(errMsg)
-	}
-
-	// Success - immediately save the certificate resource
-	err := saveCertResource(certificate)
-	if err != nil {
-		return fmt.Errorf("error saving assets for %v: %v", names, err)
-	}
-
-	return nil
-}
-
 // Revoke revokes the certificate for host via ACME protocol.
 func Revoke(host string) error {
 	if !existingCertAndKey(host) {
 		return errors.New("no certificate and key for " + host)
 	}
 
-	email := getEmail(server.Config{Host: host}, false)
+	email := getEmail(server.Config{Host: host}, true)
 	if email == "" {
 		return errors.New("email is required to revoke")
 	}
 
-	client, err := newClient(email)
+	client, err := NewACMEClient(email, true)
 	if err != nil {
 		return err
 	}
@@ -525,7 +387,7 @@ const (
 	AlternatePort = "5033"
 
 	// RenewInterval is how often to check certificates for renewal.
-	RenewInterval = 24 * time.Hour
+	RenewInterval = 6 * time.Hour
 
 	// OCSPInterval is how often to check if OCSP stapling needs updating.
 	OCSPInterval = 1 * time.Hour
@@ -550,8 +412,3 @@ var rsaKeySizeToUse = Rsa2048
 // stopChan is used to signal the maintenance goroutine
 // to terminate.
 var stopChan chan struct{}
-
-// ocspCache maps certificate bundle to OCSP response.
-// It is used during regular OCSP checks to see if the OCSP
-// response needs to be updated.
-var ocspCache = make(map[*[]byte]*ocsp.Response)

caddy/letsencrypt/letsencrypt_test.go → caddy/https/https_test.go

-package letsencrypt
+package https
 
 import (
 	"io/ioutil"
@@ -48,9 +48,9 @@ func TestConfigQualifies(t *testing.T) {
 	}{
 		{server.Config{Host: ""}, true},
 		{server.Config{Host: "localhost"}, false},
+		{server.Config{Host: "123.44.3.21"}, false},
 		{server.Config{Host: "example.com"}, true},
-		{server.Config{Host: "example.com", TLS: server.TLSConfig{Certificate: "cert.pem"}}, false},
-		{server.Config{Host: "example.com", TLS: server.TLSConfig{Key: "key.pem"}}, false},
+		{server.Config{Host: "example.com", TLS: server.TLSConfig{Manual: true}}, false},
 		{server.Config{Host: "example.com", TLS: server.TLSConfig{LetsEncryptEmail: "off"}}, false},
 		{server.Config{Host: "example.com", TLS: server.TLSConfig{LetsEncryptEmail: "foo@bar.com"}}, true},
 		{server.Config{Host: "example.com", Scheme: "http"}, false},
@@ -257,27 +257,14 @@ func TestEnableTLS(t *testing.T) {
 		server.Config{}, // not managed - no changes!
 	}
 
-	EnableTLS(configs)
+	EnableTLS(configs, false)
 
 	if !configs[0].TLS.Enabled {
 		t.Errorf("Expected config 0 to have TLS.Enabled == true, but it was false")
 	}
-	if configs[0].TLS.Certificate == "" {
-		t.Errorf("Expected config 0 to have TLS.Certificate set, but it was empty")
-	}
-	if configs[0].TLS.Key == "" {
-		t.Errorf("Expected config 0 to have TLS.Key set, but it was empty")
-	}
-
 	if configs[1].TLS.Enabled {
 		t.Errorf("Expected config 1 to have TLS.Enabled == false, but it was true")
 	}
-	if configs[1].TLS.Certificate != "" {
-		t.Errorf("Expected config 1 to have TLS.Certificate empty, but it was: %s", configs[1].TLS.Certificate)
-	}
-	if configs[1].TLS.Key != "" {
-		t.Errorf("Expected config 1 to have TLS.Key empty, but it was: %s", configs[1].TLS.Key)
-	}
 }
 
 func TestGroupConfigsByEmail(t *testing.T) {
@@ -316,9 +303,9 @@ func TestMarkQualified(t *testing.T) {
 	// TODO: TestConfigQualifies and this test share the same config list...
 	configs := []server.Config{
 		{Host: "localhost"},
+		{Host: "123.44.3.21"},
 		{Host: "example.com"},
-		{Host: "example.com", TLS: server.TLSConfig{Certificate: "cert.pem"}},
-		{Host: "example.com", TLS: server.TLSConfig{Key: "key.pem"}},
+		{Host: "example.com", TLS: server.TLSConfig{Manual: true}},
 		{Host: "example.com", TLS: server.TLSConfig{LetsEncryptEmail: "off"}},
 		{Host: "example.com", TLS: server.TLSConfig{LetsEncryptEmail: "foo@bar.com"}},
 		{Host: "example.com", Scheme: "http"},

caddy/https/maintain.go

+package https
+
+import (
+	"log"
+	"time"
+
+	"golang.org/x/crypto/ocsp"
+)
+
+// maintainAssets is a permanently-blocking function
+// that loops indefinitely and, on a regular schedule, checks
+// certificates for expiration and initiates a renewal of certs
+// that are expiring soon. It also updates OCSP stapling and
+// performs other maintenance of assets.
+//
+// You must pass in the channel which you'll close when
+// maintenance should stop, to allow this goroutine to clean up
+// after itself and unblock.
+func maintainAssets(stopChan chan struct{}) {
+	renewalTicker := time.NewTicker(RenewInterval)
+	ocspTicker := time.NewTicker(OCSPInterval)
+
+	for {
+		select {
+		case <-renewalTicker.C:
+			log.Println("[INFO] Scanning for expiring certificates")
+			client, err := NewACMEClient("", false) // renewals don't use email
+			if err != nil {
+				log.Printf("[ERROR] Creating client for renewals: %v", err)
+				continue
+			}
+			client.Configure("") // TODO: Bind address of relevant listener, yuck
+			renewManagedCertificates(client)
+			log.Println("[INFO] Done checking certificates")
+		case <-ocspTicker.C:
+			log.Println("[INFO] Scanning for stale OCSP staples")
+			updatePreloadedOCSPStaples()
+			log.Println("[INFO] Done checking OCSP staples")
+		case <-stopChan:
+			renewalTicker.Stop()
+			ocspTicker.Stop()
+			log.Println("[INFO] Stopped background maintenance routine")
+			return
+		}
+	}
+}
+
+func renewManagedCertificates(client *ACMEClient) error {
+	var renewed, deleted []Certificate
+	visitedNames := make(map[string]struct{})
+
+	certCacheMu.RLock()
+	for name, cert := range certCache {
+		if !cert.Managed {
+			continue
+		}
+
+		// the list of names on this cert should never be empty...
+		if cert.Names == nil || len(cert.Names) == 0 {
+			log.Printf("[WARNING] Certificate keyed by '%s' has no names: %v", name, cert.Names)
+			deleted = append(deleted, cert)
+			continue
+		}
+
+		// skip names whose certificate we've already renewed
+		if _, ok := visitedNames[name]; ok {
+			continue
+		}
+		for _, name := range cert.Names {
+			visitedNames[name] = struct{}{}
+		}
+
+		timeLeft := cert.NotAfter.Sub(time.Now().UTC())
+		if timeLeft < renewDurationBefore {
+			log.Printf("[INFO] Certificate for %v expires in %v; attempting renewal", cert.Names, timeLeft)
+			err := client.Renew(cert.Names[0]) // managed certs better have only one name
+			if err != nil {
+				if client.AllowPrompts {
+					// User is present, so stop immediately and report the error
+					certCacheMu.RUnlock()
+					return err
+				}
+				log.Printf("[ERROR] %v", err)
+				if cert.OnDemand {
+					deleted = append(deleted, cert)
+				}
+			} else {
+				renewed = append(renewed, cert)
+			}
+		}
+	}
+	certCacheMu.RUnlock()
+
+	// Apply changes to the cache
+	for _, cert := range renewed {
+		_, err := cacheManagedCertificate(cert.Names[0], cert.OnDemand)
+		if err != nil {
+			if client.AllowPrompts {
+				return err // operator is present, so report error immediately
+			}
+			log.Printf("[ERROR] %v", err)
+		}
+	}
+	for _, cert := range deleted {
+		certCacheMu.Lock()
+		for _, name := range cert.Names {
+			delete(certCache, name)
+		}
+		certCacheMu.Unlock()
+	}
+
+	return nil
+}
+
+func updatePreloadedOCSPStaples() {
+	// Create a temporary place to store updates
+	// until we release the potentially slow read
+	// lock so we can use a quick write lock.
+	type ocspUpdate struct {
+		rawBytes       []byte
+		parsedResponse *ocsp.Response
+	}
+	updated := make(map[string]ocspUpdate)
+
+	certCacheMu.RLock()
+	for name, cert := range certCache {
+		// we update OCSP for managed and un-managed certs here, but only
+		// if it has OCSP stapled and only for pre-loaded certificates
+		if cert.OnDemand || cert.OCSP == nil {
+			continue
+		}
+
+		// start checking OCSP staple about halfway through validity period for good measure
+		oldNextUpdate := cert.OCSP.NextUpdate
+		refreshTime := cert.OCSP.ThisUpdate.Add(oldNextUpdate.Sub(cert.OCSP.ThisUpdate) / 2)
+
+		// only check for updated OCSP validity window if the refresh time is
+		// in the past and the certificate is not expired
+		if time.Now().After(refreshTime) && time.Now().Before(cert.NotAfter) {
+			err := stapleOCSP(&cert, nil)
+			if err != nil {
+				log.Printf("[ERROR] Checking OCSP for %s: %v", name, err)
+				continue
+			}
+
+			// if the OCSP response has been updated, we use it
+			if oldNextUpdate != cert.OCSP.NextUpdate {
+				log.Printf("[INFO] Moving validity period of OCSP staple for %s from %v to %v",
+					name, oldNextUpdate, cert.OCSP.NextUpdate)
+				updated[name] = ocspUpdate{rawBytes: cert.Certificate.OCSPStaple, parsedResponse: cert.OCSP}
+			}
+		}
+	}
+	certCacheMu.RUnlock()
+
+	// This write lock should be brief since we have all the info we need now.
+	certCacheMu.Lock()
+	for name, update := range updated {
+		cert := certCache[name]
+		cert.OCSP = update.parsedResponse
+		cert.Certificate.OCSPStaple = update.rawBytes
+		certCache[name] = cert
+	}
+	certCacheMu.Unlock()
+}
+
+// renewDurationBefore is how long before expiration to renew certificates.
+const renewDurationBefore = (24 * time.Hour) * 30

caddy/setup/tls.go → caddy/https/setup.go

-package setup
+package https
 
 import (
+	"bytes"
 	"crypto/tls"
+	"encoding/pem"
+	"io/ioutil"
 	"log"
+	"os"
+	"path/filepath"
 	"strings"
 
+	"github.com/mholt/caddy/caddy/setup"
 	"github.com/mholt/caddy/middleware"
 	"github.com/mholt/caddy/server"
 )
 
-// TLS sets up the TLS configuration (but does not activate Let's Encrypt; that is handled elsewhere).
-func TLS(c *Controller) (middleware.Middleware, error) {
+// Setup sets up the TLS configuration and installs certificates that
+// are specified by the user in the config file. All the automatic HTTPS
+// stuff comes later outside of this function.
+func Setup(c *setup.Controller) (middleware.Middleware, error) {
 	if c.Scheme == "http" {
 		c.TLS.Enabled = false
 		log.Printf("[WARNING] TLS disabled for %s://%s.", c.Scheme, c.Address())
@@ -19,18 +27,21 @@ func TLS(c *Controller) (middleware.Middleware, error) {
 	}
 
 	for c.Next() {
+		var certificateFile, keyFile, loadDir string
+
 		args := c.RemainingArgs()
 		switch len(args) {
 		case 1:
 			c.TLS.LetsEncryptEmail = args[0]
 
-			// user can force-disable LE activation this way
+			// user can force-disable managed TLS this way
 			if c.TLS.LetsEncryptEmail == "off" {
 				c.TLS.Enabled = false
 			}
 		case 2:
-			c.TLS.Certificate = args[0]
-			c.TLS.Key = args[1]
+			certificateFile = args[0]
+			keyFile = args[1]
+			c.TLS.Manual = true
 		}
 
 		// Optional block with extra parameters
@@ -66,9 +77,9 @@ func TLS(c *Controller) (middleware.Middleware, error) {
 				if len(c.TLS.ClientCerts) == 0 {
 					return nil, c.ArgErr()
 				}
-			// TODO: Allow this? It's a bad idea to allow HTTP. If we do this, make sure invoking tls at all (even manually) also sets up a redirect if possible?
-			// case "allow_http":
-			// 	c.TLS.DisableHTTPRedir = true
+			case "load":
+				c.Args(&loadDir)
+				c.TLS.Manual = true
 			default:
 				return nil, c.Errf("Unknown keyword '%s'", c.Val())
 			}
@@ -78,18 +89,112 @@ func TLS(c *Controller) (middleware.Middleware, error) {
 		if len(args) == 0 && !hadBlock {
 			return nil, c.ArgErr()
 		}
+
+		// don't load certificates unless we're supposed to
+		if !c.TLS.Enabled || !c.TLS.Manual {
+			continue
+		}
+
+		// load a single certificate and key, if specified
+		if certificateFile != "" && keyFile != "" {
+			err := cacheUnmanagedCertificatePEMFile(certificateFile, keyFile)
+			if err != nil {
+				return nil, c.Errf("Unable to load certificate and key files for %s: %v", c.Host, err)
+			}
+			log.Printf("[INFO] Successfully loaded TLS assets from %s and %s", certificateFile, keyFile)
+		}
+
+		// load a directory of certificates, if specified
+		// modeled after haproxy: https://cbonte.github.io/haproxy-dconv/configuration-1.5.html#5.1-crt
+		if loadDir != "" {
+			err := filepath.Walk(loadDir, func(path string, info os.FileInfo, err error) error {
+				if err != nil {
+					log.Printf("[WARNING] Unable to traverse into %s; skipping", path)
+					return nil
+				}
+				if info.IsDir() {
+					return nil
+				}
+				if strings.HasSuffix(strings.ToLower(info.Name()), ".pem") {
+					certBuilder, keyBuilder := new(bytes.Buffer), new(bytes.Buffer)
+					var foundKey bool
+
+					bundle, err := ioutil.ReadFile(path)
+					if err != nil {
+						return err
+					}
+
+					for {
+						// Decode next block so we can see what type it is
+						var derBlock *pem.Block
+						derBlock, bundle = pem.Decode(bundle)
+						if derBlock == nil {
+							break
+						}
+
+						if derBlock.Type == "CERTIFICATE" {
+							// Re-encode certificate as PEM, appending to certificate chain
+							pem.Encode(certBuilder, derBlock)
+						} else if derBlock.Type == "EC PARAMETERS" {
+							// EC keys are composed of two blocks: parameters and key
+							// (parameter block should come first)
+							if !foundKey {
+								// Encode parameters
+								pem.Encode(keyBuilder, derBlock)
+
+								// Key must immediately follow
+								derBlock, bundle = pem.Decode(bundle)
+								if derBlock == nil || derBlock.Type != "EC PRIVATE KEY" {
+									return c.Errf("%s: expected elliptic private key to immediately follow EC parameters", path)
+								}
+								pem.Encode(keyBuilder, derBlock)
+								foundKey = true
+							}
+						} else if derBlock.Type == "PRIVATE KEY" || strings.HasSuffix(derBlock.Type, " PRIVATE KEY") {
+							// RSA key
+							if !foundKey {
+								pem.Encode(keyBuilder, derBlock)
+								foundKey = true
+							}
+						} else {
+							return c.Errf("%s: unrecognized PEM block type: %s", path, derBlock.Type)
+						}
+					}
+
+					certPEMBytes, keyPEMBytes := certBuilder.Bytes(), keyBuilder.Bytes()
+					if len(certPEMBytes) == 0 {
+						return c.Errf("%s: failed to parse PEM data", path)
+					}
+					if len(keyPEMBytes) == 0 {
+						return c.Errf("%s: no private key block found", path)
+					}
+
+					err = cacheUnmanagedCertificatePEMBytes(certPEMBytes, keyPEMBytes)
+					if err != nil {
+						return c.Errf("%s: failed to load cert and key for %s: %v", path, c.Host, err)
+					}
+					log.Printf("[INFO] Successfully loaded TLS assets from %s", path)
+				}
+				return nil
+			})
+			if err != nil {
+				return nil, err
+			}
+		}
 	}
 
-	SetDefaultTLSParams(c.Config)
+	setDefaultTLSParams(c.Config)
 
 	return nil, nil
 }
 
-// SetDefaultTLSParams sets the default TLS cipher suites, protocol versions,
+// setDefaultTLSParams sets the default TLS cipher suites, protocol versions,
 // and server preferences of a server.Config if they were not previously set
-// (it does not overwrite; only fills in missing values).
-func SetDefaultTLSParams(c *server.Config) {
-	// If no ciphers provided, use all that Caddy supports for the protocol
+// (it does not overwrite; only fills in missing values). It will also set the
+// port to 443 if not already set, TLS is enabled, TLS is manual, and the host
+// does not equal localhost.
+func setDefaultTLSParams(c *server.Config) {
+	// If no ciphers provided, use default list
 	if len(c.TLS.Ciphers) == 0 {
 		c.TLS.Ciphers = defaultCiphers
 	}
@@ -111,14 +216,14 @@ func SetDefaultTLSParams(c *server.Config) {
 
 	// Default TLS port is 443; only use if port is not manually specified,
 	// TLS is enabled, and the host is not localhost
-	if c.Port == "" && c.TLS.Enabled && c.Host != "localhost" {
+	if c.Port == "" && c.TLS.Enabled && !c.TLS.Manual && c.Host != "localhost" {
 		c.Port = "443"
 	}
 }
 
-// Map of supported protocols
-// SSLv3 will be not supported in future release
-// HTTP/2 only supports TLS 1.2 and higher
+// Map of supported protocols.
+// SSLv3 will be not supported in future release.
+// HTTP/2 only supports TLS 1.2 and higher.
 var supportedProtocols = map[string]uint16{
 	"ssl3.0": tls.VersionSSL30,
 	"tls1.0": tls.VersionTLS10,

caddy/setup/tls_test.go → caddy/https/setup_test.go

-package setup
+package https
 
 import (
 	"crypto/tls"
+	"io/ioutil"
+	"log"
+	"os"
 	"testing"
+
+	"github.com/mholt/caddy/caddy/setup"
 )
 
-func TestTLSParseBasic(t *testing.T) {
-	c := NewTestController(`tls cert.pem key.pem`)
+func TestMain(m *testing.M) {
+	// Write test certificates to disk before tests, and clean up
+	// when we're done.
+	err := ioutil.WriteFile(certFile, testCert, 0644)
+	if err != nil {
+		log.Fatal(err)
+	}
+	err = ioutil.WriteFile(keyFile, testKey, 0644)
+	if err != nil {
+		os.Remove(certFile)
+		log.Fatal(err)
+	}
+
+	result := m.Run()
 
-	_, err := TLS(c)
+	os.Remove(certFile)
+	os.Remove(keyFile)
+	os.Exit(result)
+}
+
+func TestSetupParseBasic(t *testing.T) {
+	c := setup.NewTestController(`tls ` + certFile + ` ` + keyFile + ``)
+
+	_, err := Setup(c)
 	if err != nil {
 		t.Errorf("Expected no errors, got: %v", err)
 	}
 
 	// Basic checks
-	if c.TLS.Certificate != "cert.pem" {
-		t.Errorf("Expected certificate arg to be 'cert.pem', was '%s'", c.TLS.Certificate)
-	}
-	if c.TLS.Key != "key.pem" {
-		t.Errorf("Expected key arg to be 'key.pem', was '%s'", c.TLS.Key)
+	if !c.TLS.Manual {
+		t.Error("Expected TLS Manual=true, but was false")
 	}
 	if !c.TLS.Enabled {
 		t.Error("Expected TLS Enabled=true, but was false")
@@ -63,23 +85,23 @@ func TestTLSParseBasic(t *testing.T) {
 	}
 }
 
-func TestTLSParseIncompleteParams(t *testing.T) {
+func TestSetupParseIncompleteParams(t *testing.T) {
 	// Using tls without args is an error because it's unnecessary.
-	c := NewTestController(`tls`)
-	_, err := TLS(c)
+	c := setup.NewTestController(`tls`)
+	_, err := Setup(c)
 	if err == nil {
 		t.Error("Expected an error, but didn't get one")
 	}
 }
 
-func TestTLSParseWithOptionalParams(t *testing.T) {
-	params := `tls cert.crt cert.key {
+func TestSetupParseWithOptionalParams(t *testing.T) {
+	params := `tls ` + certFile + ` ` + keyFile + ` {
             protocols ssl3.0 tls1.2
             ciphers RSA-3DES-EDE-CBC-SHA RSA-AES256-CBC-SHA ECDHE-RSA-AES128-GCM-SHA256
         }`
-	c := NewTestController(params)
+	c := setup.NewTestController(params)
 
-	_, err := TLS(c)
+	_, err := Setup(c)
 	if err != nil {
 		t.Errorf("Expected no errors, got: %v", err)
 	}
@@ -97,13 +119,13 @@ func TestTLSParseWithOptionalParams(t *testing.T) {
 	}
 }
 
-func TestTLSDefaultWithOptionalParams(t *testing.T) {
+func TestSetupDefaultWithOptionalParams(t *testing.T) {
 	params := `tls {
             ciphers RSA-3DES-EDE-CBC-SHA
         }`
-	c := NewTestController(params)
+	c := setup.NewTestController(params)
 
-	_, err := TLS(c)
+	_, err := Setup(c)
 	if err != nil {
 		t.Errorf("Expected no errors, got: %v", err)
 	}
@@ -113,7 +135,7 @@ func TestTLSDefaultWithOptionalParams(t *testing.T) {
 }
 
 // TODO: If we allow this... but probably not a good idea.
-// func TestTLSDisableHTTPRedirect(t *testing.T) {
+// func TestSetupDisableHTTPRedirect(t *testing.T) {
 // 	c := NewTestController(`tls {
 // 	    allow_http
 // 	}`)
@@ -126,34 +148,34 @@ func TestTLSDefaultWithOptionalParams(t *testing.T) {
 // 	}
 // }
 
-func TestTLSParseWithWrongOptionalParams(t *testing.T) {
+func TestSetupParseWithWrongOptionalParams(t *testing.T) {
 	// Test protocols wrong params
-	params := `tls cert.crt cert.key {
+	params := `tls ` + certFile + ` ` + keyFile + ` {
 			protocols ssl tls
 		}`
-	c := NewTestController(params)
-	_, err := TLS(c)
+	c := setup.NewTestController(params)
+	_, err := Setup(c)
 	if err == nil {
 		t.Errorf("Expected errors, but no error returned")
 	}
 
 	// Test ciphers wrong params
-	params = `tls cert.crt cert.key {
+	params = `tls ` + certFile + ` ` + keyFile + ` {
 			ciphers not-valid-cipher
 		}`
-	c = NewTestController(params)
-	_, err = TLS(c)
+	c = setup.NewTestController(params)
+	_, err = Setup(c)
 	if err == nil {
 		t.Errorf("Expected errors, but no error returned")
 	}
 }
 
-func TestTLSParseWithClientAuth(t *testing.T) {
-	params := `tls cert.crt cert.key {
+func TestSetupParseWithClientAuth(t *testing.T) {
+	params := `tls ` + certFile + ` ` + keyFile + ` {
 			clients client_ca.crt client2_ca.crt
 		}`
-	c := NewTestController(params)
-	_, err := TLS(c)
+	c := setup.NewTestController(params)
+	_, err := Setup(c)
 	if err != nil {
 		t.Errorf("Expected no errors, got: %v", err)
 	}
@@ -169,12 +191,40 @@ func TestTLSParseWithClientAuth(t *testing.T) {
 	}
 
 	// Test missing client cert file
-	params = `tls cert.crt cert.key {
+	params = `tls ` + certFile + ` ` + keyFile + ` {
 			clients
 		}`
-	c = NewTestController(params)
-	_, err = TLS(c)
+	c = setup.NewTestController(params)
+	_, err = Setup(c)
 	if err == nil {
 		t.Errorf("Expected an error, but no error returned")
 	}
 }
+
+const (
+	certFile = "test_cert.pem"
+	keyFile  = "test_key.pem"
+)
+
+var testCert = []byte(`-----BEGIN CERTIFICATE-----
+MIIBkjCCATmgAwIBAgIJANfFCBcABL6LMAkGByqGSM49BAEwFDESMBAGA1UEAxMJ
+bG9jYWxob3N0MB4XDTE2MDIxMDIyMjAyNFoXDTE4MDIwOTIyMjAyNFowFDESMBAG
+A1UEAxMJbG9jYWxob3N0MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEs22MtnG7
+9K1mvIyjEO9GLx7BFD0tBbGnwQ0VPsuCxC6IeVuXbQDLSiVQvFZ6lUszTlczNxVk
+pEfqrM6xAupB7qN1MHMwHQYDVR0OBBYEFHxYDvAxUwL4XrjPev6qZ/BiLDs5MEQG
+A1UdIwQ9MDuAFHxYDvAxUwL4XrjPev6qZ/BiLDs5oRikFjAUMRIwEAYDVQQDEwls
+b2NhbGhvc3SCCQDXxQgXAAS+izAMBgNVHRMEBTADAQH/MAkGByqGSM49BAEDSAAw
+RQIgRvBqbyJM2JCJqhA1FmcoZjeMocmhxQHTt1c+1N2wFUgCIQDtvrivbBPA688N
+Qh3sMeAKNKPsx5NxYdoWuu9KWcKz9A==
+-----END CERTIFICATE-----
+`)
+
+var testKey = []byte(`-----BEGIN EC PARAMETERS-----
+BggqhkjOPQMBBw==
+-----END EC PARAMETERS-----
+-----BEGIN EC PRIVATE KEY-----
+MHcCAQEEIGLtRmwzYVcrH3J0BnzYbGPdWVF10i9p6mxkA4+b2fURoAoGCCqGSM49
+AwEHoUQDQgAEs22MtnG79K1mvIyjEO9GLx7BFD0tBbGnwQ0VPsuCxC6IeVuXbQDL
+SiVQvFZ6lUszTlczNxVkpEfqrM6xAupB7g==
+-----END EC PRIVATE KEY-----
+`)

caddy/letsencrypt/storage.go → caddy/https/storage.go

-package letsencrypt
+package https
 
 import (
 	"path/filepath"

caddy/letsencrypt/storage_test.go → caddy/https/storage_test.go

-package letsencrypt
+package https
 
 import (
 	"path/filepath"

caddy/letsencrypt/user.go → caddy/https/user.go

-package letsencrypt
+package https
 
 import (
 	"bufio"
@@ -41,7 +41,7 @@ func (u User) GetPrivateKey() *rsa.PrivateKey {
 // getUser loads the user with the given email from disk.
 // If the user does not exist, it will create a new one,
 // but it does NOT save new users to the disk or register
-// them via ACME.
+// them via ACME. It does NOT prompt the user.
 func getUser(email string) (User, error) {
 	var user User
 
@@ -72,7 +72,8 @@ func getUser(email string) (User, error) {
 }
 
 // saveUser persists a user's key and account registration
-// to the file system. It does NOT register the user via ACME.
+// to the file system. It does NOT register the user via ACME
+// or prompt the user.
 func saveUser(user User) error {
 	// make user account folder
 	err := os.MkdirAll(storage.User(user.Email), 0700)
@@ -99,7 +100,7 @@ func saveUser(user User) error {
 // with a new private key. This function does NOT save the
 // user to disk or register it via ACME. If you want to use
 // a user account that might already exist, call getUser
-// instead.
+// instead. It does NOT prompt the user.
 func newUser(email string) (User, error) {
 	user := User{Email: email}
 	privateKey, err := rsa.GenerateKey(rand.Reader, rsaKeySizeToUse)
@@ -114,10 +115,10 @@ func newUser(email string) (User, error) {
 // address from the user to use for TLS for cfg. If it
 // cannot get an email address, it returns empty string.
 // (It will warn the user of the consequences of an
-// empty email.) If skipPrompt is true, the user will
-// NOT be prompted and an empty email will be returned
-// instead.
-func getEmail(cfg server.Config, skipPrompt bool) string {
+// empty email.) This function MAY prompt the user for
+// input. If userPresent is false, the operator will
+// NOT be prompted and an empty email may be returned.
+func getEmail(cfg server.Config, userPresent bool) string {
 	// First try the tls directive from the Caddyfile
 	leEmail := cfg.TLS.LetsEncryptEmail
 	if leEmail == "" {
@@ -135,11 +136,12 @@ func getEmail(cfg server.Config, skipPrompt bool) string {
 				}
 				if mostRecent == nil || dir.ModTime().After(mostRecent.ModTime()) {
 					leEmail = dir.Name()
+					DefaultEmail = leEmail // save for next time
 				}
 			}
 		}
 	}
-	if leEmail == "" && !skipPrompt {
+	if leEmail == "" && userPresent {
 		// Alas, we must bother the user and ask for an email address;
 		// if they proceed they also agree to the SA.
 		reader := bufio.NewReader(stdin)

caddy/letsencrypt/user_test.go → caddy/https/user_test.go

-package letsencrypt
+package https
 
 import (
 	"bytes"
@@ -140,13 +140,13 @@ func TestGetEmail(t *testing.T) {
 			LetsEncryptEmail: "test1@foo.com",
 		},
 	}
-	actual := getEmail(config, false)
+	actual := getEmail(config, true)
 	if actual != "test1@foo.com" {
 		t.Errorf("Did not get correct email from config; expected '%s' but got '%s'", "test1@foo.com", actual)
 	}
 
 	// Test2: Use default email from flag (or user previously typing it)
-	actual = getEmail(server.Config{}, false)
+	actual = getEmail(server.Config{}, true)
 	if actual != DefaultEmail {
 		t.Errorf("Did not get correct email from config; expected '%s' but got '%s'", DefaultEmail, actual)
 	}
@@ -158,7 +158,7 @@ func TestGetEmail(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Could not simulate user input, error: %v", err)
 	}
-	actual = getEmail(server.Config{}, false)
+	actual = getEmail(server.Config{}, true)
 	if actual != "test3@foo.com" {
 		t.Errorf("Did not get correct email from user input prompt; expected '%s' but got '%s'", "test3@foo.com", actual)
 	}
@@ -189,7 +189,7 @@ func TestGetEmail(t *testing.T) {
 			t.Fatalf("Could not change user folder mod time for '%s': %v", eml, err)
 		}
 	}
-	actual = getEmail(server.Config{}, false)
+	actual = getEmail(server.Config{}, true)
 	if actual != "test4-3@foo.com" {
 		t.Errorf("Did not get correct email from storage; expected '%s' but got '%s'", "test4-3@foo.com", actual)
 	}

caddy/letsencrypt/handshake.go

-package letsencrypt
-
-import (
-	"crypto/tls"
-	"errors"
-	"strings"
-	"sync"
-
-	"github.com/mholt/caddy/server"
-)
-
-// GetCertificateDuringHandshake is a function that gets a certificate during a TLS handshake.
-// It first checks an in-memory cache in case the cert was requested before, then tries to load
-// a certificate in the storage folder from disk. If it can't find an existing certificate, it
-// will try to obtain one using ACME, which will then be stored on disk and cached in memory.
-//
-// This function is safe for use by multiple concurrent goroutines.
-func GetCertificateDuringHandshake(clientHello *tls.ClientHelloInfo) (*tls.Certificate, error) {
-	// Utility function to help us load a cert from disk and put it in the cache if successful
-	loadCertFromDisk := func(domain string) *tls.Certificate {
-		cert, err := tls.LoadX509KeyPair(storage.SiteCertFile(domain), storage.SiteKeyFile(domain))
-		if err == nil {
-			certCacheMu.Lock()
-			if len(certCache) < 10000 { // limit size of cache to prevent a ridiculous, unusual kind of attack
-				certCache[domain] = &cert
-			}
-			certCacheMu.Unlock()
-			return &cert
-		}
-		return nil
-	}
-
-	// First check our in-memory cache to see if we've already loaded it
-	certCacheMu.RLock()
-	cert := server.GetCertificateFromCache(clientHello, certCache)
-	certCacheMu.RUnlock()
-	if cert != nil {
-		return cert, nil
-	}
-
-	// Then check to see if we already have one on disk; if we do, add it to cache and use it
-	name := strings.ToLower(clientHello.ServerName)
-	cert = loadCertFromDisk(name)
-	if cert != nil {
-		return cert, nil
-	}
-
-	// Only option left is to get one from LE, but the name has to qualify first
-	if !HostQualifies(name) {
-		return nil, nil
-	}
-
-	// By this point, we need to obtain one from the CA. We must protect this process
-	// from happening concurrently, so synchronize.
-	obtainCertWaitGroupsMutex.Lock()
-	wg, ok := obtainCertWaitGroups[name]
-	if ok {
-		// lucky us -- another goroutine is already obtaining the certificate.
-		// wait for it to finish obtaining the cert and then we'll use it.
-		obtainCertWaitGroupsMutex.Unlock()
-		wg.Wait()
-		return GetCertificateDuringHandshake(clientHello)
-	}
-
-	// looks like it's up to us to do all the work and obtain the cert
-	wg = new(sync.WaitGroup)
-	wg.Add(1)
-	obtainCertWaitGroups[name] = wg
-	obtainCertWaitGroupsMutex.Unlock()
-
-	// Unblock waiters and delete waitgroup when we return
-	defer func() {
-		obtainCertWaitGroupsMutex.Lock()
-		wg.Done()
-		delete(obtainCertWaitGroups, name)
-		obtainCertWaitGroupsMutex.Unlock()
-	}()
-
-	// obtain cert
-	client, err := newClientPort(DefaultEmail, AlternatePort)
-	if err != nil {
-		return nil, errors.New("error creating client: " + err.Error())
-	}
-	err = clientObtain(client, []string{name}, false)
-	if err != nil {
-		return nil, err
-	}
-
-	// load certificate into memory and return it
-	return loadCertFromDisk(name), nil
-}
-
-// obtainCertWaitGroups is used to coordinate obtaining certs for each hostname.
-var obtainCertWaitGroups = make(map[string]*sync.WaitGroup)
-var obtainCertWaitGroupsMutex sync.Mutex
-
-// certCache stores certificates that have been obtained in memory.
-var certCache = make(map[string]*tls.Certificate)
-var certCacheMu sync.RWMutex

caddy/letsencrypt/maintain.go

-package letsencrypt
-
-import (
-	"encoding/json"
-	"io/ioutil"
-	"log"
-	"time"
-
-	"github.com/mholt/caddy/server"
-	"github.com/xenolf/lego/acme"
-)
-
-// OnChange is a callback function that will be used to restart
-// the application or the part of the application that uses
-// the certificates maintained by this package. When at least
-// one certificate is renewed or an OCSP status changes, this
-// function will be called.
-var OnChange func() error
-
-// maintainAssets is a permanently-blocking function
-// that loops indefinitely and, on a regular schedule, checks
-// certificates for expiration and initiates a renewal of certs
-// that are expiring soon. It also updates OCSP stapling and
-// performs other maintenance of assets.
-//
-// You must pass in the server configs to maintain and the channel
-// which you'll close when maintenance should stop, to allow this
-// goroutine to clean up after itself and unblock.
-func maintainAssets(configs []server.Config, stopChan chan struct{}) {
-	renewalTicker := time.NewTicker(RenewInterval)
-	ocspTicker := time.NewTicker(OCSPInterval)
-
-	for {
-		select {
-		case <-renewalTicker.C:
-			n, errs := renewCertificates(configs, true)
-			if len(errs) > 0 {
-				for _, err := range errs {
-					log.Printf("[ERROR] Certificate renewal: %v", err)
-				}
-			}
-			// even if there was an error, some renewals may have succeeded
-			if n > 0 && OnChange != nil {
-				err := OnChange()
-				if err != nil {
-					log.Printf("[ERROR] OnChange after cert renewal: %v", err)
-				}
-			}
-		case <-ocspTicker.C:
-			for bundle, oldResp := range ocspCache {
-				// start checking OCSP staple about halfway through validity period for good measure
-				refreshTime := oldResp.ThisUpdate.Add(oldResp.NextUpdate.Sub(oldResp.ThisUpdate) / 2)
-
-				// only check for updated OCSP validity window if refreshTime is in the past
-				if time.Now().After(refreshTime) {
-					_, newResp, err := acme.GetOCSPForCert(*bundle)
-					if err != nil {
-						log.Printf("[ERROR] Checking OCSP for bundle: %v", err)
-						continue
-					}
-
-					// we're not looking for different status, just a more future expiration
-					if newResp.NextUpdate != oldResp.NextUpdate {
-						if OnChange != nil {
-							log.Printf("[INFO] Updating OCSP stapling to extend validity period to %v", newResp.NextUpdate)
-							err := OnChange()
-							if err != nil {
-								log.Printf("[ERROR] OnChange after OCSP trigger: %v", err)
-							}
-							break
-						}
-					}
-				}
-			}
-		case <-stopChan:
-			renewalTicker.Stop()
-			ocspTicker.Stop()
-			return
-		}
-	}
-}
-
-// renewCertificates loops through all configured site and
-// looks for certificates to renew. Nothing is mutated
-// through this function; all changes happen directly on disk.
-// It returns the number of certificates renewed and any errors
-// that occurred. It only performs a renewal if necessary.
-// If useCustomPort is true, a custom port will be used, and
-// whatever is listening at 443 better proxy ACME requests to it.
-// Otherwise, the acme package will create its own listener on 443.
-func renewCertificates(configs []server.Config, useCustomPort bool) (int, []error) {
-	log.Printf("[INFO] Checking certificates for %d hosts", len(configs))
-	var errs []error
-	var n int
-
-	for _, cfg := range configs {
-		// Host must be TLS-enabled and have existing assets managed by LE
-		if !cfg.TLS.Enabled || !existingCertAndKey(cfg.Host) {
-			continue
-		}
-
-		// Read the certificate and get the NotAfter time.
-		certBytes, err := ioutil.ReadFile(storage.SiteCertFile(cfg.Host))
-		if err != nil {
-			errs = append(errs, err)
-			continue // still have to check other certificates
-		}
-		expTime, err := acme.GetPEMCertExpiration(certBytes)
-		if err != nil {
-			errs = append(errs, err)
-			continue
-		}
-
-		// The time returned from the certificate is always in UTC.
-		// So calculate the time left with local time as UTC.
-		// Directly convert it to days for the following checks.
-		daysLeft := int(expTime.Sub(time.Now().UTC()).Hours() / 24)
-
-		// Renew if getting close to expiration.
-		if daysLeft <= renewDaysBefore {
-			log.Printf("[INFO] Certificate for %s has %d days remaining; attempting renewal", cfg.Host, daysLeft)
-			var client *acme.Client
-			if useCustomPort {
-				client, err = newClientPort("", AlternatePort) // email not used for renewal
-			} else {
-				client, err = newClient("")
-			}
-			if err != nil {
-				errs = append(errs, err)
-				continue
-			}
-
-			// Read and set up cert meta, required for renewal
-			metaBytes, err := ioutil.ReadFile(storage.SiteMetaFile(cfg.Host))
-			if err != nil {
-				errs = append(errs, err)
-				continue
-			}
-			privBytes, err := ioutil.ReadFile(storage.SiteKeyFile(cfg.Host))
-			if err != nil {
-				errs = append(errs, err)
-				continue
-			}
-			var certMeta acme.CertificateResource
-			err = json.Unmarshal(metaBytes, &certMeta)
-			certMeta.Certificate = certBytes
-			certMeta.PrivateKey = privBytes
-
-			// Renew certificate
-		Renew:
-			newCertMeta, err := client.RenewCertificate(certMeta, true)
-			if err != nil {
-				if _, ok := err.(acme.TOSError); ok {
-					err := client.AgreeToTOS()
-					if err != nil {
-						errs = append(errs, err)
-					}
-					goto Renew
-				}
-
-				time.Sleep(10 * time.Second)
-				newCertMeta, err = client.RenewCertificate(certMeta, true)
-				if err != nil {
-					errs = append(errs, err)
-					continue
-				}
-			}
-
-			saveCertResource(newCertMeta)
-			n++
-		} else if daysLeft <= renewDaysBefore+7 && daysLeft >= renewDaysBefore+6 {
-			log.Printf("[WARNING] Certificate for %s has %d days remaining; will automatically renew when %d days remain\n", cfg.Host, daysLeft, renewDaysBefore)
-		}
-	}
-
-	return n, errs
-}
-
-// renewDaysBefore is how many days before expiration to renew certificates.
-const renewDaysBefore = 14

caddy/restart.go

@@ -12,7 +12,7 @@ import (
 	"os/exec"
 	"path"
 
-	"github.com/mholt/caddy/caddy/letsencrypt"
+	"github.com/mholt/caddy/caddy/https"
 )
 
 func init() {
@@ -133,13 +133,15 @@ func getCertsForNewCaddyfile(newCaddyfile Input) error {
 	}
 
 	// first mark the configs that are qualified for managed TLS
-	letsencrypt.MarkQualified(configs)
+	https.MarkQualified(configs)
 
-	// we must make sure port is set before we group by bind address
-	letsencrypt.EnableTLS(configs)
+	// since we group by bind address to obtain certs, we must call
+	// EnableTLS to make sure the port is set properly first
+	// (can ignore error since we aren't actually using the certs)
+	https.EnableTLS(configs, false)
 
 	// place certs on the disk
-	err = letsencrypt.ObtainCerts(configs, letsencrypt.AlternatePort)
+	err = https.ObtainCerts(configs, false)
 	if err != nil {
 		return errors.New("obtaining certs: " + err.Error())
 	}

main.go

@@ -13,7 +13,7 @@ import (
 	"time"
 
 	"github.com/mholt/caddy/caddy"
-	"github.com/mholt/caddy/caddy/letsencrypt"
+	"github.com/mholt/caddy/caddy/https"
 	"github.com/xenolf/lego/acme"
 )
 
@@ -32,14 +32,14 @@ const (
 
 func init() {
 	caddy.TrapSignals()
-	flag.BoolVar(&letsencrypt.Agreed, "agree", false, "Agree to Let's Encrypt Subscriber Agreement")
-	flag.StringVar(&letsencrypt.CAUrl, "ca", "https://acme-v01.api.letsencrypt.org/directory", "Certificate authority ACME server")
+	flag.BoolVar(&https.Agreed, "agree", false, "Agree to Let's Encrypt Subscriber Agreement")
+	flag.StringVar(&https.CAUrl, "ca", "https://acme-v01.api.letsencrypt.org/directory", "Certificate authority ACME server")
 	flag.StringVar(&conf, "conf", "", "Configuration file to use (default="+caddy.DefaultConfigFile+")")
 	flag.StringVar(&cpu, "cpu", "100%", "CPU cap")
-	flag.StringVar(&letsencrypt.DefaultEmail, "email", "", "Default Let's Encrypt account email address")
+	flag.StringVar(&https.DefaultEmail, "email", "", "Default Let's Encrypt account email address")
 	flag.DurationVar(&caddy.GracefulTimeout, "grace", 5*time.Second, "Maximum duration of graceful shutdown")
 	flag.StringVar(&caddy.Host, "host", caddy.DefaultHost, "Default host")
-	flag.BoolVar(&caddy.HTTP2, "http2", true, "HTTP/2 support") // TODO: temporary flag until http2 merged into std lib
+	flag.BoolVar(&caddy.HTTP2, "http2", true, "HTTP/2 support")
 	flag.StringVar(&logfile, "log", "", "Process log file")
 	flag.StringVar(&caddy.PidFile, "pidfile", "", "Path to write pid file")
 	flag.StringVar(&caddy.Port, "port", caddy.DefaultPort, "Default port")
@@ -73,7 +73,7 @@ func main() {
 	}
 
 	if revoke != "" {
-		err := letsencrypt.Revoke(revoke)
+		err := https.Revoke(revoke)
 		if err != nil {
 			log.Fatal(err)
 		}

server/config.go

@@ -66,12 +66,9 @@ func (c Config) Address() string {
 // TLSConfig describes how TLS should be configured and used.
 type TLSConfig struct {
 	Enabled                  bool
-	Certificate      string
-	Key              string
 	LetsEncryptEmail         string
 	Managed                  bool // will be set to true if config qualifies for automatic, managed TLS
-	//DisableHTTPRedir         bool // TODO: not a good idea - should we really allow it?
-	OCSPStaple               []byte
+	Manual                   bool // will be set to true if user provides the cert and key files
 	Ciphers                  []uint16
 	ProtocolMinVersion       uint16
 	ProtocolMaxVersion       uint16

server/server.go

@@ -13,7 +13,6 @@ import (
 	"net/http"
 	"os"
 	"runtime"
-	"strings"
 	"sync"
 	"time"
 )
@@ -25,8 +24,9 @@ import (
 // graceful termination (POSIX only).
 type Server struct {
 	*http.Server
-	HTTP2       bool                   // temporary while http2 is not in std lib (TODO: remove flag when part of std lib)
+	HTTP2       bool                   // whether to enable HTTP/2
 	tls         bool                   // whether this server is serving all HTTPS hosts or not
+	OnDemandTLS bool                   // whether this server supports on-demand TLS (load certs at handshake-time)
 	vhosts      map[string]virtualHost // virtual hosts keyed by their address
 	listener    ListenerFile           // the listener which is bound to the socket
 	listenerMu  sync.Mutex             // protects listener
@@ -60,20 +60,29 @@ type OptionalCallback func(http.ResponseWriter, *http.Request) bool
 // as it stands, you should dispose of a server after stopping it.
 // The behavior of serving with a spent server is undefined.
 func New(addr string, configs []Config, gracefulTimeout time.Duration) (*Server, error) {
-	var tls bool
+	var useTLS, useOnDemandTLS bool
 	if len(configs) > 0 {
-		tls = configs[0].TLS.Enabled
+		useTLS = configs[0].TLS.Enabled
+		host, _, err := net.SplitHostPort(addr)
+		if err != nil {
+			host = addr
+		}
+		if useTLS && host == "" && !configs[0].TLS.Manual {
+			useOnDemandTLS = true
+		}
 	}
 
 	s := &Server{
 		Server: &http.Server{
 			Addr:      addr,
+			TLSConfig: new(tls.Config),
 			// TODO: Make these values configurable?
 			// ReadTimeout:    2 * time.Minute,
 			// WriteTimeout:   2 * time.Minute,
 			// MaxHeaderBytes: 1 << 16,
 		},
-		tls:         tls,
+		tls:         useTLS,
+		OnDemandTLS: useOnDemandTLS,
 		vhosts:      make(map[string]virtualHost),
 		startChan:   make(chan struct{}),
 		connTimeout: gracefulTimeout,
@@ -168,7 +177,7 @@ func (s *Server) serve(ln ListenerFile) error {
 		for _, vh := range s.vhosts {
 			tlsConfigs = append(tlsConfigs, vh.config.TLS)
 		}
-		return serveTLSWithSNI(s, s.listener, tlsConfigs)
+		return serveTLS(s, s.listener, tlsConfigs)
 	}
 
 	close(s.startChan) // unblock anyone waiting for this to start listening
@@ -196,106 +205,32 @@ func (s *Server) setup() error {
 	return nil
 }
 
-// serveTLSWithSNI serves TLS with Server Name Indication (SNI) support, which allows
-// multiple sites (different hostnames) to be served from the same address. It also
-// supports client authentication if srv has it enabled. It blocks until s quits.
-//
-// This method is adapted from the std lib's net/http ServeTLS function, which was written
-// by the Go Authors. It has been modified to support multiple certificate/key pairs,
-// client authentication, and our custom Server type.
-func serveTLSWithSNI(s *Server, ln net.Listener, tlsConfigs []TLSConfig) error {
-	config := cloneTLSConfig(s.TLSConfig)
-
-	// Here we diverge from the stdlib a bit by loading multiple certs/key pairs
-	// then we map the server names to their certs
-	for _, tlsConfig := range tlsConfigs {
-		if tlsConfig.Certificate == "" || tlsConfig.Key == "" {
-			continue
-		}
-		cert, err := tls.LoadX509KeyPair(tlsConfig.Certificate, tlsConfig.Key)
-		if err != nil {
-			defer close(s.startChan)
-			return fmt.Errorf("loading certificate and key pair: %v", err)
-		}
-		cert.OCSPStaple = tlsConfig.OCSPStaple
-		config.Certificates = append(config.Certificates, cert)
-	}
-	if len(config.Certificates) > 0 {
-		config.BuildNameToCertificate()
-	}
-
-	config.GetCertificate = func(clientHello *tls.ClientHelloInfo) (*tls.Certificate, error) {
-		// TODO: When Caddy starts, if it is to issue certs dynamically, we need
-		// terms agreement and an email address. make sure this is enforced at server
-		// start if the Caddyfile enables dynamic certificate issuance!
-
-		// Check NameToCertificate like the std lib does in "getCertificate" (unexported, bah)
-		cert := GetCertificateFromCache(clientHello, config.NameToCertificate)
-		if cert != nil {
-			return cert, nil
-		}
-
-		if s.SNICallback != nil {
-			return s.SNICallback(clientHello)
-		}
-
-		return nil, nil
-	}
-
+// serveTLS serves TLS with SNI and client auth support if s has them enabled. It
+// blocks until s quits.
+func serveTLS(s *Server, ln net.Listener, tlsConfigs []TLSConfig) error {
 	// Customize our TLS configuration
-	config.MinVersion = tlsConfigs[0].ProtocolMinVersion
-	config.MaxVersion = tlsConfigs[0].ProtocolMaxVersion
-	config.CipherSuites = tlsConfigs[0].Ciphers
-	config.PreferServerCipherSuites = tlsConfigs[0].PreferServerCipherSuites
+	s.TLSConfig.MinVersion = tlsConfigs[0].ProtocolMinVersion
+	s.TLSConfig.MaxVersion = tlsConfigs[0].ProtocolMaxVersion
+	s.TLSConfig.CipherSuites = tlsConfigs[0].Ciphers
+	s.TLSConfig.PreferServerCipherSuites = tlsConfigs[0].PreferServerCipherSuites
 
 	// TLS client authentication, if user enabled it
-	err := setupClientAuth(tlsConfigs, config)
+	err := setupClientAuth(tlsConfigs, s.TLSConfig)
 	if err != nil {
 		defer close(s.startChan)
 		return err
 	}
-	s.TLSConfig = config
 
 	// Create TLS listener - note that we do not replace s.listener
 	// with this TLS listener; tls.listener is unexported and does
 	// not implement the File() method we need for graceful restarts
 	// on POSIX systems.
-	ln = tls.NewListener(ln, config)
+	ln = tls.NewListener(ln, s.TLSConfig)
 
 	close(s.startChan) // unblock anyone waiting for this to start listening
 	return s.Server.Serve(ln)
 }
 
-// Borrowed from the Go standard library, crypto/tls pacakge, common.go.
-// It has been modified to fit this program.
-// Original license:
-//
-// Copyright 2009 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-func GetCertificateFromCache(clientHello *tls.ClientHelloInfo, cache map[string]*tls.Certificate) *tls.Certificate {
-	name := strings.ToLower(clientHello.ServerName)
-	for len(name) > 0 && name[len(name)-1] == '.' {
-		name = name[:len(name)-1]
-	}
-
-	// exact match? great! use it
-	if cert, ok := cache[name]; ok {
-		return cert
-	}
-
-	// try replacing labels in the name with wildcards until we get a match.
-	labels := strings.Split(name, ".")
-	for i := range labels {
-		labels[i] = "*"
-		candidate := strings.Join(labels, ".")
-		if cert, ok := cache[candidate]; ok {
-			return cert
-		}
-	}
-	return nil
-}
-
 // Stop stops the server. It blocks until the server is
 // totally stopped. On POSIX systems, it will wait for
 // connections to close (up to a max timeout of a few
@@ -482,6 +417,8 @@ func (ln tcpKeepAliveListener) File() (*os.File, error) {
 }
 
 // copied from net/http/transport.go
+/*
+ TODO - remove - not necessary?
 func cloneTLSConfig(cfg *tls.Config) *tls.Config {
 	if cfg == nil {
 		return &tls.Config{}
@@ -507,7 +444,7 @@ func cloneTLSConfig(cfg *tls.Config) *tls.Config {
 		MaxVersion:               cfg.MaxVersion,
 		CurvePreferences:         cfg.CurvePreferences,
 	}
-}
+}*/
 
 // ShutdownCallbacks executes all the shutdown callbacks
 // for all the virtualhosts in servers, and returns all the