Skip to content

C
caddy
Commit 393bc299 authored by Matthew Holt's avatar Matthew Holt
Browse files
Options

Add clustering plugin types; use latest certmagic.Storage interface

parent 33f2b16a
Show whitespace changes
Inline Side-by-side
Showing with 268 additions and 263 deletions
caddy.go
View file @ 393bc299
......@@ -41,10 +41,12 @@ import (
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/mholt/caddy/caddyfile"
"github.com/mholt/caddy/telemetry"
"github.com/mholt/certmagic"
)
// Configurable application parameters
......@@ -462,6 +464,25 @@ func (i *Instance) Caddyfile() Input {
//
// This function blocks until all the servers are listening.
func Start(cdyfile Input) (*Instance, error) {
// set up the clustering plugin, if there is one (this should be done
// exactly once -- but we can't do it during init when they're still
// getting plugged in, so do it when starting the first instance)
if atomic.CompareAndSwapInt32(&clusterPluginSetup, 0, 1) {
clusterPluginName := os.Getenv("CADDY_CLUSTERING")
if clusterPluginName == "" {
clusterPluginName = "file" // name of default storage plugin as registered in caddytls package
}
clusterFn, ok := clusterProviders[clusterPluginName]
if !ok {
return nil, fmt.Errorf("unrecognized cluster plugin (was it included in the Caddy build?): %s", clusterPluginName)
}
storage, err := clusterFn()
if err != nil {
return nil, fmt.Errorf("constructing cluster plugin %s: %v", clusterPluginName, err)
}
certmagic.DefaultStorage = storage
}
inst := &Instance{serverType: cdyfile.ServerType(), wg: new(sync.WaitGroup), Storage: make(map[interface{}]interface{})}
err := startWithListenerFds(cdyfile, inst, nil)
if err != nil {
......@@ -985,5 +1006,7 @@ var (
DefaultConfigFile = "Caddyfile"
)
var clusterPluginSetup int32 // access atomically
// CtxKey is a value type for use with context.WithValue.
type CtxKey string
caddyhttp/caddyhttp_test.go
View file @ 393bc299
......@@ -25,9 +25,9 @@ import (
// ensure that the standard plugins are in fact plugged in
// and registered properly; this is a quick/naive way to do it.
func TestStandardPlugins(t *testing.T) {
numStandardPlugins := 30 // importing caddyhttp plugs in this many plugins
numStandardPlugins := 31 // importing caddyhttp plugs in this many plugins
s := caddy.DescribePlugins()
if got, want := strings.Count(s, "\n"), numStandardPlugins+5; got != want {
if got, want := strings.Count(s, "\n"), numStandardPlugins+7; got != want {
t.Errorf("Expected all standard plugins to be plugged in, got:\n%s", s)
}
}
caddytls/setup.go
View file @ 393bc299
......@@ -35,8 +35,8 @@ import (
func init() {
caddy.RegisterPlugin("tls", caddy.Plugin{Action: setupTLS})
// ensure TLS assets are stored and accessed from the CADDYPATH
certmagic.DefaultStorage = certmagic.FileStorage{Path: caddy.AssetsPath()}
// ensure the default Storage implementation is plugged in
caddy.RegisterClusterPlugin("file", constructDefaultClusterPlugin)
}
// setupTLS sets up the TLS configuration and installs certificates that
......@@ -442,3 +442,7 @@ func loadCertsInDir(cfg *Config, c *caddy.Controller, dir string) error {
return nil
})
}
func constructDefaultClusterPlugin() (certmagic.Storage, error) {
return certmagic.FileStorage{Path: caddy.AssetsPath()}, nil
}
caddytls/tls.go
View file @ 393bc299
......@@ -108,13 +108,3 @@ func RegisterDNSProvider(name string, provider DNSProviderConstructor) {
dnsProviders[name] = provider
caddy.RegisterPlugin("tls.dns."+name, caddy.Plugin{})
}
// TODO...
// var storageProviders = make(map[string]StorageConstructor)
// // RegisterStorageProvider registers provider by name for storing tls data
// func RegisterStorageProvider(name string, provider StorageConstructor) {
// storageProviders[name] = provider
// caddy.RegisterPlugin("tls.storage."+name, caddy.Plugin{})
// }
plugins.go
View file @ 393bc299
......@@ -22,6 +22,7 @@ import (
"sync"
"github.com/mholt/caddy/caddyfile"
"github.com/mholt/certmagic"
)
// These are all the registered plugins.
......@@ -73,6 +74,13 @@ func DescribePlugins() string {
}
}
if len(pl["clustering"]) > 0 {
str += "\nClustering plugins:\n"
for _, name := range pl["clustering"] {
str += " " + name + "\n"
}
}
str += "\nOther plugins:\n"
for _, name := range pl["others"] {
str += " " + name + "\n"
......@@ -99,6 +107,11 @@ func ListPlugins() map[string][]string {
p["caddyfile_loaders"] = append(p["caddyfile_loaders"], defaultCaddyfileLoader.name)
}
// cluster plugins in registration order
for name := range clusterProviders {
p["clustering"] = append(p["clsutering"], name)
}
// List the event hook plugins
eventHooks.Range(func(k, _ interface{}) bool {
p["event_hooks"] = append(p["event_hooks"], k.(string))
......@@ -443,6 +456,21 @@ func loadCaddyfileInput(serverType string) (Input, error) {
return caddyfileToUse, nil
}
// ClusterPluginConstructor is a function type that is used to
// instantiate a new implementation of both certmagic.Storage
// and certmagic.Locker, which are required for successful
// use in cluster environments.
type ClusterPluginConstructor func() (certmagic.Storage, error)
// clusterProviders is the list of storage providers
var clusterProviders = make(map[string]ClusterPluginConstructor)
// RegisterClusterPlugin registers provider by name for facilitating
// cluster-wide operations like storage and synchronization.
func RegisterClusterPlugin(name string, provider ClusterPluginConstructor) {
clusterProviders[name] = provider
}
// OnProcessExit is a list of functions to run when the process
// exits -- they are ONLY for cleanup and should not block,
// return errors, or do anything fancy. They will be run with
......
vendor/github.com/mholt/certmagic/certmagic.go
View file @ 393bc299
......@@ -12,6 +12,25 @@
// See the License for the specific language governing permissions and
// limitations under the License.
// Package certmagic automates the obtaining and renewal of TLS certificates,
// including TLS & HTTPS best practices such as robust OCSP stapling, caching,
// HTTP->HTTPS redirects, and more.
//
// Its high-level API serves your HTTP handlers over HTTPS by simply giving
// the domain name(s) and the http.Handler; CertMagic will create and run
// the HTTPS server for you, fully managing certificates during the lifetime
// of the server. Similarly, it can be used to start TLS listeners or return
// a ready-to-use tls.Config -- whatever layer you need TLS for, CertMagic
// makes it easy.
//
// If you need more control, create a Config using New() and then call
// Manage() on the config; but you'll have to be sure to solve the HTTP
// and TLS-ALPN challenges yourself (unless you disabled them or use the
// DNS challenge) by using the provided Config.GetCertificate function
// in your tls.Config and/or Config.HTTPChallangeHandler in your HTTP
// handler.
//
// See the package's README for more instruction.
package certmagic
import (
......@@ -166,46 +185,6 @@ func manageWithDefaultConfig(domainNames []string, disableHTTPChallenge bool) (*
return cfg, cfg.Manage(domainNames)
}
// Locker facilitates synchronization of certificate tasks across
// machines and networks.
type Locker interface {
// TryLock will attempt to acquire the lock for key. If a
// lock could be obtained, nil values are returned as no
// waiting is required. If not (meaning another process is
// already working on key), a Waiter value will be returned,
// upon which you should Wait() until it is finished.
//
// The key should be a carefully-chosen value that uniquely
// and precisely identifies the operation being locked. For
// example, if it is for a certificate obtain or renew with
// the ACME protocol to the same CA endpoint (remembering
// that an obtain and renew are the same according to ACME,
// thus both obtain and renew should share a lock key), a
// good key would identify that operation by some name,
// concatenated with the domain name and the CA endpoint.
//
// TryLock never blocks; it always returns without waiting.
//
// To prevent deadlocks, all implementations (where this concern
// is relevant) should put a reasonable expiration on the lock in
// case Unlock is unable to be called due to some sort of storage
// system failure or crash.
TryLock(key string) (Waiter, error)
// Unlock releases the lock for key. This method must ONLY be
// called after a successful call to TryLock where no Waiter was
// returned, and only after the operation requiring the lock is
// finished, even if it returned an error or timed out. Unlock
// should also clean up any unused resources allocated during
// TryLock.
Unlock(key string) error
}
// Waiter is a type that can block until a lock is released.
type Waiter interface {
Wait()
}
// OnDemandConfig contains some state relevant for providing
// on-demand TLS.
type OnDemandConfig struct {
......
vendor/github.com/mholt/certmagic/client.go
View file @ 393bc299
......@@ -217,9 +217,8 @@ func (cfg *Config) lockKey(op, domainName string) string {
// Callers who have access to a Config value should use the ObtainCert
// method on that instead of this lower-level method.
func (c *acmeClient) Obtain(name string) error {
if c.config.Sync != nil {
lockKey := c.config.lockKey("cert_acme", name)
waiter, err := c.config.Sync.TryLock(lockKey)
waiter, err := c.config.certCache.storage.TryLock(lockKey)
if err != nil {
return err
}
......@@ -229,11 +228,10 @@ func (c *acmeClient) Obtain(name string) error {
return nil // we assume the process with the lock succeeded, rather than hammering this execution path again
}
defer func() {
if err := c.config.Sync.Unlock(lockKey); err != nil {
if err := c.config.certCache.storage.Unlock(lockKey); err != nil {
log.Printf("[ERROR] Unable to unlock obtain call for %s: %v", name, err)
}
}()
}
for attempts := 0; attempts < 2; attempts++ {
request := certificate.ObtainRequest{
......@@ -276,23 +274,21 @@ func (c *acmeClient) Obtain(name string) error {
// Callers who have access to a Config value should use the RenewCert
// method on that instead of this lower-level method.
func (c *acmeClient) Renew(name string) error {
if c.config.Sync != nil {
lockKey := c.config.lockKey("cert_acme", name)
waiter, err := c.config.Sync.TryLock(lockKey)
waiter, err := c.config.certCache.storage.TryLock(lockKey)
if err != nil {
return err
}
if waiter != nil {
log.Printf("[INFO] Certificate for %s is already being renewed elsewhere and stored; waiting", name)
waiter.Wait()
return nil // assume that the worker that renewed the cert succeeded; avoid hammering this path over and over
return nil // assume that the worker that renewed the cert succeeded to avoid hammering this path over and over
}
defer func() {
if err := c.config.Sync.Unlock(lockKey); err != nil {
if err := c.config.certCache.storage.Unlock(lockKey); err != nil {
log.Printf("[ERROR] Unable to unlock renew call for %s: %v", name, err)
}
}()
}
// Prepare for renewal (load PEM cert, key, and meta)
certRes, err := c.config.loadCertResource(name)
......
vendor/github.com/mholt/certmagic/config.go
View file @ 393bc299
......@@ -38,15 +38,6 @@ type Config struct {
// selecting an existing ACME server account
Email string
// The synchronization implementation - although
// it is not strictly required to have a Sync
// value in general, all instances running in
// in a cluster for the same domain names must
// specify a Sync and use the same one, otherwise
// some cert operations will not be properly
// coordinated
Sync Locker
// Set to true if agreed to the CA's
// subscriber agreement
Agreed bool
......@@ -198,20 +189,6 @@ func NewWithCache(certCache *Cache, cfg Config) *Config {
cfg.MustStaple = MustStaple
}
// if no sync facility is provided, we'll default to
// a file system synchronizer backed by the storage
// given to certCache (if it is one), or just a simple
// in-memory sync facility otherwise (strictly speaking,
// a sync is not required; only if running multiple
// instances for the same domain names concurrently)
if cfg.Sync == nil {
if ccfs, ok := certCache.storage.(FileStorage); ok {
cfg.Sync = NewFileStorageLocker(ccfs)
} else {
cfg.Sync = NewMemoryLocker()
}
}
// ensure the unexported fields are valid
cfg.certificates = make(map[string]string)
cfg.certCache = certCache
......@@ -222,7 +199,12 @@ func NewWithCache(certCache *Cache, cfg Config) *Config {
}
// Manage causes the certificates for domainNames to be managed
// according to cfg.
// according to cfg. If cfg is enabled for OnDemand, then this
// simply whitelists the domain names. Otherwise, the certificate(s)
// for each name are loaded from storage or obtained from the CA;
// and if loaded from storage, renewed if they are expiring or
// expired. It then caches the certificate in memory and is
// prepared to serve them up during TLS handshakes.
func (cfg *Config) Manage(domainNames []string) error {
for _, domainName := range domainNames {
// if on-demand is configured, simply whitelist this name
......
vendor/github.com/mholt/certmagic/filestorage.go
View file @ 393bc299
......@@ -15,10 +15,13 @@
package certmagic
import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"runtime"
"sync"
"time"
)
// FileStorage facilitates forming file paths derived from a root
......@@ -123,4 +126,114 @@ func dataDir() string {
return filepath.Join(baseDir, "certmagic")
}
// TryLock attempts to get a lock for name, otherwise it returns
// a Waiter value to wait until the other process is finished.
func (fs FileStorage) TryLock(key string) (Waiter, error) {
fileStorageNameLocksMu.Lock()
defer fileStorageNameLocksMu.Unlock()
// see if lock already exists within this process - allows
// for faster unlocking since we don't have to poll the disk
fw, ok := fileStorageNameLocks[key]
if ok {
// lock already created within process, let caller wait on it
return fw, nil
}
// attempt to persist lock to disk by creating lock file
// parent dir must exist
lockDir := fs.lockDir()
if err := os.MkdirAll(lockDir, 0700); err != nil {
return nil, err
}
fw = &fileStorageWaiter{
filename: filepath.Join(lockDir, safeKey(key)+".lock"),
wg: new(sync.WaitGroup),
}
// create the file in a special mode such that an
// error is returned if it already exists
lf, err := os.OpenFile(fw.filename, os.O_CREATE|os.O_EXCL, 0644)
if err != nil {
if os.IsExist(err) {
// another process has the lock; use it to wait
return fw, nil
}
// otherwise, this was some unexpected error
return nil, err
}
lf.Close()
// looks like we get the lock
fw.wg.Add(1)
fileStorageNameLocks[key] = fw
return nil, nil
}
// Unlock releases the lock for name.
func (fs FileStorage) Unlock(key string) error {
fileStorageNameLocksMu.Lock()
defer fileStorageNameLocksMu.Unlock()
fw, ok := fileStorageNameLocks[key]
if !ok {
return fmt.Errorf("FileStorage: no lock to release for %s", key)
}
// remove lock file
os.Remove(fw.filename)
// if parent folder is now empty, remove it too to keep it tidy
dir, err := os.Open(fs.lockDir()) // OK to ignore error here
if err == nil {
items, _ := dir.Readdirnames(3) // OK to ignore error here
if len(items) == 0 {
os.Remove(dir.Name())
}
dir.Close()
}
// clean up in memory
fw.wg.Done()
delete(fileStorageNameLocks, key)
return nil
}
func (fs FileStorage) lockDir() string {
return filepath.Join(fs.Path, "locks")
}
// fileStorageWaiter waits for a file to disappear; it
// polls the file system to check for the existence of
// a file. It also uses a WaitGroup to optimize the
// polling in the case when this process is the only
// one waiting. (Other processes that are waiting for
// the lock will still block, but must wait for the
// polling to get their answer.)
type fileStorageWaiter struct {
filename string
wg *sync.WaitGroup
}
// Wait waits until the lock is released.
func (fw *fileStorageWaiter) Wait() {
start := time.Now()
fw.wg.Wait()
for time.Since(start) < 1*time.Hour {
_, err := os.Stat(fw.filename)
if os.IsNotExist(err) {
return
}
time.Sleep(1 * time.Second)
}
}
var fileStorageNameLocks = make(map[string]*fileStorageWaiter)
var fileStorageNameLocksMu sync.Mutex
var _ Storage = FileStorage{}
var _ Waiter = &fileStorageWaiter{}
vendor/github.com/mholt/certmagic/filestoragesync.go deleted 100644 → 0
View file @ 33f2b16a
// Copyright 2015 Matthew Holt
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package certmagic
import (
"fmt"
"os"
"path/filepath"
"sync"
"time"
)
// FileStorageLocker implements the Locker interface
// using the file system. An empty value is NOT VALID,
// so you must use NewFileStorageLocker() to get one.
type FileStorageLocker struct {
fs FileStorage
}
// NewFileStorageLocker returns a valid Locker backed by fs.
func NewFileStorageLocker(fs FileStorage) *FileStorageLocker {
return &FileStorageLocker{fs: fs}
}
// TryLock attempts to get a lock for name, otherwise it returns
// a Waiter value to wait until the other process is finished.
func (l *FileStorageLocker) TryLock(name string) (Waiter, error) {
fileStorageNameLocksMu.Lock()
defer fileStorageNameLocksMu.Unlock()
// see if lock already exists within this process
fw, ok := fileStorageNameLocks[name]
if ok {
// lock already created within process, let caller wait on it
return fw, nil
}
// attempt to persist lock to disk by creating lock file
// parent dir must exist
lockDir := l.lockDir()
if err := os.MkdirAll(lockDir, 0700); err != nil {
return nil, err
}
fw = &FileStorageWaiter{
filename: filepath.Join(lockDir, safeKey(name)+".lock"),
wg: new(sync.WaitGroup),
}
// create the file in a special mode such that an
// error is returned if it already exists
lf, err := os.OpenFile(fw.filename, os.O_CREATE|os.O_EXCL, 0644)
if err != nil {
if os.IsExist(err) {
// another process has the lock; use it to wait
return fw, nil
}
// otherwise, this was some unexpected error
return nil, err
}
lf.Close()
// looks like we get the lock
fw.wg.Add(1)
fileStorageNameLocks[name] = fw
return nil, nil
}
// Unlock releases the lock for name.
func (l *FileStorageLocker) Unlock(name string) error {
fileStorageNameLocksMu.Lock()
defer fileStorageNameLocksMu.Unlock()
fw, ok := fileStorageNameLocks[name]
if !ok {
return fmt.Errorf("FileStorageLocker: no lock to release for %s", name)
}
// remove lock file
os.Remove(fw.filename)
// if parent folder is now empty, remove it too to keep it tidy
dir, err := os.Open(l.lockDir()) // OK to ignore error here
if err == nil {
items, _ := dir.Readdirnames(3) // OK to ignore error here
if len(items) == 0 {
os.Remove(dir.Name())
}
dir.Close()
}
// clean up in memory
fw.wg.Done()
delete(fileStorageNameLocks, name)
return nil
}
func (l *FileStorageLocker) lockDir() string {
return filepath.Join(l.fs.Path, "locks")
}
// FileStorageWaiter waits for a file to disappear; it
// polls the file system to check for the existence of
// a file. It also uses a WaitGroup to optimize the
// polling in the case when this process is the only
// one waiting. (Other processes that are waiting
// for the lock will still block, but must wait
// for the poll intervals to get their answer.)
type FileStorageWaiter struct {
filename string
wg *sync.WaitGroup
}
// Wait waits until the lock is released.
func (fw *FileStorageWaiter) Wait() {
start := time.Now()
fw.wg.Wait()
for time.Since(start) < 1*time.Hour {
_, err := os.Stat(fw.filename)
if os.IsNotExist(err) {
return
}
time.Sleep(1 * time.Second)
}
}
var fileStorageNameLocks = make(map[string]*FileStorageWaiter)
var fileStorageNameLocksMu sync.Mutex
var _ Locker = &FileStorageLocker{}
var _ Waiter = &FileStorageWaiter{}
vendor/github.com/mholt/certmagic/storage.go
View file @ 393bc299
......@@ -31,9 +31,9 @@ import (
// in order to share certificates and other TLS resources
// with the cluster.
type Storage interface {
// Exists returns true if the key exists
// and there was no error checking.
Exists(key string) bool
// Locker provides atomic synchronization
// operations, making Storage safe to share.
Locker
// Store puts value at key.
Store(key string, value []byte) error
......@@ -44,6 +44,10 @@ type Storage interface {
// Delete deletes key.
Delete(key string) error
// Exists returns true if the key exists
// and there was no error checking.
Exists(key string) bool
// List returns all keys that match prefix.
List(prefix string) ([]string, error)
......@@ -51,6 +55,41 @@ type Storage interface {
Stat(key string) (KeyInfo, error)
}
// Locker facilitates synchronization of certificate tasks across
// machines and networks.
type Locker interface {
// TryLock will attempt to acquire the lock for key. If a
// lock could be obtained, nil values are returned as no
// waiting is required. If not (meaning another process is
// already working on key), a Waiter value will be returned,
// upon which you should Wait() until it is finished.
//
// The actual implementation of obtaining of a lock must be
// an atomic operation so that multiple TryLock calls at the
// same time always results in only one caller receiving the
// lock. TryLock always returns without waiting.
//
// To prevent deadlocks, all implementations (where this concern
// is relevant) should put a reasonable expiration on the lock in
// case Unlock is unable to be called due to some sort of network
// or system failure or crash.
TryLock(key string) (Waiter, error)
// Unlock releases the lock for key. This method must ONLY be
// called after a successful call to TryLock where no Waiter was
// returned, and only after the operation requiring the lock is
// finished, even if it errored or timed out. It is INCORRECT to
// call Unlock if any non-nil value was returned from a call to
// TryLock or if Unlock was not called at all. Unlock should also
// clean up any unused resources allocated during TryLock.
Unlock(key string) error
}
// Waiter is a type that can block until a lock is released.
type Waiter interface {
Wait()
}
// KeyInfo holds information about a key in storage.
type KeyInfo struct {
Key string
......@@ -207,6 +246,3 @@ var defaultFileStorage = FileStorage{Path: dataDir()}
// DefaultStorage is the default Storage implementation.
var DefaultStorage Storage = defaultFileStorage
// DefaultSync is a default sync to use.
var DefaultSync Locker
vendor/manifest
View file @ 393bc299
......@@ -138,7 +138,7 @@
"importpath": "github.com/mholt/certmagic",
"repository": "https://github.com/mholt/certmagic",
"vcs": "git",
"revision": "4dd0c62355ec3c3732f18c506449fc9b5f9f4c59",
"revision": "8b6ddf223c912a863aaadd388bfdd29be295fb5d",
"branch": "master",
"notests": true
},
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7