Commit 3f1f6720 authored by Abiola Ibrahim's avatar Abiola Ibrahim

Decouple git middleware from caddy core. Now available as an add-on at...

Decouple git middleware from caddy core. Now available as an add-on at https://github.com/abiosoft/caddy-git.
parent ab0cbf3e
......@@ -48,7 +48,6 @@ var directiveOrder = []directive{
// Other directives that don't create HTTP handlers
{"startup", setup.Startup},
{"shutdown", setup.Shutdown},
{"git", setup.Git},
// Directives that inject handlers (middleware)
{"log", setup.Log},
......
package setup
import (
"fmt"
"net/url"
"path/filepath"
"runtime"
"strconv"
"strings"
"time"
"github.com/mholt/caddy/middleware"
"github.com/mholt/caddy/middleware/git"
"github.com/mholt/caddy/middleware/git/webhook"
)
// Git configures a new Git service routine.
func Git(c *Controller) (middleware.Middleware, error) {
repo, err := gitParse(c)
if err != nil {
return nil, err
}
// If a HookUrl is set, we switch to event based pulling.
// Install the url handler
if repo.HookUrl != "" {
c.Startup = append(c.Startup, func() error {
return repo.Pull()
})
webhook := &webhook.WebHook{Repo: repo}
return func(next middleware.Handler) middleware.Handler {
webhook.Next = next
return webhook
}, nil
} else {
c.Startup = append(c.Startup, func() error {
// Start service routine in background
git.Start(repo)
// Do a pull right away to return error
return repo.Pull()
})
}
return nil, err
}
func gitParse(c *Controller) (*git.Repo, error) {
repo := &git.Repo{Branch: "master", Interval: git.DefaultInterval, Path: c.Root}
for c.Next() {
args := c.RemainingArgs()
switch len(args) {
case 2:
repo.Path = filepath.Clean(c.Root + string(filepath.Separator) + args[1])
fallthrough
case 1:
repo.URL = args[0]
}
for c.NextBlock() {
switch c.Val() {
case "repo":
if !c.NextArg() {
return nil, c.ArgErr()
}
repo.URL = c.Val()
case "path":
if !c.NextArg() {
return nil, c.ArgErr()
}
repo.Path = filepath.Clean(c.Root + string(filepath.Separator) + c.Val())
case "branch":
if !c.NextArg() {
return nil, c.ArgErr()
}
repo.Branch = c.Val()
case "key":
if !c.NextArg() {
return nil, c.ArgErr()
}
repo.KeyPath = c.Val()
case "interval":
if !c.NextArg() {
return nil, c.ArgErr()
}
t, _ := strconv.Atoi(c.Val())
if t > 0 {
repo.Interval = time.Duration(t) * time.Second
}
case "hook":
if !c.NextArg() {
return nil, c.ArgErr()
}
repo.HookUrl = c.Val()
// optional secret for validation
if c.NextArg() {
repo.HookSecret = c.Val()
}
case "then":
thenArgs := c.RemainingArgs()
if len(thenArgs) == 0 {
return nil, c.ArgErr()
}
repo.Then = strings.Join(thenArgs, " ")
default:
return nil, c.ArgErr()
}
}
}
// if repo is not specified, return error
if repo.URL == "" {
return nil, c.ArgErr()
}
// if private key is not specified, convert repository URL to https
// to avoid ssh authentication
// else validate git URL
// Note: private key support not yet available on Windows
var err error
if repo.KeyPath == "" {
repo.URL, repo.Host, err = sanitizeHTTP(repo.URL)
} else {
repo.URL, repo.Host, err = sanitizeGit(repo.URL)
// TODO add Windows support for private repos
if runtime.GOOS == "windows" {
return nil, fmt.Errorf("private repository not yet supported on Windows")
}
}
if err != nil {
return nil, err
}
// validate git requirements
if err = git.Init(); err != nil {
return nil, err
}
return repo, repo.Prepare()
}
// sanitizeHTTP cleans up repository URL and converts to https format
// if currently in ssh format.
// Returns sanitized url, hostName (e.g. github.com, bitbucket.com)
// and possible error
func sanitizeHTTP(repoURL string) (string, string, error) {
url, err := url.Parse(repoURL)
if err != nil {
return "", "", err
}
if url.Host == "" && strings.HasPrefix(url.Path, "git@") {
url.Path = url.Path[len("git@"):]
i := strings.Index(url.Path, ":")
if i < 0 {
return "", "", fmt.Errorf("invalid git url %s", repoURL)
}
url.Host = url.Path[:i]
url.Path = "/" + url.Path[i+1:]
}
repoURL = "https://" + url.Host + url.Path
// add .git suffix if missing
if !strings.HasSuffix(repoURL, ".git") {
repoURL += ".git"
}
return repoURL, url.Host, nil
}
// sanitizeGit cleans up repository url and converts to ssh format for private
// repositories if required.
// Returns sanitized url, hostName (e.g. github.com, bitbucket.com)
// and possible error
func sanitizeGit(repoURL string) (string, string, error) {
repoURL = strings.TrimSpace(repoURL)
// check if valid ssh format
if !strings.HasPrefix(repoURL, "git@") || strings.Index(repoURL, ":") < len("git@a:") {
// check if valid http format and convert to ssh
if url, err := url.Parse(repoURL); err == nil && strings.HasPrefix(url.Scheme, "http") {
repoURL = fmt.Sprintf("git@%v:%v", url.Host, url.Path[1:])
} else {
return "", "", fmt.Errorf("invalid git url %s", repoURL)
}
}
hostURL := repoURL[len("git@"):]
i := strings.Index(hostURL, ":")
host := hostURL[:i]
// add .git suffix if missing
if !strings.HasSuffix(repoURL, ".git") {
repoURL += ".git"
}
return repoURL, host, nil
}
package setup
import (
"io/ioutil"
"strings"
"testing"
"time"
"github.com/mholt/caddy/middleware/git"
"github.com/mholt/caddy/middleware/git/gittest"
)
// init sets the OS used to fakeOS
func init() {
git.SetOS(gittest.FakeOS)
}
func check(t *testing.T, err error) {
if err != nil {
t.Errorf("Expected no errors, but got: %v", err)
}
}
func TestGit(t *testing.T) {
c := NewTestController(`git git@github.com:mholt/caddy.git`)
mid, err := Git(c)
check(t, err)
if mid != nil {
t.Fatal("Git middleware is a background service and expected to be nil.")
}
}
func TestIntervals(t *testing.T) {
tests := []string{
`git git@github.com:user/repo { interval 10 }`,
`git git@github.com:user/repo { interval 5 }`,
`git git@github.com:user/repo { interval 2 }`,
`git git@github.com:user/repo { interval 1 }`,
`git git@github.com:user/repo { interval 6 }`,
}
for i, test := range tests {
git.SetLogger(gittest.NewLogger(gittest.Open("file")))
c1 := NewTestController(test)
repo, err := gitParse(c1)
check(t, err)
c2 := NewTestController(test)
_, err = Git(c2)
check(t, err)
// start startup services
err = c2.Startup[0]()
check(t, err)
// wait for first background pull
gittest.Sleep(time.Millisecond * 100)
// switch logger to test file
logFile := gittest.Open("file")
git.SetLogger(gittest.NewLogger(logFile))
// sleep for the interval
gittest.Sleep(repo.Interval)
// get log output
out, err := ioutil.ReadAll(logFile)
check(t, err)
// if greater than minimum interval
if repo.Interval >= time.Second*5 {
expected := `https://github.com/user/repo.git pulled.
No new changes.`
// ensure pull is done by tracing the output
if expected != strings.TrimSpace(string(out)) {
t.Errorf("Test %v: Expected %v found %v", i, expected, string(out))
}
} else {
// ensure pull is ignored by confirming no output
if string(out) != "" {
t.Errorf("Test %v: Expected no output but found %v", i, string(out))
}
}
// stop background thread monitor
git.Services.Stop(repo.URL, 1)
}
}
func TestGitParse(t *testing.T) {
tests := []struct {
input string
shouldErr bool
expected *git.Repo
}{
{`git git@github.com:user/repo`, false, &git.Repo{
URL: "https://github.com/user/repo.git",
}},
{`git github.com/user/repo`, false, &git.Repo{
URL: "https://github.com/user/repo.git",
}},
{`git git@github.com/user/repo`, true, nil},
{`git http://github.com/user/repo`, false, &git.Repo{
URL: "https://github.com/user/repo.git",
}},
{`git https://github.com/user/repo`, false, &git.Repo{
URL: "https://github.com/user/repo.git",
}},
{`git http://github.com/user/repo {
key ~/.key
}`, false, &git.Repo{
KeyPath: "~/.key",
URL: "git@github.com:user/repo.git",
}},
{`git git@github.com:user/repo {
key ~/.key
}`, false, &git.Repo{
KeyPath: "~/.key",
URL: "git@github.com:user/repo.git",
}},
{`git `, true, nil},
{`git {
}`, true, nil},
{`git {
repo git@github.com:user/repo.git`, true, nil},
{`git {
repo git@github.com:user/repo
key ~/.key
}`, false, &git.Repo{
KeyPath: "~/.key",
URL: "git@github.com:user/repo.git",
}},
{`git {
repo git@github.com:user/repo
key ~/.key
interval 600
}`, false, &git.Repo{
KeyPath: "~/.key",
URL: "git@github.com:user/repo.git",
Interval: time.Second * 600,
}},
{`git {
repo git@github.com:user/repo
branch dev
}`, false, &git.Repo{
Branch: "dev",
URL: "https://github.com/user/repo.git",
}},
{`git {
key ~/.key
}`, true, nil},
{`git {
repo git@github.com:user/repo
key ~/.key
then echo hello world
}`, false, &git.Repo{
KeyPath: "~/.key",
URL: "git@github.com:user/repo.git",
Then: "echo hello world",
}},
}
for i, test := range tests {
c := NewTestController(test.input)
repo, err := gitParse(c)
if !test.shouldErr && err != nil {
t.Errorf("Test %v should not error but found %v", i, err)
continue
}
if test.shouldErr && err == nil {
t.Errorf("Test %v should error but found nil", i)
continue
}
if !reposEqual(test.expected, repo) {
t.Errorf("Test %v expects %v but found %v", i, test.expected, repo)
}
}
}
func reposEqual(expected, repo *git.Repo) bool {
if expected == nil {
return repo == nil
}
if expected.Branch != "" && expected.Branch != repo.Branch {
return false
}
if expected.Host != "" && expected.Host != repo.Host {
return false
}
if expected.Interval != 0 && expected.Interval != repo.Interval {
return false
}
if expected.KeyPath != "" && expected.KeyPath != repo.KeyPath {
return false
}
if expected.Path != "" && expected.Path != repo.Path {
return false
}
if expected.Then != "" && expected.Then != repo.Then {
return false
}
if expected.URL != "" && expected.URL != repo.URL {
return false
}
return true
}
// Package git is the middleware that pull sites from git repo
//
// Caddyfile Syntax :
// git repo path {
// repo
// path
// branch
// key
// interval
// then command args
// }
// repo - git repository
// compulsory. Both ssh (e.g. git@github.com:user/project.git)
// and https(e.g. https://github.com/user/project) are supported.
// Can be specified in either config block or top level
//
// path - directory to pull into, relative to site root
// optional. Defaults to site root.
//
// branch - git branch or tag
// optional. Defaults to master
//
// key - path to private ssh key
// optional. Required for private repositories. e.g. /home/user/.ssh/id_rsa
//
// interval- interval between git pulls in seconds
// optional. Defaults to 3600 (1 Hour).
//
// then - command to execute after successful pull
// optional. If set, will execute only when there are new changes.
//
// Examples :
//
// public repo pulled into site root
// git github.com/user/myproject
//
// public repo pulled into <root>/mysite
// git https://github.com/user/myproject mysite
//
// private repo pulled into <root>/mysite with tag v1.0 and interval of 1 day.
// git {
// repo git@github.com:user/myproject
// branch v1.0
// path mysite
// key /home/user/.ssh/id_rsa
// interval 86400 # 1 day
// }
//
// Caddyfile with private git repo and php support via fastcgi.
// path defaults to /var/www/html/myphpsite as specified in root config.
//
// 0.0.0.0:8080
//
// git {
// repo git@github.com:user/myphpsite
// key /home/user/.ssh/id_rsa
// interval 86400 # 1 day
// }
//
// fastcgi / 127.0.0.1:9000 php
//
// root /var/www/html/myphpsite
//
// A pull is first attempted after initialization. Afterwards, a pull is attempted
// after request to server and if time taken since last successful pull is higher than interval.
//
// After the first successful pull (should be during initialization except an error occurs),
// subsequent pulls are done in background and do not impact request time.
//
// Note: private repositories are currently only supported and tested on Linux and OSX
package git
package git
import (
"bytes"
"fmt"
"os"
"strings"
"sync"
"time"
"github.com/mholt/caddy/middleware"
"github.com/mholt/caddy/middleware/git/gitos"
)
// DefaultInterval is the minimum interval to delay before
// requesting another git pull
const DefaultInterval time.Duration = time.Hour * 1
// Number of retries if git pull fails
const numRetries = 3
// gitBinary holds the absolute path to git executable
var gitBinary string
// shell holds the shell to be used. Either sh or bash.
var shell string
// initMutex prevents parallel attempt to validate
// git requirements.
var initMutex = sync.Mutex{}
// Services holds all git pulling services and provides the function to
// stop them.
var Services = &services{}
// Repo is the structure that holds required information
// of a git repository.
type Repo struct {
URL string // Repository URL
Path string // Directory to pull to
Host string // Git domain host e.g. github.com
Branch string // Git branch
KeyPath string // Path to private ssh key
Interval time.Duration // Interval between pulls
Then string // Command to execute after successful git pull
pulled bool // true if there was a successful pull
lastPull time.Time // time of the last successful pull
lastCommit string // hash for the most recent commit
sync.Mutex
HookUrl string // url to listen on for webhooks
HookSecret string // secret to validate hooks
}
// Pull attempts a git clone.
// It retries at most numRetries times if error occurs
func (r *Repo) Pull() error {
r.Lock()
defer r.Unlock()
// prevent a pull if the last one was less than 5 seconds ago
if gos.TimeSince(r.lastPull) < 5*time.Second {
return nil
}
// keep last commit hash for comparison later
lastCommit := r.lastCommit
var err error
// Attempt to pull at most numRetries times
for i := 0; i < numRetries; i++ {
if err = r.pull(); err == nil {
break
}
Logger().Println(err)
}
if err != nil {
return err
}
// check if there are new changes,
// then execute post pull command
if r.lastCommit == lastCommit {
Logger().Println("No new changes.")
return nil
}
return r.postPullCommand()
}
// Pull performs git clone, or git pull if repository exists
func (r *Repo) pull() error {
params := []string{"clone", "-b", r.Branch, r.URL, r.Path}
if r.pulled {
params = []string{"pull", "origin", r.Branch}
}
// if key is specified, pull using ssh key
if r.KeyPath != "" {
return r.pullWithKey(params)
}
dir := ""
if r.pulled {
dir = r.Path
}
var err error
if err = runCmd(gitBinary, params, dir); err == nil {
r.pulled = true
r.lastPull = time.Now()
Logger().Printf("%v pulled.\n", r.URL)
r.lastCommit, err = r.getMostRecentCommit()
}
return err
}
// pullWithKey is used for private repositories and requires an ssh key.
// Note: currently only limited to Linux and OSX.
func (r *Repo) pullWithKey(params []string) error {
var gitSSH, script gitos.File
// ensure temporary files deleted after usage
defer func() {
if gitSSH != nil {
gos.Remove(gitSSH.Name())
}
if script != nil {
gos.Remove(script.Name())
}
}()
var err error
// write git.sh script to temp file
gitSSH, err = writeScriptFile(gitWrapperScript())
if err != nil {
return err
}
// write git clone bash script to file
script, err = writeScriptFile(bashScript(gitSSH.Name(), r, params))
if err != nil {
return err
}
dir := ""
if r.pulled {
dir = r.Path
}
if err = runCmd(script.Name(), nil, dir); err == nil {
r.pulled = true
r.lastPull = time.Now()
Logger().Printf("%v pulled.\n", r.URL)
r.lastCommit, err = r.getMostRecentCommit()
}
return err
}
// Prepare prepares for a git pull
// and validates the configured directory
func (r *Repo) Prepare() error {
// check if directory exists or is empty
// if not, create directory
fs, err := gos.ReadDir(r.Path)
if err != nil || len(fs) == 0 {
return gos.MkdirAll(r.Path, os.FileMode(0755))
}
// validate git repo
isGit := false
for _, f := range fs {
if f.IsDir() && f.Name() == ".git" {
isGit = true
break
}
}
if isGit {
// check if same repository
var repoURL string
if repoURL, err = r.getRepoURL(); err == nil {
// add .git suffix if missing for adequate comparison.
if !strings.HasSuffix(repoURL, ".git") {
repoURL += ".git"
}
if repoURL == r.URL {
r.pulled = true
return nil
}
}
if err != nil {
return fmt.Errorf("cannot retrieve repo url for %v Error: %v", r.Path, err)
}
return fmt.Errorf("another git repo '%v' exists at %v", repoURL, r.Path)
}
return fmt.Errorf("cannot git clone into %v, directory not empty.", r.Path)
}
// getMostRecentCommit gets the hash of the most recent commit to the
// repository. Useful for checking if changes occur.
func (r *Repo) getMostRecentCommit() (string, error) {
command := gitBinary + ` --no-pager log -n 1 --pretty=format:"%H"`
c, args, err := middleware.SplitCommandAndArgs(command)
if err != nil {
return "", err
}
return runCmdOutput(c, args, r.Path)
}
// getRepoURL retrieves remote origin url for the git repository at path
func (r *Repo) getRepoURL() (string, error) {
_, err := gos.Stat(r.Path)
if err != nil {
return "", err
}
args := []string{"config", "--get", "remote.origin.url"}
return runCmdOutput(gitBinary, args, r.Path)
}
// postPullCommand executes r.Then.
// It is trigged after successful git pull
func (r *Repo) postPullCommand() error {
if r.Then == "" {
return nil
}
c, args, err := middleware.SplitCommandAndArgs(r.Then)
if err != nil {
return err
}
if err = runCmd(c, args, r.Path); err == nil {
Logger().Printf("Command %v successful.\n", r.Then)
}
return err
}
// Init validates git installation, locates the git executable
// binary in PATH and check for available shell to use.
func Init() error {
// prevent concurrent call
initMutex.Lock()
defer initMutex.Unlock()
// if validation has been done before and binary located in
// PATH, return.
if gitBinary != "" {
return nil
}
// locate git binary in path
var err error
if gitBinary, err = gos.LookPath("git"); err != nil {
return fmt.Errorf("git middleware requires git installed. Cannot find git binary in PATH")
}
// locate bash in PATH. If not found, fallback to sh.
// If neither is found, return error.
shell = "bash"
if _, err = gos.LookPath("bash"); err != nil {
shell = "sh"
if _, err = gos.LookPath("sh"); err != nil {
return fmt.Errorf("git middleware requires either bash or sh.")
}
}
return nil
}
// runCmd is a helper function to run commands.
// It runs command with args from directory at dir.
// The executed process outputs to os.Stderr
func runCmd(command string, args []string, dir string) error {
cmd := gos.Command(command, args...)
cmd.Stdout(os.Stderr)
cmd.Stderr(os.Stderr)
cmd.Dir(dir)
if err := cmd.Start(); err != nil {
return err
}
return cmd.Wait()
}
// runCmdOutput is a helper function to run commands and return output.
// It runs command with args from directory at dir.
// If successful, returns output and nil error
func runCmdOutput(command string, args []string, dir string) (string, error) {
cmd := gos.Command(command, args...)
cmd.Dir(dir)
var err error
if output, err := cmd.Output(); err == nil {
return string(bytes.TrimSpace(output)), nil
}
return "", err
}
// writeScriptFile writes content to a temporary file.
// It changes the temporary file mode to executable and
// closes it to prepare it for execution.
func writeScriptFile(content []byte) (file gitos.File, err error) {
if file, err = gos.TempFile("", "caddy"); err != nil {
return nil, err
}
if _, err = file.Write(content); err != nil {
return nil, err
}
if err = file.Chmod(os.FileMode(0755)); err != nil {
return nil, err
}
return file, file.Close()
}
// gitWrapperScript forms content for git.sh script
func gitWrapperScript() []byte {
return []byte(fmt.Sprintf(`#!/bin/%v
# The MIT License (MIT)
# Copyright (c) 2013 Alvin Abad
if [ $# -eq 0 ]; then
echo "Git wrapper script that can specify an ssh-key file
Usage:
git.sh -i ssh-key-file git-command
"
exit 1
fi
# remove temporary file on exit
trap 'rm -f /tmp/.git_ssh.$$' 0
if [ "$1" = "-i" ]; then
SSH_KEY=$2; shift; shift
echo "ssh -i $SSH_KEY \$@" > /tmp/.git_ssh.$$
chmod +x /tmp/.git_ssh.$$
export GIT_SSH=/tmp/.git_ssh.$$
fi
# in case the git command is repeated
[ "$1" = "git" ] && shift
# Run the git command
%v "$@"
`, shell, gitBinary))
}
// bashScript forms content of bash script to clone or update a repo using ssh
func bashScript(gitShPath string, repo *Repo, params []string) []byte {
return []byte(fmt.Sprintf(`#!/bin/%v
mkdir -p ~/.ssh;
touch ~/.ssh/known_hosts;
ssh-keyscan -t rsa,dsa %v 2>&1 | sort -u - ~/.ssh/known_hosts > ~/.ssh/tmp_hosts;
cat ~/.ssh/tmp_hosts >> ~/.ssh/known_hosts;
%v -i %v %v;
`, shell, repo.Host, gitShPath, repo.KeyPath, strings.Join(params, " ")))
}
package git
import (
"io/ioutil"
"log"
"testing"
"time"
"github.com/mholt/caddy/middleware/git/gittest"
)
// init sets the OS used to fakeOS.
func init() {
SetOS(gittest.FakeOS)
}
func check(t *testing.T, err error) {
if err != nil {
t.Errorf("Error not expected but found %v", err)
}
}
func TestInit(t *testing.T) {
err := Init()
check(t, err)
}
func TestHelpers(t *testing.T) {
f, err := writeScriptFile([]byte("script"))
check(t, err)
var b [6]byte
_, err = f.Read(b[:])
check(t, err)
if string(b[:]) != "script" {
t.Errorf("Expected script found %v", string(b[:]))
}
out, err := runCmdOutput(gitBinary, []string{"-version"}, "")
check(t, err)
if out != gittest.CmdOutput {
t.Errorf("Expected %v found %v", gittest.CmdOutput, out)
}
err = runCmd(gitBinary, []string{"-version"}, "")
check(t, err)
wScript := gitWrapperScript()
if string(wScript) != expectedWrapperScript {
t.Errorf("Expected %v found %v", expectedWrapperScript, string(wScript))
}
f, err = writeScriptFile(wScript)
check(t, err)
repo := &Repo{Host: "github.com", KeyPath: "~/.key"}
script := string(bashScript(f.Name(), repo, []string{"clone", "git@github.com/repo/user"}))
if script != expectedBashScript {
t.Errorf("Expected %v found %v", expectedBashScript, script)
}
}
func TestGit(t *testing.T) {
// prepare
repos := []*Repo{
nil,
&Repo{Path: "gitdir", URL: "success.git"},
}
for _, r := range repos {
repo := createRepo(r)
err := repo.Prepare()
check(t, err)
}
// pull with success
logFile := gittest.Open("file")
SetLogger(log.New(logFile, "", 0))
tests := []struct {
repo *Repo
output string
}{
{
&Repo{Path: "gitdir", URL: "git@github.com:user/repo.git", KeyPath: "~/.key", Then: "echo Hello"},
`git@github.com:user/repo.git pulled.
Command echo Hello successful.
`,
},
{
&Repo{Path: "gitdir", URL: "https://github.com/user/repo.git", Then: "echo Hello"},
`https://github.com/user/repo.git pulled.
Command echo Hello successful.
`,
},
{
&Repo{URL: "git@github.com:user/repo"},
`git@github.com:user/repo pulled.
`,
},
}
for i, test := range tests {
gittest.CmdOutput = test.repo.URL
test.repo = createRepo(test.repo)
err := test.repo.Prepare()
check(t, err)
err = test.repo.Pull()
check(t, err)
out, err := ioutil.ReadAll(logFile)
check(t, err)
if test.output != string(out) {
t.Errorf("Pull with Success %v: Expected %v found %v", i, test.output, string(out))
}
}
// pull with error
repos = []*Repo{
&Repo{Path: "gitdir", URL: "http://github.com:u/repo.git"},
&Repo{Path: "gitdir", URL: "https://github.com/user/repo.git", Then: "echo Hello"},
&Repo{Path: "gitdir"},
&Repo{Path: "gitdir", KeyPath: ".key"},
}
gittest.CmdOutput = "git@github.com:u1/repo.git"
for i, repo := range repos {
repo = createRepo(repo)
err := repo.Prepare()
if err == nil {
t.Errorf("Pull with Error %v: Error expected but not found %v", i, err)
continue
}
expected := "another git repo 'git@github.com:u1/repo.git' exists at gitdir"
if expected != err.Error() {
t.Errorf("Pull with Error %v: Expected %v found %v", i, expected, err.Error())
}
}
// timeout checks
timeoutTests := []struct {
repo *Repo
shouldPull bool
}{
{&Repo{Interval: time.Millisecond * 4900}, false},
{&Repo{Interval: time.Millisecond * 1}, false},
{&Repo{Interval: time.Second * 5}, true},
{&Repo{Interval: time.Second * 10}, true},
}
for i, r := range timeoutTests {
r.repo = createRepo(r.repo)
err := r.repo.Prepare()
check(t, err)
err = r.repo.Pull()
check(t, err)
before := r.repo.lastPull
gittest.Sleep(r.repo.Interval)
err = r.repo.Pull()
after := r.repo.lastPull
check(t, err)
expected := after.After(before)
if expected != r.shouldPull {
t.Errorf("Pull with Error %v: Expected %v found %v", i, expected, r.shouldPull)
}
}
}
func createRepo(r *Repo) *Repo {
repo := &Repo{
URL: "git@github.com/user/test",
Path: ".",
Host: "github.com",
Branch: "master",
Interval: time.Second * 60,
}
if r == nil {
return repo
}
if r.Branch != "" {
repo.Branch = r.Branch
}
if r.Host != "" {
repo.Branch = r.Branch
}
if r.Interval != 0 {
repo.Interval = r.Interval
}
if r.KeyPath != "" {
repo.KeyPath = r.KeyPath
}
if r.Path != "" {
repo.Path = r.Path
}
if r.Then != "" {
repo.Then = r.Then
}
if r.URL != "" {
repo.URL = r.URL
}
return repo
}
var expectedBashScript = `#!/bin/bash
mkdir -p ~/.ssh;
touch ~/.ssh/known_hosts;
ssh-keyscan -t rsa,dsa github.com 2>&1 | sort -u - ~/.ssh/known_hosts > ~/.ssh/tmp_hosts;
cat ~/.ssh/tmp_hosts >> ~/.ssh/known_hosts;
` + gittest.TempFileName + ` -i ~/.key clone git@github.com/repo/user;
`
var expectedWrapperScript = `#!/bin/bash
# The MIT License (MIT)
# Copyright (c) 2013 Alvin Abad
if [ $# -eq 0 ]; then
echo "Git wrapper script that can specify an ssh-key file
Usage:
git.sh -i ssh-key-file git-command
"
exit 1
fi
# remove temporary file on exit
trap 'rm -f /tmp/.git_ssh.$$' 0
if [ "$1" = "-i" ]; then
SSH_KEY=$2; shift; shift
echo "ssh -i $SSH_KEY \$@" > /tmp/.git_ssh.$$
chmod +x /tmp/.git_ssh.$$
export GIT_SSH=/tmp/.git_ssh.$$
fi
# in case the git command is repeated
[ "$1" = "git" ] && shift
# Run the git command
/usr/bin/git "$@"
`
package gitos
import (
"io"
"io/ioutil"
"os"
"os/exec"
"time"
)
// File is an abstraction for file (os.File).
type File interface {
// Name returns the name of the file
Name() string
// Stat returns the FileInfo structure describing file.
Stat() (os.FileInfo, error)
// Close closes the File, rendering it unusable for I/O.
Close() error
// Chmod changes the mode of the file.
Chmod(os.FileMode) error
// Read reads up to len(b) bytes from the File. It returns the number of
// bytes read and an error, if any.
Read([]byte) (int, error)
// Write writes len(b) bytes to the File. It returns the number of bytes
// written and an error, if any.
Write([]byte) (int, error)
}
// Cmd is an abstraction for external commands (os.Cmd).
type Cmd interface {
// Run starts the specified command and waits for it to complete.
Run() error
// Start starts the specified command but does not wait for it to complete.
Start() error
// Wait waits for the command to exit. It must have been started by Start.
Wait() error
// Output runs the command and returns its standard output.
Output() ([]byte, error)
// Dir sets the working directory of the command.
Dir(string)
// Stdin sets the process's standard input.
Stdin(io.Reader)
// Stdout sets the process's standard output.
Stdout(io.Writer)
// Stderr sets the process's standard output.
Stderr(io.Writer)
}
// gitCmd represents external commands executed by git.
type gitCmd struct {
*exec.Cmd
}
// Dir sets the working directory of the command.
func (g *gitCmd) Dir(dir string) {
g.Cmd.Dir = dir
}
// Stdin sets the process's standard input.
func (g *gitCmd) Stdin(stdin io.Reader) {
g.Cmd.Stdin = stdin
}
// Stdout sets the process's standard output.
func (g *gitCmd) Stdout(stdout io.Writer) {
g.Cmd.Stdout = stdout
}
// Stderr sets the process's standard output.
func (g *gitCmd) Stderr(stderr io.Writer) {
g.Cmd.Stderr = stderr
}
// OS is an abstraction for required OS level functions.
type OS interface {
// Command returns the Cmd to execute the named program with the
// given arguments.
Command(string, ...string) Cmd
// Mkdir creates a new directory with the specified name and permission
// bits.
Mkdir(string, os.FileMode) error
// MkdirAll creates a directory named path, along with any necessary
// parents.
MkdirAll(string, os.FileMode) error
// Stat returns a FileInfo describing the named file.
Stat(string) (os.FileInfo, error)
// Remove removes the named file or directory.
Remove(string) error
// ReadDir reads the directory named by dirname and returns a list of
// directory entries.
ReadDir(string) ([]os.FileInfo, error)
// LookPath searches for an executable binary named file in the directories
// named by the PATH environment variable.
LookPath(string) (string, error)
// TempFile creates a new temporary file in the directory dir with a name
// beginning with prefix, opens the file for reading and writing, and
// returns the resulting File.
TempFile(string, string) (File, error)
// Sleep pauses the current goroutine for at least the duration d. A
// negative or zero duration causes Sleep to return immediately.
Sleep(time.Duration)
// NewTicker returns a new Ticker containing a channel that will send the
// time with a period specified by the argument.
NewTicker(time.Duration) Ticker
// TimeSince returns the time elapsed since the argument.
TimeSince(time.Time) time.Duration
}
// Ticker is an abstraction for Ticker (time.Ticker)
type Ticker interface {
C() <-chan time.Time
Stop()
}
// GitTicker is the implementation of Ticker for git.
type GitTicker struct {
*time.Ticker
}
// C returns the channel on which the ticks are delivered.s
func (g *GitTicker) C() <-chan time.Time {
return g.Ticker.C
}
// GitOS is the implementation of OS for git.
type GitOS struct{}
// Mkdir calls os.Mkdir.
func (g GitOS) Mkdir(name string, perm os.FileMode) error {
return os.Mkdir(name, perm)
}
// MkdirAll calls os.MkdirAll.
func (g GitOS) MkdirAll(path string, perm os.FileMode) error {
return os.MkdirAll(path, perm)
}
// Stat calls os.Stat.
func (g GitOS) Stat(name string) (os.FileInfo, error) {
return os.Stat(name)
}
// Remove calls os.Remove.
func (g GitOS) Remove(name string) error {
return os.Remove(name)
}
// LookPath calls exec.LookPath.
func (g GitOS) LookPath(file string) (string, error) {
return exec.LookPath(file)
}
// TempFile calls ioutil.TempFile.
func (g GitOS) TempFile(dir, prefix string) (File, error) {
return ioutil.TempFile(dir, prefix)
}
// ReadDir calls ioutil.ReadDir.
func (g GitOS) ReadDir(dirname string) ([]os.FileInfo, error) {
return ioutil.ReadDir(dirname)
}
// Command calls exec.Command.
func (g GitOS) Command(name string, args ...string) Cmd {
return &gitCmd{exec.Command(name, args...)}
}
// Sleep calls time.Sleep.
func (g GitOS) Sleep(d time.Duration) {
time.Sleep(d)
}
// New Ticker calls time.NewTicker.
func (g GitOS) NewTicker(d time.Duration) Ticker {
return &GitTicker{time.NewTicker(d)}
}
// TimeSince calls time.Since
func (g GitOS) TimeSince(t time.Time) time.Duration {
return time.Since(t)
}
// Package gittest is a test package for the git middleware.
// It implements a mock gitos.OS, gitos.Cmd and gitos.File.
package gittest
import (
"io"
"log"
"os"
"sync"
"time"
"github.com/mholt/caddy/middleware/git/gitos"
)
// FakeOS implements a mock gitos.OS, gitos.Cmd and gitos.File.
var FakeOS = fakeOS{}
// CmdOutput is the output of any call to the mocked gitos.Cmd's Output().
var CmdOutput = "success"
// TempFileName is the name of any file returned by mocked gitos.OS's TempFile().
var TempFileName = "tempfile"
// TimeSpeed is how faster the mocked gitos.Ticker and gitos.Sleep should run.
var TimeSpeed = 5
// dirs mocks a fake git dir if filename is "gitdir".
var dirs = map[string][]os.FileInfo{
"gitdir": {
fakeInfo{name: ".git", dir: true},
},
}
// Open creates a new mock gitos.File.
func Open(name string) gitos.File {
return &fakeFile{name: name}
}
// Sleep calls fake time.Sleep
func Sleep(d time.Duration) {
FakeOS.Sleep(d)
}
// NewLogger creates a logger that logs to f
func NewLogger(f gitos.File) *log.Logger {
return log.New(f, "", 0)
}
// fakeFile is a mock gitos.File.
type fakeFile struct {
name string
dir bool
content []byte
info fakeInfo
sync.Mutex
}
func (f fakeFile) Name() string {
return f.name
}
func (f fakeFile) Stat() (os.FileInfo, error) {
return fakeInfo{name: f.name}, nil
}
func (f fakeFile) Close() error {
return nil
}
func (f fakeFile) Chmod(mode os.FileMode) error {
f.info.mode = mode
return nil
}
func (f *fakeFile) Read(b []byte) (int, error) {
f.Lock()
defer f.Unlock()
if len(f.content) == 0 {
return 0, io.EOF
}
n := copy(b, f.content)
f.content = f.content[n:]
return n, nil
}
func (f *fakeFile) Write(b []byte) (int, error) {
f.Lock()
defer f.Unlock()
f.content = append(f.content, b...)
return len(b), nil
}
// fakeCmd is a mock gitos.Cmd.
type fakeCmd struct{}
func (f fakeCmd) Run() error {
return nil
}
func (f fakeCmd) Start() error {
return nil
}
func (f fakeCmd) Wait() error {
return nil
}
func (f fakeCmd) Output() ([]byte, error) {
return []byte(CmdOutput), nil
}
func (f fakeCmd) Dir(dir string) {}
func (f fakeCmd) Stdin(stdin io.Reader) {}
func (f fakeCmd) Stdout(stdout io.Writer) {}
func (f fakeCmd) Stderr(stderr io.Writer) {}
// fakeInfo is a mock os.FileInfo.
type fakeInfo struct {
name string
dir bool
mode os.FileMode
}
func (f fakeInfo) Name() string {
return f.name
}
func (f fakeInfo) Size() int64 {
return 1024
}
func (f fakeInfo) Mode() os.FileMode {
return f.mode
}
func (f fakeInfo) ModTime() time.Time {
return time.Now().Truncate(time.Hour)
}
func (f fakeInfo) IsDir() bool {
return f.dir
}
func (f fakeInfo) Sys() interface{} {
return nil
}
// fakeTicker is a mock gitos.Ticker
type fakeTicker struct {
*time.Ticker
}
func (f fakeTicker) C() <-chan time.Time {
return f.Ticker.C
}
// fakeOS is a mock gitos.OS.
type fakeOS struct{}
func (f fakeOS) Mkdir(name string, perm os.FileMode) error {
return nil
}
func (f fakeOS) MkdirAll(path string, perm os.FileMode) error {
return nil
}
func (f fakeOS) Stat(name string) (os.FileInfo, error) {
return fakeInfo{name: name}, nil
}
func (f fakeOS) Remove(name string) error {
return nil
}
func (f fakeOS) LookPath(file string) (string, error) {
return "/usr/bin/" + file, nil
}
func (f fakeOS) TempFile(dir, prefix string) (gitos.File, error) {
return &fakeFile{name: TempFileName, info: fakeInfo{name: TempFileName}}, nil
}
func (f fakeOS) ReadDir(dirname string) ([]os.FileInfo, error) {
if f, ok := dirs[dirname]; ok {
return f, nil
}
return nil, nil
}
func (f fakeOS) Command(name string, args ...string) gitos.Cmd {
return fakeCmd{}
}
func (f fakeOS) Sleep(d time.Duration) {
time.Sleep(d / time.Duration(TimeSpeed))
}
func (f fakeOS) NewTicker(d time.Duration) gitos.Ticker {
return &fakeTicker{time.NewTicker(d / time.Duration(TimeSpeed))}
}
func (f fakeOS) TimeSince(t time.Time) time.Duration {
return time.Since(t) * time.Duration(TimeSpeed)
}
package git
import (
"log"
"os"
"sync"
)
// logger is used to log errors
var logger = &gitLogger{l: log.New(os.Stderr, "", log.LstdFlags)}
// gitLogger wraps log.Logger with mutex for thread safety.
type gitLogger struct {
l *log.Logger
sync.RWMutex
}
func (g *gitLogger) logger() *log.Logger {
g.RLock()
defer g.RUnlock()
return g.l
}
func (g *gitLogger) setLogger(l *log.Logger) {
g.Lock()
g.l = l
g.Unlock()
}
// Logger gets the currently available logger
func Logger() *log.Logger {
return logger.logger()
}
// SetLogger sets the current logger to l
func SetLogger(l *log.Logger) {
logger.setLogger(l)
}
package git
import "github.com/mholt/caddy/middleware/git/gitos"
// gos is the OS used by git.
var gos gitos.OS = gitos.GitOS{}
// SetOS sets the OS to be used. Intended to be used for tests
// to abstract OS level git actions.
func SetOS(os gitos.OS) {
gos = os
}
package git
import (
"sync"
"github.com/mholt/caddy/middleware/git/gitos"
)
// repoService is the service that runs in background and periodically
// pull from the repository.
type repoService struct {
repo *Repo
ticker gitos.Ticker // ticker to tick at intervals
halt chan struct{} // channel to notify service to halt and stop pulling.
}
// Start starts a new background service to pull periodically.
func Start(repo *Repo) {
service := &repoService{
repo,
gos.NewTicker(repo.Interval),
make(chan struct{}),
}
go func(s *repoService) {
for {
select {
case <-s.ticker.C():
err := repo.Pull()
if err != nil {
Logger().Println(err)
}
case <-s.halt:
s.ticker.Stop()
return
}
}
}(service)
// add to services to make it stoppable
Services.add(service)
}
// services stores all repoServices
type services struct {
services []*repoService
sync.Mutex
}
// add adds a new service to list of services.
func (s *services) add(r *repoService) {
s.Lock()
defer s.Unlock()
s.services = append(s.services, r)
}
// Stop stops at most `limit` running services pulling from git repo at
// repoURL. It waits until the service is terminated before returning.
// If limit is less than zero, it is ignored.
// TODO find better ways to identify repos
func (s *services) Stop(repoURL string, limit int) {
s.Lock()
defer s.Unlock()
// locate repos
for i, j := 0, 0; i < len(s.services) && ((limit >= 0 && j < limit) || limit < 0); i++ {
service := s.services[i]
if service.repo.URL == repoURL {
// send halt signal
service.halt <- struct{}{}
s.services[i] = nil
j++
}
}
// remove them from repos list
services := s.services[:0]
for _, s := range s.services {
if s != nil {
services = append(services, s)
}
}
s.services = services
}
package git
import (
"fmt"
"testing"
"time"
"github.com/mholt/caddy/middleware/git/gittest"
)
func init() {
SetOS(gittest.FakeOS)
}
func Test(t *testing.T) {
repo := &Repo{URL: "git@github.com", Interval: time.Second}
Start(repo)
if len(Services.services) != 1 {
t.Errorf("Expected 1 service, found %v", len(Services.services))
}
Services.Stop(repo.URL, 1)
if len(Services.services) != 0 {
t.Errorf("Expected 1 service, found %v", len(Services.services))
}
repos := make([]*Repo, 5)
for i := 0; i < 5; i++ {
repos[i] = &Repo{URL: fmt.Sprintf("test%v", i), Interval: time.Second * 2}
Start(repos[i])
if len(Services.services) != i+1 {
t.Errorf("Expected %v service(s), found %v", i+1, len(Services.services))
}
}
gos.Sleep(time.Second * 5)
Services.Stop(repos[0].URL, 1)
if len(Services.services) != 4 {
t.Errorf("Expected %v service(s), found %v", 4, len(Services.services))
}
repo = &Repo{URL: "git@github.com", Interval: time.Second}
Start(repo)
if len(Services.services) != 5 {
t.Errorf("Expected %v service(s), found %v", 5, len(Services.services))
}
repo = &Repo{URL: "git@github.com", Interval: time.Second * 2}
Start(repo)
if len(Services.services) != 6 {
t.Errorf("Expected %v service(s), found %v", 6, len(Services.services))
}
gos.Sleep(time.Second * 5)
Services.Stop(repo.URL, -1)
if len(Services.services) != 4 {
t.Errorf("Expected %v service(s), found %v", 4, len(Services.services))
}
}
package webhook
import (
"crypto/hmac"
"crypto/sha1"
"encoding/hex"
"encoding/json"
"errors"
"io/ioutil"
"log"
"net/http"
"strings"
"github.com/mholt/caddy/middleware/git"
)
type GithubHook struct{}
type ghRelease struct {
Action string `json:"action"`
Release struct {
TagName string `json:"tag_name"`
Name interface{} `json:"name"`
} `json:"release"`
}
type ghPush struct {
Ref string `json:"ref"`
}
// logger is an helper function to retrieve the available logger
func logger() *log.Logger {
return git.Logger()
}
func (g GithubHook) DoesHandle(h http.Header) bool {
userAgent := h.Get("User-Agent")
// GitHub always uses a user-agent like "GitHub-Hookshot/<id>"
if userAgent != "" && strings.HasPrefix(userAgent, "GitHub-Hookshot") {
return true
}
return false
}
func (g GithubHook) Handle(w http.ResponseWriter, r *http.Request, repo *git.Repo) (int, error) {
if r.Method != "POST" {
return http.StatusMethodNotAllowed, errors.New("the request had an invalid method.")
}
// read full body - required for signature
body, err := ioutil.ReadAll(r.Body)
err = g.handleSignature(r, body, repo.HookSecret)
if err != nil {
return http.StatusBadRequest, err
}
event := r.Header.Get("X-Github-Event")
if event == "" {
return http.StatusBadRequest, errors.New("the 'X-Github-Event' header is required but was missing.")
}
switch event {
case "ping":
w.Write([]byte("pong"))
case "push":
err := g.handlePush(body, repo)
if err != nil {
return http.StatusBadRequest, err
}
case "release":
err := g.handleRelease(body, repo)
if err != nil {
return http.StatusBadRequest, err
}
// return 400 if we do not handle the event type.
// This is to visually show the user a configuration error in the GH ui.
default:
return http.StatusBadRequest, nil
}
return http.StatusOK, nil
}
// Check for an optional signature in the request
// if it is signed, verify the signature.
func (g GithubHook) handleSignature(r *http.Request, body []byte, secret string) error {
signature := r.Header.Get("X-Hub-Signature")
if signature != "" {
if secret == "" {
logger().Print("Unable to verify request signature. Secret not set in caddyfile!\n")
} else {
mac := hmac.New(sha1.New, []byte(secret))
mac.Write(body)
expectedMac := hex.EncodeToString(mac.Sum(nil))
if signature[5:] != expectedMac {
return errors.New("could not verify request signature. The signature is invalid!")
}
}
}
return nil
}
func (g GithubHook) handlePush(body []byte, repo *git.Repo) error {
var push ghPush
err := json.Unmarshal(body, &push)
if err != nil {
return err
}
// extract the branch being pushed from the ref string
// and if it matches with our locally tracked one, pull.
refSlice := strings.Split(push.Ref, "/")
if len(refSlice) != 3 {
return errors.New("the push request contained an invalid reference string.")
}
branch := refSlice[2]
if branch == repo.Branch {
logger().Print("Received pull notification for the tracking branch, updating...\n")
repo.Pull()
}
return nil
}
func (g GithubHook) handleRelease(body []byte, repo *git.Repo) error {
var release ghRelease
err := json.Unmarshal(body, &release)
if err != nil {
return err
}
if release.Release.TagName == "" {
return errors.New("the release request contained an invalid TagName.")
}
logger().Printf("Received new release '%s'. -> Updating local repository to this release.\n", release.Release.Name)
// Update the local branch to the release tag name
// this will pull the release tag.
repo.Branch = release.Release.TagName
repo.Pull()
return nil
}
package webhook
import (
"bytes"
"github.com/mholt/caddy/middleware/git"
"net/http"
"net/http/httptest"
"testing"
)
func TestGithubDeployPush(t *testing.T) {
repo := &git.Repo{Branch: "master", HookUrl: "/github_deploy", HookSecret: "supersecret"}
ghHook := GithubHook{}
for i, test := range []struct {
body string
event string
responseBody string
code int
}{
{"", "", "", 400},
{"", "push", "", 400},
{pushBodyOther, "push", "", 200},
{pushBodyPartial, "push", "", 400},
{"", "release", "", 400},
{"", "ping", "pong", 200},
} {
req, err := http.NewRequest("POST", "/github_deploy", bytes.NewBuffer([]byte(test.body)))
if err != nil {
t.Fatalf("Test %v: Could not create HTTP request: %v", i, err)
}
if test.event != "" {
req.Header.Add("X-Github-Event", test.event)
}
rec := httptest.NewRecorder()
code, err := ghHook.Handle(rec, req, repo)
if code != test.code {
t.Errorf("Test %d: Expected response code to be %d but was %d", i, test.code, code)
}
if rec.Body.String() != test.responseBody {
t.Errorf("Test %d: Expected response body to be '%v' but was '%v'", i, test.responseBody, rec.Body.String())
}
}
}
var pushBodyPartial = `
{
"ref": ""
}
`
var pushBodyOther = `
{
"ref": "refs/heads/some-other-branch"
}
`
package webhook
import (
"github.com/mholt/caddy/middleware"
"github.com/mholt/caddy/middleware/git"
"net/http"
)
// Middleware for handling web hooks of git providers
type WebHook struct {
Repo *git.Repo
Next middleware.Handler
}
// Interface for specific providers to implement.
type hookHandler interface {
DoesHandle(http.Header) bool
Handle(w http.ResponseWriter, r *http.Request, repo *git.Repo) (int, error)
}
// Slice of all registered hookHandlers.
// Register new hook handlers here!
var handlers = []hookHandler{
GithubHook{},
}
// ServeHTTP implements the middlware.Handler interface.
func (h WebHook) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, error) {
if r.URL.Path == h.Repo.HookUrl {
for _, handler := range handlers {
// if a handler indicates it does handle the request,
// we do not try other handlers. Only one handler ever
// handles a specific request.
if handler.DoesHandle(r.Header) {
return handler.Handle(w, r, h.Repo)
}
}
}
return h.Next.ServeHTTP(w, r)
}
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment