Commit 540f0ead authored by Kirill Smelkov's avatar Kirill Smelkov

*: Add cancellation handling via contexts

Add ctx parameter to cmd_pull, cmd_restore and inner functions that they
call that can block and handle ctx cancel where we can. For now both
pull and restore are always run under background context, but in the
next patch we'll connect SIGINT+SIGTERM to cancel spawned work.

In general a service - even a command line utility - needs to handle
cancellation properly and itself to maintain consistency of external
state. See e.g.

	https://callistaenterprise.se/blogg/teknik/2019/10/05/go-worker-cancellation/

for example.
parent 6af054b0
This diff is collapsed.
...@@ -20,6 +20,7 @@ ...@@ -20,6 +20,7 @@
package main package main
import ( import (
"context"
"fmt" "fmt"
"io/ioutil" "io/ioutil"
"os" "os"
...@@ -70,12 +71,14 @@ func xgittype(s string) git.ObjectType { ...@@ -70,12 +71,14 @@ func xgittype(s string) git.ObjectType {
// xnoref asserts that git reference ref does not exists. // xnoref asserts that git reference ref does not exists.
func xnoref(ref string) { func xnoref(ref string) {
xgit("update-ref", "--stdin", RunWith{stdin: fmt.Sprintf("verify refs/%s %s\n", ref, Sha1{})}) xgit(context.Background(), "update-ref", "--stdin", RunWith{stdin: fmt.Sprintf("verify refs/%s %s\n", ref, Sha1{})})
} }
// verify end-to-end pull-restore // verify end-to-end pull-restore
func TestPullRestore(t *testing.T) { func TestPullRestore(t *testing.T) {
ctx := context.Background()
// if something raises -> don't let testing panic - report it as proper error with context. // if something raises -> don't let testing panic - report it as proper error with context.
here := my.FuncName() here := my.FuncName()
defer exc.Catch(func(e *exc.Error) { defer exc.Catch(func(e *exc.Error) {
...@@ -114,7 +117,7 @@ func TestPullRestore(t *testing.T) { ...@@ -114,7 +117,7 @@ func TestPullRestore(t *testing.T) {
} }
// init backup repository // init backup repository
xgit("init", "--bare", "backup.git") xgit(ctx, "init", "--bare", "backup.git")
xchdir(t, "backup.git") xchdir(t, "backup.git")
gb, err := git.OpenRepository(".") gb, err := git.OpenRepository(".")
if err != nil { if err != nil {
...@@ -123,10 +126,10 @@ func TestPullRestore(t *testing.T) { ...@@ -123,10 +126,10 @@ func TestPullRestore(t *testing.T) {
// pull from testdata // pull from testdata
my0 := mydir + "/testdata/0" my0 := mydir + "/testdata/0"
cmd_pull(gb, []string{my0 + ":b0"}) // only empty repo in testdata/0 cmd_pull(ctx, gb, []string{my0 + ":b0"}) // only empty repo in testdata/0
my1 := mydir + "/testdata/1" my1 := mydir + "/testdata/1"
cmd_pull(gb, []string{my1 + ":b1"}) cmd_pull(ctx, gb, []string{my1 + ":b1"})
// verify tag/tree/blob encoding is 1) consistent and 2) always the same. // verify tag/tree/blob encoding is 1) consistent and 2) always the same.
// we need it be always the same so different git-backup versions can // we need it be always the same so different git-backup versions can
...@@ -158,8 +161,8 @@ func TestPullRestore(t *testing.T) { ...@@ -158,8 +161,8 @@ func TestPullRestore(t *testing.T) {
} }
// encoding original object should give sha1_ // encoding original object should give sha1_
obj_type := xgit("cat-file", "-t", nc.sha1) obj_type := xgit(ctx, "cat-file", "-t", nc.sha1)
sha1_ := obj_represent_as_commit(gb, nc.sha1, xgittype(obj_type)) sha1_ := obj_represent_as_commit(ctx, gb, nc.sha1, xgittype(obj_type))
if sha1_ != nc.sha1_ { if sha1_ != nc.sha1_ {
t.Fatalf("encode %s -> %s ; want %s", sha1, sha1_, nc.sha1_) t.Fatalf("encode %s -> %s ; want %s", sha1, sha1_, nc.sha1_)
} }
...@@ -181,10 +184,10 @@ func TestPullRestore(t *testing.T) { ...@@ -181,10 +184,10 @@ func TestPullRestore(t *testing.T) {
} }
// prune all non-reachable objects (e.g. tags just pulled - they were encoded as commits) // prune all non-reachable objects (e.g. tags just pulled - they were encoded as commits)
xgit("prune") xgit(ctx, "prune")
// verify backup repo is all ok // verify backup repo is all ok
xgit("fsck") xgit(ctx, "fsck")
// verify that just pulled tag objects are now gone after pruning - // verify that just pulled tag objects are now gone after pruning -
// - they become not directly git-present. The only possibility to // - they become not directly git-present. The only possibility to
...@@ -193,7 +196,7 @@ func TestPullRestore(t *testing.T) { ...@@ -193,7 +196,7 @@ func TestPullRestore(t *testing.T) {
if !nc.istag { if !nc.istag {
continue continue
} }
gerr, _, _ := ggit("cat-file", "-p", nc.sha1) gerr, _, _ := ggit(ctx, "cat-file", "-p", nc.sha1)
if gerr == nil { if gerr == nil {
t.Fatalf("tag %s still present in backup.git after git-prune", nc.sha1) t.Fatalf("tag %s still present in backup.git after git-prune", nc.sha1)
} }
...@@ -210,14 +213,14 @@ func TestPullRestore(t *testing.T) { ...@@ -210,14 +213,14 @@ func TestPullRestore(t *testing.T) {
afterPull() afterPull()
// pull again - it should be noop // pull again - it should be noop
h1 := xgitSha1("rev-parse", "HEAD") h1 := xgitSha1(ctx, "rev-parse", "HEAD")
cmd_pull(gb, []string{my1 + ":b1"}) cmd_pull(ctx, gb, []string{my1 + ":b1"})
afterPull() afterPull()
h2 := xgitSha1("rev-parse", "HEAD") h2 := xgitSha1(ctx, "rev-parse", "HEAD")
if h1 == h2 { if h1 == h2 {
t.Fatal("pull: second run did not ajusted HEAD") t.Fatal("pull: second run did not ajusted HEAD")
} }
δ12 := xgit("diff", h1, h2) δ12 := xgit(ctx, "diff", h1, h2)
if δ12 != "" { if δ12 != "" {
t.Fatalf("pull: second run was not noop: δ:\n%s", δ12) t.Fatalf("pull: second run was not noop: δ:\n%s", δ12)
} }
...@@ -225,10 +228,10 @@ func TestPullRestore(t *testing.T) { ...@@ -225,10 +228,10 @@ func TestPullRestore(t *testing.T) {
// restore backup // restore backup
work1 := workdir + "/1" work1 := workdir + "/1"
cmd_restore(gb, []string{"HEAD", "b1:" + work1}) cmd_restore(ctx, gb, []string{"HEAD", "b1:" + work1})
// verify files restored to the same as original // verify files restored to the same as original
gerr, diff, _ := ggit("diff", "--no-index", "--raw", "--exit-code", my1, work1) gerr, diff, _ := ggit(ctx, "diff", "--no-index", "--raw", "--exit-code", my1, work1)
// 0 - no diff, 1 - has diff, 2 - problem // 0 - no diff, 1 - has diff, 2 - problem
if gerr != nil && gerr.Sys().(syscall.WaitStatus).ExitStatus() > 1 { if gerr != nil && gerr.Sys().(syscall.WaitStatus).ExitStatus() > 1 {
t.Fatal(gerr) t.Fatal(gerr)
...@@ -267,12 +270,12 @@ func TestPullRestore(t *testing.T) { ...@@ -267,12 +270,12 @@ func TestPullRestore(t *testing.T) {
for _, repo := range R { for _, repo := range R {
// fsck just in case // fsck just in case
xgit("--git-dir="+repo.path, "fsck") xgit(ctx, "--git-dir="+repo.path, "fsck")
// NOTE for-each-ref sorts output by refname // NOTE for-each-ref sorts output by refname
repo.reflist = xgit("--git-dir="+repo.path, "for-each-ref") repo.reflist = xgit(ctx, "--git-dir="+repo.path, "for-each-ref")
// NOTE rev-list emits objects in reverse chronological order, // NOTE rev-list emits objects in reverse chronological order,
// starting from refs roots which are also ordered by refname // starting from refs roots which are also ordered by refname
repo.revlist = xgit("--git-dir="+repo.path, "rev-list", "--all", "--objects") repo.revlist = xgit(ctx, "--git-dir="+repo.path, "rev-list", "--all", "--objects")
} }
if R[0].reflist != R[1].reflist { if R[0].reflist != R[1].reflist {
...@@ -301,7 +304,7 @@ func TestPullRestore(t *testing.T) { ...@@ -301,7 +304,7 @@ func TestPullRestore(t *testing.T) {
xnoref("backup.locked") xnoref("backup.locked")
}) })
cmd_pull(gb, []string{my2 + ":b2"}) cmd_pull(ctx, gb, []string{my2 + ":b2"})
t.Fatal("pull corrupt.git: did not complain") t.Fatal("pull corrupt.git: did not complain")
}() }()
...@@ -341,7 +344,7 @@ func TestPullRestore(t *testing.T) { ...@@ -341,7 +344,7 @@ func TestPullRestore(t *testing.T) {
err = os.Setenv("HOME", my3+"/incomplete-send-pack.git/"+kind) err = os.Setenv("HOME", my3+"/incomplete-send-pack.git/"+kind)
exc.Raiseif(err) exc.Raiseif(err)
cmd_pull(gb, []string{my3 + ":b3"}) cmd_pull(ctx, gb, []string{my3 + ":b3"})
t.Fatalf("pull incomplete-send-pack.git/%s: did not complain", kind) t.Fatalf("pull incomplete-send-pack.git/%s: did not complain", kind)
} }
...@@ -358,7 +361,7 @@ func TestPullRestore(t *testing.T) { ...@@ -358,7 +361,7 @@ func TestPullRestore(t *testing.T) {
// pulling incomplete-send-pack.git without pack-objects hook must succeed: // pulling incomplete-send-pack.git without pack-objects hook must succeed:
// without $HOME tweaks full and complete pack is sent. // without $HOME tweaks full and complete pack is sent.
cmd_pull(gb, []string{my3 + ":b3"}) cmd_pull(ctx, gb, []string{my3 + ":b3"})
} }
func TestRepoRefSplit(t *testing.T) { func TestRepoRefSplit(t *testing.T) {
......
// Copyright (C) 2015-2016 Nexedi SA and Contributors. // Copyright (C) 2015-2020 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com> // Kirill Smelkov <kirr@nexedi.com>
// //
// This program is free software: you can Use, Study, Modify and Redistribute // This program is free software: you can Use, Study, Modify and Redistribute
...@@ -22,6 +22,7 @@ package main ...@@ -22,6 +22,7 @@ package main
import ( import (
"bytes" "bytes"
"context"
"fmt" "fmt"
"os" "os"
"os/exec" "os/exec"
...@@ -48,18 +49,20 @@ type RunWith struct { ...@@ -48,18 +49,20 @@ type RunWith struct {
} }
// run `git *argv` -> error, stdout, stderr // run `git *argv` -> error, stdout, stderr
func _git(argv []string, ctx RunWith) (err error, stdout, stderr string) { func _git(ctx context.Context, argv []string, rctx RunWith) (err error, stdout, stderr string) {
debugf("git %s", strings.Join(argv, " ")) debugf("git %s", strings.Join(argv, " "))
cmd := exec.Command("git", argv...) // XXX exec.CommandContext does `kill -9` on ctx cancel
// XXX -> rework to `kill -TERM` so that spawned process can finish cleanly?
cmd := exec.CommandContext(ctx, "git", argv...)
stdoutBuf := bytes.Buffer{} stdoutBuf := bytes.Buffer{}
stderrBuf := bytes.Buffer{} stderrBuf := bytes.Buffer{}
if ctx.stdin != "" { if rctx.stdin != "" {
cmd.Stdin = strings.NewReader(ctx.stdin) cmd.Stdin = strings.NewReader(rctx.stdin)
} }
switch ctx.stdout { switch rctx.stdout {
case PIPE: case PIPE:
cmd.Stdout = &stdoutBuf cmd.Stdout = &stdoutBuf
case DontRedirect: case DontRedirect:
...@@ -68,7 +71,7 @@ func _git(argv []string, ctx RunWith) (err error, stdout, stderr string) { ...@@ -68,7 +71,7 @@ func _git(argv []string, ctx RunWith) (err error, stdout, stderr string) {
panic("git: stdout redirect mode invalid") panic("git: stdout redirect mode invalid")
} }
switch ctx.stderr { switch rctx.stderr {
case PIPE: case PIPE:
cmd.Stderr = &stderrBuf cmd.Stderr = &stderrBuf
case DontRedirect: case DontRedirect:
...@@ -77,9 +80,9 @@ func _git(argv []string, ctx RunWith) (err error, stdout, stderr string) { ...@@ -77,9 +80,9 @@ func _git(argv []string, ctx RunWith) (err error, stdout, stderr string) {
panic("git: stderr redirect mode invalid") panic("git: stderr redirect mode invalid")
} }
if ctx.env != nil { if rctx.env != nil {
env := []string{} env := []string{}
for k, v := range ctx.env { for k, v := range rctx.env {
env = append(env, k+"="+v) env = append(env, k+"="+v)
} }
cmd.Env = env cmd.Env = env
...@@ -89,7 +92,7 @@ func _git(argv []string, ctx RunWith) (err error, stdout, stderr string) { ...@@ -89,7 +92,7 @@ func _git(argv []string, ctx RunWith) (err error, stdout, stderr string) {
stdout = mem.String(stdoutBuf.Bytes()) stdout = mem.String(stdoutBuf.Bytes())
stderr = mem.String(stderrBuf.Bytes()) stderr = mem.String(stderrBuf.Bytes())
if !ctx.raw { if !rctx.raw {
// prettify stdout (e.g. so that 'sha1\n' becomes 'sha1' and can be used directly // prettify stdout (e.g. so that 'sha1\n' becomes 'sha1' and can be used directly
stdout = strings.TrimSpace(stdout) stdout = strings.TrimSpace(stdout)
stderr = strings.TrimSpace(stderr) stderr = strings.TrimSpace(stderr)
...@@ -138,9 +141,9 @@ func (e *GitErrContext) Error() string { ...@@ -138,9 +141,9 @@ func (e *GitErrContext) Error() string {
return msg return msg
} }
// argv -> []string, ctx (for passing argv + RunWith handy - see ggit() for details) // ctx, argv -> ctx, []string, rctx (for passing argv + RunWith handy - see ggit() for details)
func _gitargv(argv ...interface{}) (argvs []string, ctx RunWith) { func _gitargv(ctx context.Context, argv ...interface{}) (_ context.Context, argvs []string, rctx RunWith) {
ctx_seen := false rctx_seen := false
for _, arg := range argv { for _, arg := range argv {
switch arg := arg.(type) { switch arg := arg.(type) {
...@@ -149,47 +152,47 @@ func _gitargv(argv ...interface{}) (argvs []string, ctx RunWith) { ...@@ -149,47 +152,47 @@ func _gitargv(argv ...interface{}) (argvs []string, ctx RunWith) {
default: default:
argvs = append(argvs, fmt.Sprint(arg)) argvs = append(argvs, fmt.Sprint(arg))
case RunWith: case RunWith:
if ctx_seen { if rctx_seen {
panic("git: multiple RunWith contexts") panic("git: multiple RunWith contexts")
} }
ctx, ctx_seen = arg, true rctx, rctx_seen = arg, true
} }
} }
return argvs, ctx return ctx, argvs, rctx
} }
// run `git *argv` -> err, stdout, stderr // run `git *argv` -> err, stdout, stderr
// - arguments are automatically converted to strings // - arguments are automatically converted to strings
// - RunWith argument is passed as ctx // - RunWith argument is passed as rctx
// - error is returned only when git command could run and exits with error status // - error is returned only when git command could run and exits with error status
// - on other errors - exception is raised // - on other errors - exception is raised
// //
// NOTE err is concrete *GitError, not error // NOTE err is concrete *GitError, not error
func ggit(argv ...interface{}) (err *GitError, stdout, stderr string) { func ggit(ctx context.Context, argv ...interface{}) (err *GitError, stdout, stderr string) {
return ggit2(_gitargv(argv...)) return ggit2(_gitargv(ctx, argv...))
} }
func ggit2(argv []string, ctx RunWith) (err *GitError, stdout, stderr string) { func ggit2(ctx context.Context, argv []string, rctx RunWith) (err *GitError, stdout, stderr string) {
e, stdout, stderr := _git(argv, ctx) e, stdout, stderr := _git(ctx, argv, rctx)
eexec, _ := e.(*exec.ExitError) eexec, _ := e.(*exec.ExitError)
if e != nil && eexec == nil { if e != nil && eexec == nil {
exc.Raisef("git %s : %s", strings.Join(argv, " "), e) exc.Raisef("git %s : %s", strings.Join(argv, " "), e)
} }
if eexec != nil { if eexec != nil {
err = &GitError{GitErrContext{argv, ctx.stdin, stdout, stderr}, eexec} err = &GitError{GitErrContext{argv, rctx.stdin, stdout, stderr}, eexec}
} }
return err, stdout, stderr return err, stdout, stderr
} }
// run `git *argv` -> stdout // run `git *argv` -> stdout
// on error - raise exception // on error - raise exception
func xgit(argv ...interface{}) string { func xgit(ctx context.Context, argv ...interface{}) string {
return xgit2(_gitargv(argv...)) return xgit2(_gitargv(ctx, argv...))
} }
func xgit2(argv []string, ctx RunWith) string { func xgit2(ctx context.Context, argv []string, rctx RunWith) string {
gerr, stdout, _ := ggit2(argv, ctx) gerr, stdout, _ := ggit2(ctx, argv, rctx)
if gerr != nil { if gerr != nil {
exc.Raise(gerr) exc.Raise(gerr)
} }
...@@ -197,8 +200,8 @@ func xgit2(argv []string, ctx RunWith) string { ...@@ -197,8 +200,8 @@ func xgit2(argv []string, ctx RunWith) string {
} }
// like xgit(), but automatically parse stdout to Sha1 // like xgit(), but automatically parse stdout to Sha1
func xgitSha1(argv ...interface{}) Sha1 { func xgitSha1(ctx context.Context, argv ...interface{}) Sha1 {
return xgit2Sha1(_gitargv(argv...)) return xgit2Sha1(_gitargv(ctx, argv...))
} }
// error when git output is not valid sha1 // error when git output is not valid sha1
...@@ -212,14 +215,14 @@ func (e *GitSha1Error) Error() string { ...@@ -212,14 +215,14 @@ func (e *GitSha1Error) Error() string {
return msg return msg
} }
func xgit2Sha1(argv []string, ctx RunWith) Sha1 { func xgit2Sha1(ctx context.Context, argv []string, rctx RunWith) Sha1 {
gerr, stdout, stderr := ggit2(argv, ctx) gerr, stdout, stderr := ggit2(ctx, argv, rctx)
if gerr != nil { if gerr != nil {
exc.Raise(gerr) exc.Raise(gerr)
} }
sha1, err := Sha1Parse(stdout) sha1, err := Sha1Parse(stdout)
if err != nil { if err != nil {
exc.Raise(&GitSha1Error{GitErrContext{argv, ctx.stdin, stdout, stderr}}) exc.Raise(&GitSha1Error{GitErrContext{argv, rctx.stdin, stdout, stderr}})
} }
return sha1 return sha1
} }
// Copyright (C) 2015-2016 Nexedi SA and Contributors. // Copyright (C) 2015-2020 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com> // Kirill Smelkov <kirr@nexedi.com>
// //
// This program is free software: you can Use, Study, Modify and Redistribute // This program is free software: you can Use, Study, Modify and Redistribute
...@@ -21,6 +21,7 @@ package main ...@@ -21,6 +21,7 @@ package main
// Git-backup | Git object: Blob Tree Commit Tag // Git-backup | Git object: Blob Tree Commit Tag
import ( import (
"context"
"errors" "errors"
"fmt" "fmt"
"os" "os"
...@@ -163,9 +164,9 @@ func (e *InvalidLstreeEntry) Error() string { ...@@ -163,9 +164,9 @@ func (e *InvalidLstreeEntry) Error() string {
// create empty git tree -> tree sha1 // create empty git tree -> tree sha1
var tree_empty Sha1 var tree_empty Sha1
func mktree_empty() Sha1 { func mktree_empty(ctx context.Context) Sha1 {
if tree_empty.IsNull() { if tree_empty.IsNull() {
tree_empty = xgitSha1("mktree", RunWith{stdin: ""}) tree_empty = xgitSha1(ctx, "mktree", RunWith{stdin: ""})
} }
return tree_empty return tree_empty
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment