*: Add cancellation handling via contexts

Add ctx parameter to cmd_pull, cmd_restore and inner functions that they
call that can block and handle ctx cancel where we can. For now both
pull and restore are always run under background context, but in the
next patch we'll connect SIGINT+SIGTERM to cancel spawned work.

In general a service - even a command line utility - needs to handle
cancellation properly and itself to maintain consistency of external
state. See e.g.

	https://callistaenterprise.se/blogg/teknik/2019/10/05/go-worker-cancellation/

for example.

git-backup_test.go

@@ -20,6 +20,7 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"io/ioutil"
 	"os"
@@ -70,12 +71,14 @@ func xgittype(s string) git.ObjectType {
 
 // xnoref asserts that git reference ref does not exists.
 func xnoref(ref string) {
-	xgit("update-ref", "--stdin", RunWith{stdin: fmt.Sprintf("verify refs/%s %s\n", ref, Sha1{})})
+	xgit(context.Background(), "update-ref", "--stdin", RunWith{stdin: fmt.Sprintf("verify refs/%s %s\n", ref, Sha1{})})
 }
 
 
 // verify end-to-end pull-restore
 func TestPullRestore(t *testing.T) {
+	ctx := context.Background()
+
 	// if something raises -> don't let testing panic - report it as proper error with context.
 	here := my.FuncName()
 	defer exc.Catch(func(e *exc.Error) {
@@ -114,7 +117,7 @@ func TestPullRestore(t *testing.T) {
 	}
 
 	// init backup repository
-	xgit("init", "--bare", "backup.git")
+	xgit(ctx, "init", "--bare", "backup.git")
 	xchdir(t, "backup.git")
 	gb, err := git.OpenRepository(".")
 	if err != nil {
@@ -123,10 +126,10 @@ func TestPullRestore(t *testing.T) {
 
 	// pull from testdata
 	my0 := mydir + "/testdata/0"
-	cmd_pull(gb, []string{my0 + ":b0"}) // only empty repo in testdata/0
+	cmd_pull(ctx, gb, []string{my0 + ":b0"}) // only empty repo in testdata/0
 
 	my1 := mydir + "/testdata/1"
-	cmd_pull(gb, []string{my1 + ":b1"})
+	cmd_pull(ctx, gb, []string{my1 + ":b1"})
 
 	// verify tag/tree/blob encoding is 1) consistent and 2) always the same.
 	// we need it be always the same so different git-backup versions can
@@ -158,8 +161,8 @@ func TestPullRestore(t *testing.T) {
 		}
 
 		// encoding original object should give sha1_
-		obj_type := xgit("cat-file", "-t", nc.sha1)
-		sha1_ := obj_represent_as_commit(gb, nc.sha1, xgittype(obj_type))
+		obj_type := xgit(ctx, "cat-file", "-t", nc.sha1)
+		sha1_ := obj_represent_as_commit(ctx, gb, nc.sha1, xgittype(obj_type))
 		if sha1_ != nc.sha1_ {
 			t.Fatalf("encode %s -> %s ;  want %s", sha1, sha1_, nc.sha1_)
 		}
@@ -181,10 +184,10 @@ func TestPullRestore(t *testing.T) {
 		}
 
 		// prune all non-reachable objects (e.g. tags just pulled - they were encoded as commits)
-		xgit("prune")
+		xgit(ctx, "prune")
 
 		// verify backup repo is all ok
-		xgit("fsck")
+		xgit(ctx, "fsck")
 
 		// verify that just pulled tag objects are now gone after pruning -
 		// - they become not directly git-present. The only possibility to
@@ -193,7 +196,7 @@ func TestPullRestore(t *testing.T) {
 			if !nc.istag {
 				continue
 			}
-			gerr, _, _ := ggit("cat-file", "-p", nc.sha1)
+			gerr, _, _ := ggit(ctx, "cat-file", "-p", nc.sha1)
 			if gerr == nil {
 				t.Fatalf("tag %s still present in backup.git after git-prune", nc.sha1)
 			}
@@ -210,14 +213,14 @@ func TestPullRestore(t *testing.T) {
 	afterPull()
 
 	// pull again - it should be noop
-	h1 := xgitSha1("rev-parse", "HEAD")
-	cmd_pull(gb, []string{my1 + ":b1"})
+	h1 := xgitSha1(ctx, "rev-parse", "HEAD")
+	cmd_pull(ctx, gb, []string{my1 + ":b1"})
 	afterPull()
-	h2 := xgitSha1("rev-parse", "HEAD")
+	h2 := xgitSha1(ctx, "rev-parse", "HEAD")
 	if h1 == h2 {
 		t.Fatal("pull: second run did not ajusted HEAD")
 	}
-	δ12 := xgit("diff", h1, h2)
+	δ12 := xgit(ctx, "diff", h1, h2)
 	if δ12 != "" {
 		t.Fatalf("pull: second run was not noop: δ:\n%s", δ12)
 	}
@@ -225,10 +228,10 @@ func TestPullRestore(t *testing.T) {
 
 	// restore backup
 	work1 := workdir + "/1"
-	cmd_restore(gb, []string{"HEAD", "b1:" + work1})
+	cmd_restore(ctx, gb, []string{"HEAD", "b1:" + work1})
 
 	// verify files restored to the same as original
-	gerr, diff, _ := ggit("diff", "--no-index", "--raw", "--exit-code", my1, work1)
+	gerr, diff, _ := ggit(ctx, "diff", "--no-index", "--raw", "--exit-code", my1, work1)
 	// 0 - no diff, 1 - has diff, 2 - problem
 	if gerr != nil && gerr.Sys().(syscall.WaitStatus).ExitStatus() > 1 {
 		t.Fatal(gerr)
@@ -267,12 +270,12 @@ func TestPullRestore(t *testing.T) {
 
 		for _, repo := range R {
 			// fsck just in case
-			xgit("--git-dir="+repo.path, "fsck")
+			xgit(ctx, "--git-dir="+repo.path, "fsck")
 			// NOTE for-each-ref sorts output by refname
-			repo.reflist = xgit("--git-dir="+repo.path, "for-each-ref")
+			repo.reflist = xgit(ctx, "--git-dir="+repo.path, "for-each-ref")
 			// NOTE rev-list emits objects in reverse chronological order,
 			//      starting from refs roots which are also ordered by refname
-			repo.revlist = xgit("--git-dir="+repo.path, "rev-list", "--all", "--objects")
+			repo.revlist = xgit(ctx, "--git-dir="+repo.path, "rev-list", "--all", "--objects")
 		}
 
 		if R[0].reflist != R[1].reflist {
@@ -301,7 +304,7 @@ func TestPullRestore(t *testing.T) {
 			xnoref("backup.locked")
 		})
 
-		cmd_pull(gb, []string{my2 + ":b2"})
+		cmd_pull(ctx, gb, []string{my2 + ":b2"})
 		t.Fatal("pull corrupt.git: did not complain")
 	}()
 
@@ -341,7 +344,7 @@ func TestPullRestore(t *testing.T) {
 		err = os.Setenv("HOME", my3+"/incomplete-send-pack.git/"+kind)
 		exc.Raiseif(err)
 
-		cmd_pull(gb, []string{my3 + ":b3"})
+		cmd_pull(ctx, gb, []string{my3 + ":b3"})
 		t.Fatalf("pull incomplete-send-pack.git/%s: did not complain", kind)
 	}
 
@@ -358,7 +361,7 @@ func TestPullRestore(t *testing.T) {
 
 	// pulling incomplete-send-pack.git without pack-objects hook must succeed:
 	// without $HOME tweaks full and complete pack is sent.
-	cmd_pull(gb, []string{my3 + ":b3"})
+	cmd_pull(ctx, gb, []string{my3 + ":b3"})
 }
 
 func TestRepoRefSplit(t *testing.T) {

git.go

-// Copyright (C) 2015-2016  Nexedi SA and Contributors.
+// Copyright (C) 2015-2020  Nexedi SA and Contributors.
 //                          Kirill Smelkov <kirr@nexedi.com>
 //
 // This program is free software: you can Use, Study, Modify and Redistribute
@@ -22,6 +22,7 @@ package main
 
 import (
 	"bytes"
+	"context"
 	"fmt"
 	"os"
 	"os/exec"
@@ -48,18 +49,20 @@ type RunWith struct {
 }
 
 // run `git *argv` -> error, stdout, stderr
-func _git(argv []string, ctx RunWith) (err error, stdout, stderr string) {
+func _git(ctx context.Context, argv []string, rctx RunWith) (err error, stdout, stderr string) {
 	debugf("git %s", strings.Join(argv, " "))
 
-	cmd := exec.Command("git", argv...)
+	// XXX exec.CommandContext does `kill -9` on ctx cancel
+	// XXX -> rework to `kill -TERM` so that spawned process can finish cleanly?
+	cmd := exec.CommandContext(ctx, "git", argv...)
 	stdoutBuf := bytes.Buffer{}
 	stderrBuf := bytes.Buffer{}
 
-	if ctx.stdin != "" {
-		cmd.Stdin = strings.NewReader(ctx.stdin)
+	if rctx.stdin != "" {
+		cmd.Stdin = strings.NewReader(rctx.stdin)
 	}
 
-	switch ctx.stdout {
+	switch rctx.stdout {
 	case PIPE:
 		cmd.Stdout = &stdoutBuf
 	case DontRedirect:
@@ -68,7 +71,7 @@ func _git(argv []string, ctx RunWith) (err error, stdout, stderr string) {
 		panic("git: stdout redirect mode invalid")
 	}
 
-	switch ctx.stderr {
+	switch rctx.stderr {
 	case PIPE:
 		cmd.Stderr = &stderrBuf
 	case DontRedirect:
@@ -77,9 +80,9 @@ func _git(argv []string, ctx RunWith) (err error, stdout, stderr string) {
 		panic("git: stderr redirect mode invalid")
 	}
 
-	if ctx.env != nil {
+	if rctx.env != nil {
 		env := []string{}
-		for k, v := range ctx.env {
+		for k, v := range rctx.env {
 			env = append(env, k+"="+v)
 		}
 		cmd.Env = env
@@ -89,7 +92,7 @@ func _git(argv []string, ctx RunWith) (err error, stdout, stderr string) {
 	stdout = mem.String(stdoutBuf.Bytes())
 	stderr = mem.String(stderrBuf.Bytes())
 
-	if !ctx.raw {
+	if !rctx.raw {
 		// prettify stdout (e.g. so that 'sha1\n' becomes 'sha1' and can be used directly
 		stdout = strings.TrimSpace(stdout)
 		stderr = strings.TrimSpace(stderr)
@@ -138,9 +141,9 @@ func (e *GitErrContext) Error() string {
 	return msg
 }
 
-// argv -> []string, ctx    (for passing argv + RunWith handy - see ggit() for details)
-func _gitargv(argv ...interface{}) (argvs []string, ctx RunWith) {
-	ctx_seen := false
+// ctx, argv -> ctx, []string, rctx  (for passing argv + RunWith handy - see ggit() for details)
+func _gitargv(ctx context.Context, argv ...interface{}) (_ context.Context, argvs []string, rctx RunWith) {
+	rctx_seen := false
 
 	for _, arg := range argv {
 		switch arg := arg.(type) {
@@ -149,47 +152,47 @@ func _gitargv(argv ...interface{}) (argvs []string, ctx RunWith) {
 		default:
 			argvs = append(argvs, fmt.Sprint(arg))
 		case RunWith:
-			if ctx_seen {
+			if rctx_seen {
 				panic("git: multiple RunWith contexts")
 			}
-			ctx, ctx_seen = arg, true
+			rctx, rctx_seen = arg, true
 		}
 	}
 
-	return argvs, ctx
+	return ctx, argvs, rctx
 }
 
 // run `git *argv` -> err, stdout, stderr
 // - arguments are automatically converted to strings
-// - RunWith argument is passed as ctx
+// - RunWith argument is passed as rctx
 // - error is returned only when git command could run and exits with error status
 // - on other errors - exception is raised
 //
 // NOTE err is concrete *GitError, not error
-func ggit(argv ...interface{}) (err *GitError, stdout, stderr string) {
-	return ggit2(_gitargv(argv...))
+func ggit(ctx context.Context, argv ...interface{}) (err *GitError, stdout, stderr string) {
+	return ggit2(_gitargv(ctx, argv...))
 }
 
-func ggit2(argv []string, ctx RunWith) (err *GitError, stdout, stderr string) {
-	e, stdout, stderr := _git(argv, ctx)
+func ggit2(ctx context.Context, argv []string, rctx RunWith) (err *GitError, stdout, stderr string) {
+	e, stdout, stderr := _git(ctx, argv, rctx)
 	eexec, _ := e.(*exec.ExitError)
 	if e != nil && eexec == nil {
 		exc.Raisef("git %s : %s", strings.Join(argv, " "), e)
 	}
 	if eexec != nil {
-		err = &GitError{GitErrContext{argv, ctx.stdin, stdout, stderr}, eexec}
+		err = &GitError{GitErrContext{argv, rctx.stdin, stdout, stderr}, eexec}
 	}
 	return err, stdout, stderr
 }
 
 // run `git *argv` -> stdout
 // on error - raise exception
-func xgit(argv ...interface{}) string {
-	return xgit2(_gitargv(argv...))
+func xgit(ctx context.Context, argv ...interface{}) string {
+	return xgit2(_gitargv(ctx, argv...))
 }
 
-func xgit2(argv []string, ctx RunWith) string {
-	gerr, stdout, _ := ggit2(argv, ctx)
+func xgit2(ctx context.Context, argv []string, rctx RunWith) string {
+	gerr, stdout, _ := ggit2(ctx, argv, rctx)
 	if gerr != nil {
 		exc.Raise(gerr)
 	}
@@ -197,8 +200,8 @@ func xgit2(argv []string, ctx RunWith) string {
 }
 
 // like xgit(), but automatically parse stdout to Sha1
-func xgitSha1(argv ...interface{}) Sha1 {
-	return xgit2Sha1(_gitargv(argv...))
+func xgitSha1(ctx context.Context, argv ...interface{}) Sha1 {
+	return xgit2Sha1(_gitargv(ctx, argv...))
 }
 
 // error when git output is not valid sha1
@@ -212,14 +215,14 @@ func (e *GitSha1Error) Error() string {
 	return msg
 }
 
-func xgit2Sha1(argv []string, ctx RunWith) Sha1 {
-	gerr, stdout, stderr := ggit2(argv, ctx)
+func xgit2Sha1(ctx context.Context, argv []string, rctx RunWith) Sha1 {
+	gerr, stdout, stderr := ggit2(ctx, argv, rctx)
 	if gerr != nil {
 		exc.Raise(gerr)
 	}
 	sha1, err := Sha1Parse(stdout)
 	if err != nil {
-		exc.Raise(&GitSha1Error{GitErrContext{argv, ctx.stdin, stdout, stderr}})
+		exc.Raise(&GitSha1Error{GitErrContext{argv, rctx.stdin, stdout, stderr}})
 	}
 	return sha1
 }

gitobjects.go

-// Copyright (C) 2015-2016  Nexedi SA and Contributors.
+// Copyright (C) 2015-2020  Nexedi SA and Contributors.
 //                          Kirill Smelkov <kirr@nexedi.com>
 //
 // This program is free software: you can Use, Study, Modify and Redistribute
@@ -21,6 +21,7 @@ package main
 // Git-backup | Git object: Blob Tree Commit Tag
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"os"
@@ -163,9 +164,9 @@ func (e *InvalidLstreeEntry) Error() string {
 
 // create empty git tree -> tree sha1
 var tree_empty Sha1
-func mktree_empty() Sha1 {
+func mktree_empty(ctx context.Context) Sha1 {
 	if tree_empty.IsNull() {
-		tree_empty = xgitSha1("mktree", RunWith{stdin: ""})
+		tree_empty = xgitSha1(ctx, "mktree", RunWith{stdin: ""})
 	}
 	return tree_empty
 }