Skip to content

G
git-backup
Commit f3f694b9 authored by Kirill Smelkov's avatar Kirill Smelkov
Browse files
Options

~gofmt

parent cc6ac54f
Hide whitespace changes
Inline Side-by-side
Showing with 1843 additions and 1844 deletions
git-backup.go
View file @ f3f694b9
...@@ -67,28 +67,28 @@ NOTE the idea of pulling all refs together is similar to git-namespaces ...@@ -67,28 +67,28 @@ NOTE the idea of pulling all refs together is similar to git-namespaces
package main package main
import ( import (
"flag" "flag"
"fmt" "fmt"
"io/ioutil" "io/ioutil"
"os" "os"
pathpkg "path" pathpkg "path"
"path/filepath" "path/filepath"
"runtime" "runtime"
"runtime/debug" "runtime/debug"
"sort" "sort"
"strings" "strings"
"sync" "sync"
"syscall" "syscall"
"time" "time"
"lab.nexedi.com/kirr/go123/exc" "lab.nexedi.com/kirr/go123/exc"
"lab.nexedi.com/kirr/go123/mem" "lab.nexedi.com/kirr/go123/mem"
"lab.nexedi.com/kirr/go123/my" "lab.nexedi.com/kirr/go123/my"
"lab.nexedi.com/kirr/go123/xerr" "lab.nexedi.com/kirr/go123/xerr"
"lab.nexedi.com/kirr/go123/xflag" "lab.nexedi.com/kirr/go123/xflag"
"lab.nexedi.com/kirr/go123/xstrings" "lab.nexedi.com/kirr/go123/xstrings"
git "github.com/libgit2/git2go" git "github.com/libgit2/git2go"
) )
// verbose output // verbose output
...@@ -99,26 +99,26 @@ import ( ...@@ -99,26 +99,26 @@ import (
var verbose = 1 var verbose = 1
func infof(format string, a ...interface{}) { func infof(format string, a ...interface{}) {
if verbose > 0 { if verbose > 0 {
fmt.Printf(format, a...) fmt.Printf(format, a...)
fmt.Println() fmt.Println()
} }
} }
// what to pass to git subprocess to stdout/stderr // what to pass to git subprocess to stdout/stderr
// DontRedirect - no-redirection, PIPE - output to us // DontRedirect - no-redirection, PIPE - output to us
func gitprogress() StdioRedirect { func gitprogress() StdioRedirect {
if verbose > 1 { if verbose > 1 {
return DontRedirect return DontRedirect
} }
return PIPE return PIPE
} }
func debugf(format string, a ...interface{}) { func debugf(format string, a ...interface{}) {
if verbose > 2 { if verbose > 2 {
fmt.Printf(format, a...) fmt.Printf(format, a...)
fmt.Println() fmt.Println()
} }
} }
// how many max jobs to spawn // how many max jobs to spawn
...@@ -128,48 +128,48 @@ var njobs = runtime.NumCPU() ...@@ -128,48 +128,48 @@ var njobs = runtime.NumCPU()
// file -> blob_sha1, mode // file -> blob_sha1, mode
func file_to_blob(g *git.Repository, path string) (Sha1, uint32) { func file_to_blob(g *git.Repository, path string) (Sha1, uint32) {
var blob_content []byte var blob_content []byte
// because we want to pass mode to outside world (to e.g. `git update-index`) // because we want to pass mode to outside world (to e.g. `git update-index`)
// we need to get native OS mode, not translated one as os.Lstat() would give us. // we need to get native OS mode, not translated one as os.Lstat() would give us.
var st syscall.Stat_t var st syscall.Stat_t
err := syscall.Lstat(path, &st) err := syscall.Lstat(path, &st)
if err != nil { if err != nil {
exc.Raise(&os.PathError{"lstat", path, err}) exc.Raise(&os.PathError{"lstat", path, err})
} }
if st.Mode&syscall.S_IFMT == syscall.S_IFLNK { if st.Mode&syscall.S_IFMT == syscall.S_IFLNK {
__, err := os.Readlink(path) __, err := os.Readlink(path)
blob_content = mem.Bytes(__) blob_content = mem.Bytes(__)
exc.Raiseif(err) exc.Raiseif(err)
} else { } else {
blob_content, err = ioutil.ReadFile(path) blob_content, err = ioutil.ReadFile(path)
exc.Raiseif(err) exc.Raiseif(err)
} }
blob_sha1, err := WriteObject(g, blob_content, git.ObjectBlob) blob_sha1, err := WriteObject(g, blob_content, git.ObjectBlob)
exc.Raiseif(err) exc.Raiseif(err)
return blob_sha1, st.Mode return blob_sha1, st.Mode
} }
// blob_sha1, mode -> file // blob_sha1, mode -> file
func blob_to_file(g *git.Repository, blob_sha1 Sha1, mode uint32, path string) { func blob_to_file(g *git.Repository, blob_sha1 Sha1, mode uint32, path string) {
blob, err := ReadObject(g, blob_sha1, git.ObjectBlob) blob, err := ReadObject(g, blob_sha1, git.ObjectBlob)
exc.Raiseif(err) exc.Raiseif(err)
blob_content := blob.Data() blob_content := blob.Data()
err = os.MkdirAll(pathpkg.Dir(path), 0777) err = os.MkdirAll(pathpkg.Dir(path), 0777)
exc.Raiseif(err) exc.Raiseif(err)
if mode&syscall.S_IFMT == syscall.S_IFLNK { if mode&syscall.S_IFMT == syscall.S_IFLNK {
err = os.Symlink(mem.String(blob_content), path) err = os.Symlink(mem.String(blob_content), path)
exc.Raiseif(err) exc.Raiseif(err)
} else { } else {
// NOTE mode is native - we cannot use ioutil.WriteFile() directly // NOTE mode is native - we cannot use ioutil.WriteFile() directly
err = writefile(path, blob_content, mode) err = writefile(path, blob_content, mode)
exc.Raiseif(err) exc.Raiseif(err)
} }
} }
// -------- tags representation -------- // -------- tags representation --------
...@@ -185,74 +185,74 @@ func blob_to_file(g *git.Repository, blob_sha1 Sha1, mode uint32, path string) { ...@@ -185,74 +185,74 @@ func blob_to_file(g *git.Repository, blob_sha1 Sha1, mode uint32, path string) {
// object and tagged object is kept there in repo thanks to it being reachable // object and tagged object is kept there in repo thanks to it being reachable
// through created commit. // through created commit.
func obj_represent_as_commit(g *git.Repository, sha1 Sha1, obj_type git.ObjectType) Sha1 { func obj_represent_as_commit(g *git.Repository, sha1 Sha1, obj_type git.ObjectType) Sha1 {
switch obj_type { switch obj_type {
case git.ObjectTag, git.ObjectTree, git.ObjectBlob: case git.ObjectTag, git.ObjectTree, git.ObjectBlob:
// ok // ok
default: default:
exc.Raisef("%s (%s): cannot encode as commit", sha1, obj_type) exc.Raisef("%s (%s): cannot encode as commit", sha1, obj_type)
} }
// first line in commit msg = object type // first line in commit msg = object type
obj_encoded := gittypestr(obj_type) + "\n" obj_encoded := gittypestr(obj_type) + "\n"
var tagged_type git.ObjectType var tagged_type git.ObjectType
var tagged_sha1 Sha1 var tagged_sha1 Sha1
// below the code layout is mainly for tag type, and we hook tree and blob // below the code layout is mainly for tag type, and we hook tree and blob
// types handling into that layout // types handling into that layout
if obj_type == git.ObjectTag { if obj_type == git.ObjectTag {
tag, tag_obj := xload_tag(g, sha1) tag, tag_obj := xload_tag(g, sha1)
tagged_type = tag.tagged_type tagged_type = tag.tagged_type
tagged_sha1 = tag.tagged_sha1 tagged_sha1 = tag.tagged_sha1
obj_encoded += mem.String(tag_obj.Data()) obj_encoded += mem.String(tag_obj.Data())
} else { } else {
// for tree/blob we only care that object stays reachable // for tree/blob we only care that object stays reachable
tagged_type = obj_type tagged_type = obj_type
tagged_sha1 = sha1 tagged_sha1 = sha1
} }
// all commits we do here - we do with fixed name/date, so transformation // all commits we do here - we do with fixed name/date, so transformation
// tag->commit is stable wrt git environment and time change // tag->commit is stable wrt git environment and time change
fixed := AuthorInfo{Name: "Git backup", Email: "git@backup.org", When: time.Unix(0, 0).UTC()} fixed := AuthorInfo{Name: "Git backup", Email: "git@backup.org", When: time.Unix(0, 0).UTC()}
zcommit_tree := func(tree Sha1, parents []Sha1, msg string) Sha1 { zcommit_tree := func(tree Sha1, parents []Sha1, msg string) Sha1 {
return xcommit_tree2(g, tree, parents, msg, fixed, fixed) return xcommit_tree2(g, tree, parents, msg, fixed, fixed)
} }
// Tag ~> Commit* // Tag ~> Commit*
// | .msg: Tag // | .msg: Tag
// v .tree -> ø // v .tree -> ø
// Commit .parent -> Commit // Commit .parent -> Commit
if tagged_type == git.ObjectCommit { if tagged_type == git.ObjectCommit {
return zcommit_tree(mktree_empty(), []Sha1{tagged_sha1}, obj_encoded) return zcommit_tree(mktree_empty(), []Sha1{tagged_sha1}, obj_encoded)
} }
// Tag ~> Commit* // Tag ~> Commit*
// | .msg: Tag // | .msg: Tag
// v .tree -> Tree // v .tree -> Tree
// Tree .parent -> ø // Tree .parent -> ø
if tagged_type == git.ObjectTree { if tagged_type == git.ObjectTree {
return zcommit_tree(tagged_sha1, []Sha1{}, obj_encoded) return zcommit_tree(tagged_sha1, []Sha1{}, obj_encoded)
} }
// Tag ~> Commit* // Tag ~> Commit*
// | .msg: Tag // | .msg: Tag
// v .tree -> Tree* "tagged" -> Blob // v .tree -> Tree* "tagged" -> Blob
// Blob .parent -> ø // Blob .parent -> ø
if tagged_type == git.ObjectBlob { if tagged_type == git.ObjectBlob {
tree_for_blob := xgitSha1("mktree", RunWith{stdin: fmt.Sprintf("100644 blob %s\ttagged\n", tagged_sha1)}) tree_for_blob := xgitSha1("mktree", RunWith{stdin: fmt.Sprintf("100644 blob %s\ttagged\n", tagged_sha1)})
return zcommit_tree(tree_for_blob, []Sha1{}, obj_encoded) return zcommit_tree(tree_for_blob, []Sha1{}, obj_encoded)
} }
// Tag₂ ~> Commit₂* // Tag₂ ~> Commit₂*
// | .msg: Tag₂ // | .msg: Tag₂
// v .tree -> ø // v .tree -> ø
// Tag₁ .parent -> Commit₁* // Tag₁ .parent -> Commit₁*
if tagged_type == git.ObjectTag { if tagged_type == git.ObjectTag {
commit1 := obj_represent_as_commit(g, tagged_sha1, tagged_type) commit1 := obj_represent_as_commit(g, tagged_sha1, tagged_type)
return zcommit_tree(mktree_empty(), []Sha1{commit1}, obj_encoded) return zcommit_tree(mktree_empty(), []Sha1{commit1}, obj_encoded)
} }
exc.Raisef("%s (%q): unknown tagged type", sha1, tagged_type) exc.Raisef("%s (%q): unknown tagged type", sha1, tagged_type)
panic(0) panic(0)
} }
// recreate tag/tree/blob from specially crafted commit // recreate tag/tree/blob from specially crafted commit
...@@ -261,68 +261,68 @@ func obj_represent_as_commit(g *git.Repository, sha1 Sha1, obj_type git.ObjectTy ...@@ -261,68 +261,68 @@ func obj_represent_as_commit(g *git.Repository, sha1 Sha1, obj_type git.ObjectTy
// - tag: recreated object sha1 // - tag: recreated object sha1
// - tree/blob: null sha1 // - tree/blob: null sha1
func obj_recreate_from_commit(g *git.Repository, commit_sha1 Sha1) Sha1 { func obj_recreate_from_commit(g *git.Repository, commit_sha1 Sha1) Sha1 {
xraise := func(info interface{}) { exc.Raise(&RecreateObjError{commit_sha1, info}) } xraise := func(info interface{}) { exc.Raise(&RecreateObjError{commit_sha1, info}) }
xraisef := func(f string, a ...interface{}) { xraise(fmt.Sprintf(f, a...)) } xraisef := func(f string, a ...interface{}) { xraise(fmt.Sprintf(f, a...)) }
commit, err := g.LookupCommit(commit_sha1.AsOid()) commit, err := g.LookupCommit(commit_sha1.AsOid())
if err != nil { if err != nil {
xraise(err) xraise(err)
} }
if commit.ParentCount() > 1 { if commit.ParentCount() > 1 {
xraise(">1 parents") xraise(">1 parents")
} }
obj_type, obj_raw, err := xstrings.HeadTail(commit.Message(), "\n") obj_type, obj_raw, err := xstrings.HeadTail(commit.Message(), "\n")
if err != nil { if err != nil {
xraise("invalid encoded format") xraise("invalid encoded format")
} }
switch obj_type { switch obj_type {
case "tag", "tree", "blob": case "tag", "tree", "blob":
// ok // ok
default: default:
xraisef("unexpected encoded object type %q", obj_type) xraisef("unexpected encoded object type %q", obj_type)
} }
// for tree/blob we do not need to do anything - that objects were reachable // for tree/blob we do not need to do anything - that objects were reachable
// from commit and are present in git db. // from commit and are present in git db.
if obj_type == "tree" || obj_type == "blob" { if obj_type == "tree" || obj_type == "blob" {
return Sha1{} return Sha1{}
} }
// re-create tag object // re-create tag object
tag_sha1, err := WriteObject(g, mem.Bytes(obj_raw), git.ObjectTag) tag_sha1, err := WriteObject(g, mem.Bytes(obj_raw), git.ObjectTag)
exc.Raiseif(err) exc.Raiseif(err)
// the original tagged object should be already in repository, because we // the original tagged object should be already in repository, because we
// always attach it to encoding commit one way or another, // always attach it to encoding commit one way or another,
// except we need to recurse, if it was Tag₂->Tag₁ // except we need to recurse, if it was Tag₂->Tag₁
tag, err := tag_parse(obj_raw) tag, err := tag_parse(obj_raw)
if err != nil { if err != nil {
xraisef("encoded tag: %s", err) xraisef("encoded tag: %s", err)
} }
if tag.tagged_type == git.ObjectTag { if tag.tagged_type == git.ObjectTag {
if commit.ParentCount() == 0 { if commit.ParentCount() == 0 {
xraise("encoded tag corrupt (tagged is tag but []parent is empty)") xraise("encoded tag corrupt (tagged is tag but []parent is empty)")
} }
obj_recreate_from_commit(g, Sha1FromOid(commit.ParentId(0))) obj_recreate_from_commit(g, Sha1FromOid(commit.ParentId(0)))
} }
return tag_sha1 return tag_sha1
} }
type RecreateObjError struct { type RecreateObjError struct {
commit_sha1 Sha1 commit_sha1 Sha1
info interface{} info interface{}
} }
func (e *RecreateObjError) Error() string { func (e *RecreateObjError) Error() string {
return fmt.Sprintf("commit %s: %s", e.commit_sha1, e.info) return fmt.Sprintf("commit %s: %s", e.commit_sha1, e.info)
} }
// -------- git-backup pull -------- // -------- git-backup pull --------
func cmd_pull_usage() { func cmd_pull_usage() {
fmt.Fprint(os.Stderr, fmt.Fprint(os.Stderr,
`git-backup pull <dir1>:<prefix1> <dir2>:<prefix2> ... `git-backup pull <dir1>:<prefix1> <dir2>:<prefix2> ...
Pull bare Git repositories & just files from dir1 into backup prefix1, Pull bare Git repositories & just files from dir1 into backup prefix1,
...@@ -331,333 +331,332 @@ from dir2 into backup prefix2, etc... ...@@ -331,333 +331,332 @@ from dir2 into backup prefix2, etc...
} }
type PullSpec struct { type PullSpec struct {
dir, prefix string dir, prefix string
} }
func cmd_pull(gb *git.Repository, argv []string) { func cmd_pull(gb *git.Repository, argv []string) {
flags := flag.FlagSet{Usage: cmd_pull_usage} flags := flag.FlagSet{Usage: cmd_pull_usage}
flags.Init("", flag.ExitOnError) flags.Init("", flag.ExitOnError)
flags.Parse(argv) flags.Parse(argv)
argv = flags.Args() argv = flags.Args()
if len(argv) < 1 { if len(argv) < 1 {
cmd_pull_usage() cmd_pull_usage()
os.Exit(1) os.Exit(1)
} }
pullspecv := []PullSpec{} pullspecv := []PullSpec{}
for _, arg := range argv { for _, arg := range argv {
dir, prefix, err := xstrings.Split2(arg, ":") dir, prefix, err := xstrings.Split2(arg, ":")
if err != nil { if err != nil {
fmt.Fprintf(os.Stderr, "E: invalid pullspec %q\n", arg) fmt.Fprintf(os.Stderr, "E: invalid pullspec %q\n", arg)
cmd_pull_usage() cmd_pull_usage()
os.Exit(1) os.Exit(1)
} }
pullspecv = append(pullspecv, PullSpec{dir, prefix}) pullspecv = append(pullspecv, PullSpec{dir, prefix})
} }
cmd_pull_(gb, pullspecv) cmd_pull_(gb, pullspecv)
} }
// Ref is info about a reference pointing to sha1. // Ref is info about a reference pointing to sha1.
type Ref struct { type Ref struct {
name string // reference name without "refs/" prefix name string // reference name without "refs/" prefix
sha1 Sha1 sha1 Sha1
} }
func cmd_pull_(gb *git.Repository, pullspecv []PullSpec) { func cmd_pull_(gb *git.Repository, pullspecv []PullSpec) {
// while pulling, we'll keep refs from all pulled repositories under temp // while pulling, we'll keep refs from all pulled repositories under temp
// unique work refs namespace. // unique work refs namespace.
backup_time := time.Now().Format("20060102-1504") // %Y%m%d-%H%M backup_time := time.Now().Format("20060102-1504") // %Y%m%d-%H%M
backup_refs_work := fmt.Sprintf("refs/backup/%s/", backup_time) // refs/backup/20150820-2109/ backup_refs_work := fmt.Sprintf("refs/backup/%s/", backup_time) // refs/backup/20150820-2109/
backup_lock := "refs/backup.locked" backup_lock := "refs/backup.locked"
// make sure another `git-backup pull` is not running // make sure another `git-backup pull` is not running
xgit("update-ref", backup_lock, mktree_empty(), Sha1{}) xgit("update-ref", backup_lock, mktree_empty(), Sha1{})
// make sure there is root commit // make sure there is root commit
var HEAD Sha1 var HEAD Sha1
var err error var err error
gerr, __, _ := ggit("rev-parse", "--verify", "HEAD") gerr, __, _ := ggit("rev-parse", "--verify", "HEAD")
if gerr != nil { if gerr != nil {
infof("# creating root commit") infof("# creating root commit")
// NOTE `git commit` does not work in bare repo - do commit by hand // NOTE `git commit` does not work in bare repo - do commit by hand
HEAD = xcommit_tree(gb, mktree_empty(), []Sha1{}, "Initialize git-backup repository") HEAD = xcommit_tree(gb, mktree_empty(), []Sha1{}, "Initialize git-backup repository")
xgit("update-ref", "-m", "git-backup pull init", "HEAD", HEAD) xgit("update-ref", "-m", "git-backup pull init", "HEAD", HEAD)
} else { } else {
HEAD, err = Sha1Parse(__) HEAD, err = Sha1Parse(__)
exc.Raiseif(err) exc.Raiseif(err)
} }
// build index of "already-have" objects: all commits + tag/tree/blob that // build index of "already-have" objects: all commits + tag/tree/blob that
// were at heads of already pulled repositories. // were at heads of already pulled repositories.
// //
// Build it once and use below to check ourselves whether a head from a pulled // Build it once and use below to check ourselves whether a head from a pulled
// repository needs to be actually fetched. If we don't, `git fetch-pack` // repository needs to be actually fetched. If we don't, `git fetch-pack`
// will do similar to "all commits" linear scan for every pulled repository, // will do similar to "all commits" linear scan for every pulled repository,
// which are many out there. // which are many out there.
alreadyHave := Sha1Set{} alreadyHave := Sha1Set{}
infof("# building \"already-have\" index") infof("# building \"already-have\" index")
// already have: all commits // already have: all commits
// //
// As of lab.nexedi.com/20180612 there are ~ 1.7·10⁷ objects total in backup. // As of lab.nexedi.com/20180612 there are ~ 1.7·10⁷ objects total in backup.
// Of those there are ~ 1.9·10⁶ commit objects, i.e. ~10% of total. // Of those there are ~ 1.9·10⁶ commit objects, i.e. ~10% of total.
// Since 1 sha1 is 2·10¹ bytes, the space needed for keeping sha1 of all // Since 1 sha1 is 2·10¹ bytes, the space needed for keeping sha1 of all
// commits is ~ 4·10⁷B = ~40MB. It is thus ok to keep this index in RAM for now. // commits is ~ 4·10⁷B = ~40MB. It is thus ok to keep this index in RAM for now.
for _, __ := range xstrings.SplitLines(xgit("rev-list", HEAD), "\n") { for _, __ := range xstrings.SplitLines(xgit("rev-list", HEAD), "\n") {
sha1, err := Sha1Parse(__) sha1, err := Sha1Parse(__)
exc.Raiseif(err) exc.Raiseif(err)
alreadyHave.Add(sha1) alreadyHave.Add(sha1)
} }
// already have: tag/tree/blob that were at heads of already pulled repositories // already have: tag/tree/blob that were at heads of already pulled repositories
// //
// As of lab.nexedi.com/20180612 there are ~ 8.4·10⁴ refs in total. // As of lab.nexedi.com/20180612 there are ~ 8.4·10⁴ refs in total.
// Of those encoded tag/tree/blob are ~ 3.2·10⁴, i.e. ~40% of total. // Of those encoded tag/tree/blob are ~ 3.2·10⁴, i.e. ~40% of total.
// The number of tag/tree/blob objects in alreadyHave is thus negligible // The number of tag/tree/blob objects in alreadyHave is thus negligible
// compared to the number of "all commits". // compared to the number of "all commits".
hcommit, err := gb.LookupCommit(HEAD.AsOid()) hcommit, err := gb.LookupCommit(HEAD.AsOid())
exc.Raiseif(err) exc.Raiseif(err)
htree, err := hcommit.Tree() htree, err := hcommit.Tree()
exc.Raiseif(err) exc.Raiseif(err)
if htree.EntryByName("backup.refs") != nil { if htree.EntryByName("backup.refs") != nil {
repotab, err := loadBackupRefs(fmt.Sprintf("%s:backup.refs", HEAD)) repotab, err := loadBackupRefs(fmt.Sprintf("%s:backup.refs", HEAD))
exc.Raiseif(err) exc.Raiseif(err)
for _, repo := range repotab { for _, repo := range repotab {
for _, xref := range repo.refs { for _, xref := range repo.refs {
if xref.sha1 != xref.sha1_ && !alreadyHave.Contains(xref.sha1) { if xref.sha1 != xref.sha1_ && !alreadyHave.Contains(xref.sha1) {
// make sure encoded tag/tree/blob objects represented as // make sure encoded tag/tree/blob objects represented as
// commits are present. We do so, because we promise to // commits are present. We do so, because we promise to
// fetch that all objects in alreadyHave are present. // fetch that all objects in alreadyHave are present.
obj_recreate_from_commit(gb, xref.sha1_) obj_recreate_from_commit(gb, xref.sha1_)
alreadyHave.Add(xref.sha1) alreadyHave.Add(xref.sha1)
} }
} }
} }
} }
// walk over specified dirs, pulling objects from git and blobbing non-git-object files // walk over specified dirs, pulling objects from git and blobbing non-git-object files
blobbedv := []string{} // info about file pulled to blob, and not yet added to index blobbedv := []string{} // info about file pulled to blob, and not yet added to index
for _, __ := range pullspecv { for _, __ := range pullspecv {
dir, prefix := __.dir, __.prefix dir, prefix := __.dir, __.prefix
// make sure index is empty for prefix (so that we start from clean // make sure index is empty for prefix (so that we start from clean
// prefix namespace and this way won't leave stale removed things) // prefix namespace and this way won't leave stale removed things)
xgit("rm", "--cached", "-r", "--ignore-unmatch", "--", prefix) xgit("rm", "--cached", "-r", "--ignore-unmatch", "--", prefix)
here := my.FuncName() here := my.FuncName()
err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) (errout error) { err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) (errout error) {
if err != nil { if err != nil {
if os.IsNotExist(err) { if os.IsNotExist(err) {
// a file or directory was removed in parallel to us scanning the tree. // a file or directory was removed in parallel to us scanning the tree.
infof("Warning: Skipping %s: %s", path, err) infof("Warning: Skipping %s: %s", path, err)
return nil return nil
} }
// any other error -> stop // any other error -> stop
return err return err
} }
// propagate exceptions properly via filepath.Walk as errors with calling context // propagate exceptions properly via filepath.Walk as errors with calling context
// (filepath is not our code) // (filepath is not our code)
defer exc.Catch(func(e *exc.Error) { defer exc.Catch(func(e *exc.Error) {
errout = exc.Addcallingcontext(here, e) errout = exc.Addcallingcontext(here, e)
}) })
// files -> blobs + queue info for adding blobs to index // files -> blobs + queue info for adding blobs to index
if !info.IsDir() { if !info.IsDir() {
infof("# file %s\t<- %s", prefix, path) infof("# file %s\t<- %s", prefix, path)
blob, mode := file_to_blob(gb, path) blob, mode := file_to_blob(gb, path)
blobbedv = append(blobbedv, blobbedv = append(blobbedv,
fmt.Sprintf("%o %s\t%s", mode, blob, reprefix(dir, prefix, path))) fmt.Sprintf("%o %s\t%s", mode, blob, reprefix(dir, prefix, path)))
return nil return nil
} }
// directories -> look for *.git and handle git object specially. // directories -> look for *.git and handle git object specially.
// do not recurse into *.git/objects/ - we'll save them specially // do not recurse into *.git/objects/ - we'll save them specially
if strings.HasSuffix(path, ".git/objects") { if strings.HasSuffix(path, ".git/objects") {
return filepath.SkipDir return filepath.SkipDir
} }
// else we recurse, but handle *.git specially - via fetching objects from it // else we recurse, but handle *.git specially - via fetching objects from it
if !strings.HasSuffix(path, ".git") { if !strings.HasSuffix(path, ".git") {
return nil return nil
} }
// git repo - let's pull all refs from it to our backup refs namespace // git repo - let's pull all refs from it to our backup refs namespace
infof("# git %s\t<- %s", prefix, path) infof("# git %s\t<- %s", prefix, path)
refv, _, err := fetch(path, alreadyHave) refv, _, err := fetch(path, alreadyHave)
exc.Raiseif(err) exc.Raiseif(err)
// TODO don't store to git references all references from fetched repository: // TODO don't store to git references all references from fetched repository:
// //
// We need to store to git references only references that were actually // We need to store to git references only references that were actually
// fetched - so that next fetch, e.g. from a fork that also has new data // fetched - so that next fetch, e.g. from a fork that also has new data
// as its upstream, won't have to transfer what we just have fetched // as its upstream, won't have to transfer what we just have fetched
// from upstream. // from upstream.
// //
// For this purpose we can also save references by naming them as their // For this purpose we can also save references by naming them as their
// sha1, not actual name, which will automatically deduplicate them in // sha1, not actual name, which will automatically deduplicate them in
// between several repositories, especially when/if pull will be made to // between several repositories, especially when/if pull will be made to
// work in parallel. // work in parallel.
// //
// Such changed-only deduplicated references should be O(δ) - usually only // Such changed-only deduplicated references should be O(δ) - usually only
// a few, and this way we will also automatically avoid O(n^2) behaviour // a few, and this way we will also automatically avoid O(n^2) behaviour
// of every git fetch scanning all local references at its startup. // of every git fetch scanning all local references at its startup.
// //
// For backup.refs, we can generate it directly from refv of all fetched // For backup.refs, we can generate it directly from refv of all fetched
// repositories saved in RAM. // repositories saved in RAM.
reporefprefix := backup_refs_work + reporefprefix := backup_refs_work +
// NOTE repo name is escaped as it can contain e.g. spaces, and refs must not // NOTE repo name is escaped as it can contain e.g. spaces, and refs must not
path_refescape(reprefix(dir, prefix, path)) path_refescape(reprefix(dir, prefix, path))
for _, ref := range refv { for _, ref := range refv {
err = mkref(gb, reporefprefix + "/" + ref.name, ref.sha1) err = mkref(gb, reporefprefix+"/"+ref.name, ref.sha1)
exc.Raiseif(err) exc.Raiseif(err)
} }
// XXX do we want to do full fsck of source git repo on pull as well ?
// XXX do we want to do full fsck of source git repo on pull as well ?
return nil
return nil })
})
// re-raise / raise error after Walk
// re-raise / raise error after Walk if err != nil {
if err != nil { e := exc.Aserror(err)
e := exc.Aserror(err) e = exc.Addcontext(e, "pulling from "+dir)
e = exc.Addcontext(e, "pulling from "+dir) exc.Raise(e)
exc.Raise(e) }
} }
}
// add to index files we converted to blobs
// add to index files we converted to blobs xgit("update-index", "--add", "--index-info", RunWith{stdin: strings.Join(blobbedv, "\n")})
xgit("update-index", "--add", "--index-info", RunWith{stdin: strings.Join(blobbedv, "\n")})
// all refs from all found git repositories populated.
// all refs from all found git repositories populated. // now prepare manifest with ref -> sha1 and do a synthetic commit merging all that sha1
// now prepare manifest with ref -> sha1 and do a synthetic commit merging all that sha1 // (so they become all reachable from HEAD -> survive repack and be transferable on git pull)
// (so they become all reachable from HEAD -> survive repack and be transferable on git pull) //
// // NOTE we handle tag/tree/blob objects specially - because these objects cannot
// NOTE we handle tag/tree/blob objects specially - because these objects cannot // be in commit parents, we convert them to specially-crafted commits and use them.
// be in commit parents, we convert them to specially-crafted commits and use them. // The commits prepared contain full info how to restore original objects.
// The commits prepared contain full info how to restore original objects.
// backup.refs format:
// backup.refs format: //
// // 1eeb0324 <prefix>/wendelin.core.git/heads/master
// 1eeb0324 <prefix>/wendelin.core.git/heads/master // 213a9243 <prefix>/wendelin.core.git/tags/v0.4 <213a9243-converted-to-commit>
// 213a9243 <prefix>/wendelin.core.git/tags/v0.4 <213a9243-converted-to-commit> // ...
// ... //
// // NOTE `git for-each-ref` sorts output by ref
// NOTE `git for-each-ref` sorts output by ref // -> backup_refs is sorted and stable between runs
// -> backup_refs is sorted and stable between runs backup_refs_dump := xgit("for-each-ref", backup_refs_work)
backup_refs_dump := xgit("for-each-ref", backup_refs_work) backup_refs_list := []Ref{} // parsed dump
backup_refs_list := []Ref{} // parsed dump backup_refsv := []string{} // backup.refs content
backup_refsv := []string{} // backup.refs content backup_refs_parents := Sha1Set{} // sha1 for commit parents, obtained from refs
backup_refs_parents := Sha1Set{} // sha1 for commit parents, obtained from refs noncommit_seen := map[Sha1]Sha1{} // {} sha1 -> sha1_ (there are many duplicate tags)
noncommit_seen := map[Sha1]Sha1{} // {} sha1 -> sha1_ (there are many duplicate tags) for _, __ := range xstrings.SplitLines(backup_refs_dump, "\n") {
for _, __ := range xstrings.SplitLines(backup_refs_dump, "\n") { sha1, type_, ref := Sha1{}, "", ""
sha1, type_, ref := Sha1{}, "", "" _, err := fmt.Sscanf(__, "%s %s %s\n", &sha1, &type_, &ref)
_, err := fmt.Sscanf(__, "%s %s %s\n", &sha1, &type_, &ref) if err != nil {
if err != nil { exc.Raisef("%s: strange for-each-ref entry %q", backup_refs_work, __)
exc.Raisef("%s: strange for-each-ref entry %q", backup_refs_work, __) }
} backup_refs_list = append(backup_refs_list, Ref{ref, sha1})
backup_refs_list = append(backup_refs_list, Ref{ref, sha1}) backup_refs_entry := fmt.Sprintf("%s %s", sha1, strip_prefix(backup_refs_work, ref))
backup_refs_entry := fmt.Sprintf("%s %s", sha1, strip_prefix(backup_refs_work, ref))
// represent tag/tree/blob as specially crafted commit, because we
// represent tag/tree/blob as specially crafted commit, because we // cannot use it as commit parent.
// cannot use it as commit parent. sha1_ := sha1
sha1_ := sha1 if type_ != "commit" {
if type_ != "commit" { //infof("obj_as_commit %s %s\t%s", sha1, type_, ref) XXX
//infof("obj_as_commit %s %s\t%s", sha1, type_, ref) XXX var seen bool
var seen bool sha1_, seen = noncommit_seen[sha1]
sha1_, seen = noncommit_seen[sha1] if !seen {
if !seen { obj_type, ok := gittype(type_)
obj_type, ok := gittype(type_) if !ok {
if !ok { exc.Raisef("%s: invalid git type in entry %q", backup_refs_work, __)
exc.Raisef("%s: invalid git type in entry %q", backup_refs_work, __) }
} sha1_ = obj_represent_as_commit(gb, sha1, obj_type)
sha1_ = obj_represent_as_commit(gb, sha1, obj_type) noncommit_seen[sha1] = sha1_
noncommit_seen[sha1] = sha1_ }
}
backup_refs_entry += fmt.Sprintf(" %s", sha1_)
backup_refs_entry += fmt.Sprintf(" %s", sha1_) }
}
backup_refsv = append(backup_refsv, backup_refs_entry)
backup_refsv = append(backup_refsv, backup_refs_entry)
if !backup_refs_parents.Contains(sha1_) { // several refs can refer to the same sha1
if !backup_refs_parents.Contains(sha1_) { // several refs can refer to the same sha1 backup_refs_parents.Add(sha1_)
backup_refs_parents.Add(sha1_) }
} }
}
backup_refs := strings.Join(backup_refsv, "\n")
backup_refs := strings.Join(backup_refsv, "\n") backup_refs_parentv := backup_refs_parents.Elements()
backup_refs_parentv := backup_refs_parents.Elements() sort.Sort(BySha1(backup_refs_parentv)) // so parents order is stable in between runs
sort.Sort(BySha1(backup_refs_parentv)) // so parents order is stable in between runs
// backup_refs -> blob
// backup_refs -> blob backup_refs_sha1 := xgitSha1("hash-object", "-w", "--stdin", RunWith{stdin: backup_refs})
backup_refs_sha1 := xgitSha1("hash-object", "-w", "--stdin", RunWith{stdin: backup_refs})
// add backup_refs blob to index
// add backup_refs blob to index xgit("update-index", "--add", "--cacheinfo", fmt.Sprintf("100644,%s,backup.refs", backup_refs_sha1))
xgit("update-index", "--add", "--cacheinfo", fmt.Sprintf("100644,%s,backup.refs", backup_refs_sha1))
// index is ready - prepare tree and commit
// index is ready - prepare tree and commit backup_tree_sha1 := xgitSha1("write-tree")
backup_tree_sha1 := xgitSha1("write-tree") commit_sha1 := xcommit_tree(gb, backup_tree_sha1, append([]Sha1{HEAD}, backup_refs_parentv...),
commit_sha1 := xcommit_tree(gb, backup_tree_sha1, append([]Sha1{HEAD}, backup_refs_parentv...), "Git-backup "+backup_time)
"Git-backup " + backup_time)
xgit("update-ref", "-m", "git-backup pull", "HEAD", commit_sha1, HEAD)
xgit("update-ref", "-m", "git-backup pull", "HEAD", commit_sha1, HEAD)
// remove no-longer needed backup refs & verify they don't stay
// remove no-longer needed backup refs & verify they don't stay backup_refs_delete := ""
backup_refs_delete := "" for _, ref := range backup_refs_list {
for _, ref := range backup_refs_list { backup_refs_delete += fmt.Sprintf("delete %s %s\n", ref.name, ref.sha1)
backup_refs_delete += fmt.Sprintf("delete %s %s\n", ref.name, ref.sha1) }
}
xgit("update-ref", "--stdin", RunWith{stdin: backup_refs_delete})
xgit("update-ref", "--stdin", RunWith{stdin: backup_refs_delete}) __ = xgit("for-each-ref", backup_refs_work)
__ = xgit("for-each-ref", backup_refs_work) if __ != "" {
if __ != "" { exc.Raisef("Backup refs under %s not deleted properly", backup_refs_work)
exc.Raisef("Backup refs under %s not deleted properly", backup_refs_work) }
}
// NOTE `delete` deletes only files, but leaves empty dirs around.
// NOTE `delete` deletes only files, but leaves empty dirs around. // more important: this affect performance of future `git-backup pull` run a *LOT*
// more important: this affect performance of future `git-backup pull` run a *LOT* //
// // reason is: `git pull` first check local refs, and for doing so it
// reason is: `git pull` first check local refs, and for doing so it // recourse into all directories, even empty ones.
// recourse into all directories, even empty ones. //
// // https://lab.nexedi.com/lab.nexedi.com/lab.nexedi.com/issues/4
// https://lab.nexedi.com/lab.nexedi.com/lab.nexedi.com/issues/4 //
// // So remove all dirs under backup_refs_work prefix in the end.
// So remove all dirs under backup_refs_work prefix in the end. //
// // TODO Revisit this when reworking fetch to be parallel. Reason is: in
// TODO Revisit this when reworking fetch to be parallel. Reason is: in // the process of pulling repositories, the more references we
// the process of pulling repositories, the more references we // accumulate, the longer pull starts to be, so it becomes O(n^2).
// accumulate, the longer pull starts to be, so it becomes O(n^2). //
// // -> what to do is described nearby fetch/mkref call.
// -> what to do is described nearby fetch/mkref call. gitdir := xgit("rev-parse", "--git-dir")
gitdir := xgit("rev-parse", "--git-dir") err = os.RemoveAll(gitdir + "/" + backup_refs_work)
err = os.RemoveAll(gitdir+"/"+backup_refs_work) exc.Raiseif(err) // NOTE err is nil if path does not exist
exc.Raiseif(err) // NOTE err is nil if path does not exist
// if we have working copy - update it
// if we have working copy - update it bare := xgit("rev-parse", "--is-bare-repository")
bare := xgit("rev-parse", "--is-bare-repository") if bare != "true" {
if bare != "true" { // `git checkout-index -af` -- does not delete deleted files
// `git checkout-index -af` -- does not delete deleted files // `git read-tree -v -u --reset HEAD~ HEAD` -- needs index matching
// `git read-tree -v -u --reset HEAD~ HEAD` -- needs index matching // original worktree to properly work, but we already have updated index
// original worktree to properly work, but we already have updated index //
// // so we get changes we committed as diff and apply to worktree
// so we get changes we committed as diff and apply to worktree diff := xgit("diff", "--binary", HEAD, "HEAD", RunWith{raw: true})
diff := xgit("diff", "--binary", HEAD, "HEAD", RunWith{raw: true}) if diff != "" {
if diff != "" { diffstat := xgit("apply", "--stat", "--apply", "--binary", "--whitespace=nowarn",
diffstat := xgit("apply", "--stat", "--apply", "--binary", "--whitespace=nowarn", RunWith{stdin: diff, raw: true})
RunWith{stdin: diff, raw: true}) infof("%s", diffstat)
infof("%s", diffstat) }
} }
}
// we are done - unlock
// we are done - unlock xgit("update-ref", "-d", backup_lock)
xgit("update-ref", "-d", backup_lock)
} }
// fetch makes sure all objects from a repository are present in backup place. // fetch makes sure all objects from a repository are present in backup place.
...@@ -683,133 +682,133 @@ func cmd_pull_(gb *git.Repository, pullspecv []PullSpec) { ...@@ -683,133 +682,133 @@ func cmd_pull_(gb *git.Repository, pullspecv []PullSpec) {
// Note: fetch does not create any local references - the references returned // Note: fetch does not create any local references - the references returned
// only describe state of references in fetched source repository. // only describe state of references in fetched source repository.
func fetch(repo string, alreadyHave Sha1Set) (refv, fetchedv []Ref, err error) { func fetch(repo string, alreadyHave Sha1Set) (refv, fetchedv []Ref, err error) {
defer xerr.Contextf(&err, "fetch %s", repo) defer xerr.Contextf(&err, "fetch %s", repo)
// first check which references are advertised // first check which references are advertised
refv, err = lsremote(repo) refv, err = lsremote(repo)
if err != nil { if err != nil {
return nil, nil, err return nil, nil, err
} }
// check if we already have something // check if we already have something
var fetchv []Ref // references we need to actually fetch. var fetchv []Ref // references we need to actually fetch.
for _, ref := range refv { for _, ref := range refv {
if !alreadyHave.Contains(ref.sha1) { if !alreadyHave.Contains(ref.sha1) {
fetchv = append(fetchv, ref) fetchv = append(fetchv, ref)
} }
} }
// if there is nothing to fetch - we are done // if there is nothing to fetch - we are done
if len(fetchv) == 0 { if len(fetchv) == 0 {
return refv, fetchv, nil return refv, fetchv, nil
} }
// fetch by sha1 what we don't already have from advertised. // fetch by sha1 what we don't already have from advertised.
// //
// even if refs would change after ls-remote but before here, we should be // even if refs would change after ls-remote but before here, we should be
// getting exactly what was advertised. // getting exactly what was advertised.
// //
// related link on the subject: // related link on the subject:
// https://git.kernel.org/pub/scm/git/git.git/commit/?h=051e4005a3 // https://git.kernel.org/pub/scm/git/git.git/commit/?h=051e4005a3
var argv []interface{} var argv []interface{}
arg := func(v ...interface{}) { argv = append(argv, v...) } arg := func(v ...interface{}) { argv = append(argv, v...) }
arg( arg(
// check objects for corruption as they are fetched // check objects for corruption as they are fetched
"-c", "fetch.fsckObjects=true", "-c", "fetch.fsckObjects=true",
"fetch-pack", "--thin", "fetch-pack", "--thin",
// force upload-pack to allow us asking any sha1 we want. // force upload-pack to allow us asking any sha1 we want.
// needed because advertised refs we got at lsremote time could have changed. // needed because advertised refs we got at lsremote time could have changed.
"--upload-pack=git -c uploadpack.allowAnySHA1InWant=true" + "--upload-pack=git -c uploadpack.allowAnySHA1InWant=true"+
// workarounds for git < 2.11.1, which does not have uploadpack.allowAnySHA1InWant: // workarounds for git < 2.11.1, which does not have uploadpack.allowAnySHA1InWant:
" -c uploadpack.allowTipSHA1InWant=true -c uploadpack.allowReachableSHA1InWant=true" + " -c uploadpack.allowTipSHA1InWant=true -c uploadpack.allowReachableSHA1InWant=true"+
// //
" upload-pack", " upload-pack",
repo) repo)
for _, ref := range fetchv { for _, ref := range fetchv {
arg(ref.sha1) arg(ref.sha1)
} }
arg(RunWith{stderr: gitprogress()}) arg(RunWith{stderr: gitprogress()})
gerr, _, _ := ggit(argv...) gerr, _, _ := ggit(argv...)
if gerr != nil { if gerr != nil {
return nil, nil, gerr return nil, nil, gerr
} }
// fetch-pack ran ok - now check that all fetched tips are indeed fully // fetch-pack ran ok - now check that all fetched tips are indeed fully
// connected and that we also have all referenced blob/tree objects. The // connected and that we also have all referenced blob/tree objects. The
// reason for this check is that source repository could send us a pack with // reason for this check is that source repository could send us a pack with
// e.g. some objects missing and this way even if fetch-pack would report // e.g. some objects missing and this way even if fetch-pack would report
// success, chances could be we won't have all the objects we think we // success, chances could be we won't have all the objects we think we
// fetched. // fetched.
// //
// when checking we assume that the roots we already have at all our // when checking we assume that the roots we already have at all our
// references are ok. // references are ok.
// //
// related link on the subject: // related link on the subject:
// https://git.kernel.org/pub/scm/git/git.git/commit/?h=6d4bb3833c // https://git.kernel.org/pub/scm/git/git.git/commit/?h=6d4bb3833c
argv = nil argv = nil
arg("rev-list", "--quiet", "--objects", "--not", "--all", "--not") arg("rev-list", "--quiet", "--objects", "--not", "--all", "--not")
for _, ref := range fetchv { for _, ref := range fetchv {
arg(ref.sha1) arg(ref.sha1)
} }
arg(RunWith{stderr: gitprogress()}) arg(RunWith{stderr: gitprogress()})
gerr, _, _ = ggit(argv...) gerr, _, _ = ggit(argv...)
if gerr != nil { if gerr != nil {
return nil, nil, fmt.Errorf("remote did not send all neccessary objects") return nil, nil, fmt.Errorf("remote did not send all neccessary objects")
} }
// fetched ok // fetched ok
return refv, fetchv, nil return refv, fetchv, nil
} }
// lsremote lists all references advertised by repo. // lsremote lists all references advertised by repo.
func lsremote(repo string) (refv []Ref, err error) { func lsremote(repo string) (refv []Ref, err error) {
defer xerr.Contextf(&err, "lsremote %s", repo) defer xerr.Contextf(&err, "lsremote %s", repo)
// NOTE --refs instructs to omit peeled refs like // NOTE --refs instructs to omit peeled refs like
// //
// c668db59ccc59e97ce81f769d9f4633e27ad3bdb refs/tags/v0.1 // c668db59ccc59e97ce81f769d9f4633e27ad3bdb refs/tags/v0.1
// 4b6821f4a4e4c9648941120ccbab03982e33104f refs/tags/v0.1^{} <-- // 4b6821f4a4e4c9648941120ccbab03982e33104f refs/tags/v0.1^{} <--
// //
// because fetch-pack errors on them: // because fetch-pack errors on them:
// //
// https://public-inbox.org/git/20180610143231.7131-1-kirr@nexedi.com/ // https://public-inbox.org/git/20180610143231.7131-1-kirr@nexedi.com/
// //
// we don't need to pull them anyway. // we don't need to pull them anyway.
gerr, stdout, _ := ggit("ls-remote", "--refs", repo) gerr, stdout, _ := ggit("ls-remote", "--refs", repo)
if gerr != nil { if gerr != nil {
return nil, gerr return nil, gerr
} }
// oid refname // oid refname
// oid refname // oid refname
// ... // ...
for _, entry := range xstrings.SplitLines(stdout, "\n") { for _, entry := range xstrings.SplitLines(stdout, "\n") {
sha1, ref := Sha1{}, "" sha1, ref := Sha1{}, ""
_, err := fmt.Sscanf(entry, "%s %s\n", &sha1, &ref) _, err := fmt.Sscanf(entry, "%s %s\n", &sha1, &ref)
if err != nil { if err != nil {
return nil, fmt.Errorf("strange output entry: %q", entry) return nil, fmt.Errorf("strange output entry: %q", entry)
} }
// Ref says its name goes without "refs/" prefix. // Ref says its name goes without "refs/" prefix.
if !strings.HasPrefix(ref, "refs/") { if !strings.HasPrefix(ref, "refs/") {
return nil, fmt.Errorf("non-refs/ reference: %q", ref) return nil, fmt.Errorf("non-refs/ reference: %q", ref)
} }
ref = strings.TrimPrefix(ref, "refs/") ref = strings.TrimPrefix(ref, "refs/")
refv = append(refv, Ref{ref, sha1}) refv = append(refv, Ref{ref, sha1})
} }
return refv, nil return refv, nil
} }
// -------- git-backup restore -------- // -------- git-backup restore --------
func cmd_restore_usage() { func cmd_restore_usage() {
fmt.Fprint(os.Stderr, fmt.Fprint(os.Stderr,
`git-backup restore <commit-ish> <prefix1>:<dir1> <prefix2>:<dir2> ... `git-backup restore <commit-ish> <prefix1>:<dir1> <prefix2>:<dir2> ...
Restore Git repositories & just files from backup prefix1 into dir1, Restore Git repositories & just files from backup prefix1 into dir1,
...@@ -820,61 +819,61 @@ Backup state to restore is taken from <commit-ish>. ...@@ -820,61 +819,61 @@ Backup state to restore is taken from <commit-ish>.
} }
type RestoreSpec struct { type RestoreSpec struct {
prefix, dir string prefix, dir string
} }
func cmd_restore(gb *git.Repository, argv []string) { func cmd_restore(gb *git.Repository, argv []string) {
flags := flag.FlagSet{Usage: cmd_restore_usage} flags := flag.FlagSet{Usage: cmd_restore_usage}
flags.Init("", flag.ExitOnError) flags.Init("", flag.ExitOnError)
flags.Parse(argv) flags.Parse(argv)
argv = flags.Args() argv = flags.Args()
if len(argv) < 2 { if len(argv) < 2 {
cmd_restore_usage() cmd_restore_usage()
os.Exit(1) os.Exit(1)
} }
HEAD := argv[0] HEAD := argv[0]
restorespecv := []RestoreSpec{} restorespecv := []RestoreSpec{}
for _, arg := range argv[1:] { for _, arg := range argv[1:] {
prefix, dir, err := xstrings.Split2(arg, ":") prefix, dir, err := xstrings.Split2(arg, ":")
if err != nil { if err != nil {
fmt.Fprintf(os.Stderr, "E: invalid restorespec %q\n", arg) fmt.Fprintf(os.Stderr, "E: invalid restorespec %q\n", arg)
cmd_restore_usage() cmd_restore_usage()
os.Exit(1) os.Exit(1)
} }
restorespecv = append(restorespecv, RestoreSpec{prefix, dir}) restorespecv = append(restorespecv, RestoreSpec{prefix, dir})
} }
cmd_restore_(gb, HEAD, restorespecv) cmd_restore_(gb, HEAD, restorespecv)
} }
// kirr/wendelin.core.git/heads/master -> kirr/wendelin.core.git, heads/master // kirr/wendelin.core.git/heads/master -> kirr/wendelin.core.git, heads/master
// tiwariayush/Discussion%20Forum%20.git/... -> tiwariayush/Discussion Forum .git, ... // tiwariayush/Discussion%20Forum%20.git/... -> tiwariayush/Discussion Forum .git, ...
func reporef_split(reporef string) (repo, ref string) { func reporef_split(reporef string) (repo, ref string) {
dotgit := strings.Index(reporef, ".git/") dotgit := strings.Index(reporef, ".git/")
if dotgit == -1 { if dotgit == -1 {
exc.Raisef("E: %s is not a ref for a git repo", reporef) exc.Raisef("E: %s is not a ref for a git repo", reporef)
} }
repo, ref = reporef[:dotgit+4], reporef[dotgit+4+1:] repo, ref = reporef[:dotgit+4], reporef[dotgit+4+1:]
repo, err := path_refunescape(repo) // unescape repo name we originally escaped when making backup repo, err := path_refunescape(repo) // unescape repo name we originally escaped when making backup
exc.Raiseif(err) exc.Raiseif(err)
return repo, ref return repo, ref
} }
// sha1 value(s) for a ref in 'backup.refs' // sha1 value(s) for a ref in 'backup.refs'
type BackupRefSha1 struct { type BackupRefSha1 struct {
sha1 Sha1 // original sha1 this ref was pointing to in original repo sha1 Sha1 // original sha1 this ref was pointing to in original repo
sha1_ Sha1 // sha1 actually used to represent sha1's object in backup repo sha1_ Sha1 // sha1 actually used to represent sha1's object in backup repo
// (for tag/tree/blob - they are converted to commits) // (for tag/tree/blob - they are converted to commits)
} }
// BackupRef represents 1 reference entry in 'backup.refs' (repo prefix stripped) // BackupRef represents 1 reference entry in 'backup.refs' (repo prefix stripped)
type BackupRef struct { type BackupRef struct {
name string // reference name without "refs/" prefix name string // reference name without "refs/" prefix
BackupRefSha1 BackupRefSha1
} }
// {} refname -> sha1, sha1_ // {} refname -> sha1, sha1_
...@@ -882,17 +881,17 @@ type RefMap map[string]BackupRefSha1 ...@@ -882,17 +881,17 @@ type RefMap map[string]BackupRefSha1
// info about a repository from backup.refs // info about a repository from backup.refs
type BackupRepo struct { type BackupRepo struct {
repopath string // full repo path with backup prefix repopath string // full repo path with backup prefix
refs RefMap refs RefMap
} }
// all RefMap values as flat []BackupRef // all RefMap values as flat []BackupRef
func (m RefMap) Values() []BackupRef { func (m RefMap) Values() []BackupRef {
ev := make([]BackupRef, 0, len(m)) ev := make([]BackupRef, 0, len(m))
for ref, refsha1 := range m { for ref, refsha1 := range m {
ev = append(ev, BackupRef{ref, refsha1}) ev = append(ev, BackupRef{ref, refsha1})
} }
return ev return ev
} }
// for sorting []BackupRef by refname // for sorting []BackupRef by refname
...@@ -904,22 +903,22 @@ func (br ByRefname) Less(i, j int) bool { return strings.Compare(br[i].name, br[ ...@@ -904,22 +903,22 @@ func (br ByRefname) Less(i, j int) bool { return strings.Compare(br[i].name, br[
// all sha1 heads RefMap points to, in sorted order // all sha1 heads RefMap points to, in sorted order
func (m RefMap) Sha1Heads() []Sha1 { func (m RefMap) Sha1Heads() []Sha1 {
hs := Sha1Set{} hs := Sha1Set{}
for _, refsha1 := range m { for _, refsha1 := range m {
hs.Add(refsha1.sha1) hs.Add(refsha1.sha1)
} }
headv := hs.Elements() headv := hs.Elements()
sort.Sort(BySha1(headv)) sort.Sort(BySha1(headv))
return headv return headv
} }
// like Sha1Heads() but returns heads in text format delimited by "\n" // like Sha1Heads() but returns heads in text format delimited by "\n"
func (m RefMap) Sha1HeadsStr() string { func (m RefMap) Sha1HeadsStr() string {
s := "" s := ""
for _, sha1 := range m.Sha1Heads() { for _, sha1 := range m.Sha1Heads() {
s += sha1.String() + "\n" s += sha1.String() + "\n"
} }
return s return s
} }
// for sorting []BackupRepo by repopath // for sorting []BackupRepo by repopath
...@@ -931,282 +930,282 @@ func (br ByRepoPath) Less(i, j int) bool { return strings.Compare(br[i].repopath ...@@ -931,282 +930,282 @@ func (br ByRepoPath) Less(i, j int) bool { return strings.Compare(br[i].repopath
// also for searching sorted []BackupRepo by repopath prefix // also for searching sorted []BackupRepo by repopath prefix
func (br ByRepoPath) Search(prefix string) int { func (br ByRepoPath) Search(prefix string) int {
return sort.Search(len(br), func (i int) bool { return sort.Search(len(br), func(i int) bool {
return strings.Compare(br[i].repopath, prefix) >= 0 return strings.Compare(br[i].repopath, prefix) >= 0
}) })
} }
// request to extract a pack // request to extract a pack
type PackExtractReq struct { type PackExtractReq struct {
refs RefMap // extract pack with objects from this heads refs RefMap // extract pack with objects from this heads
repopath string // into repository located here repopath string // into repository located here
// for info only: request was generated restoring from under this backup prefix // for info only: request was generated restoring from under this backup prefix
prefix string prefix string
} }
func cmd_restore_(gb *git.Repository, HEAD_ string, restorespecv []RestoreSpec) { func cmd_restore_(gb *git.Repository, HEAD_ string, restorespecv []RestoreSpec) {
HEAD := xgitSha1("rev-parse", "--verify", HEAD_) HEAD := xgitSha1("rev-parse", "--verify", HEAD_)
// read backup refs index // read backup refs index
repotab, err := loadBackupRefs(fmt.Sprintf("%s:backup.refs", HEAD)) repotab, err := loadBackupRefs(fmt.Sprintf("%s:backup.refs", HEAD))
exc.Raiseif(err) exc.Raiseif(err)
// flattened & sorted repotab // flattened & sorted repotab
// NOTE sorted - to process repos always in the same order & for searching // NOTE sorted - to process repos always in the same order & for searching
repov := make([]*BackupRepo, 0, len(repotab)) repov := make([]*BackupRepo, 0, len(repotab))
for _, repo := range repotab { for _, repo := range repotab {
repov = append(repov, repo) repov = append(repov, repo)
} }
sort.Sort(ByRepoPath(repov)) sort.Sort(ByRepoPath(repov))
// repotab no longer needed // repotab no longer needed
repotab = nil repotab = nil
packxq := make(chan PackExtractReq, 2*njobs) // requests to extract packs packxq := make(chan PackExtractReq, 2*njobs) // requests to extract packs
errch := make(chan error) // errors from workers errch := make(chan error) // errors from workers
stopch := make(chan struct{}) // broadcasts restore has to be cancelled stopch := make(chan struct{}) // broadcasts restore has to be cancelled
wg := sync.WaitGroup{} wg := sync.WaitGroup{}
// main worker: walk over specified prefixes restoring files and // main worker: walk over specified prefixes restoring files and
// scheduling pack extraction requests from *.git -> packxq // scheduling pack extraction requests from *.git -> packxq
wg.Add(1) wg.Add(1)
go func() { go func() {
defer wg.Done() defer wg.Done()
defer close(packxq) defer close(packxq)
// raised err -> errch // raised err -> errch
here := my.FuncName() here := my.FuncName()
defer exc.Catch(func(e *exc.Error) { defer exc.Catch(func(e *exc.Error) {
errch <- exc.Addcallingcontext(here, e) errch <- exc.Addcallingcontext(here, e)
}) })
runloop: runloop:
for _, __ := range restorespecv { for _, __ := range restorespecv {
prefix, dir := __.prefix, __.dir prefix, dir := __.prefix, __.dir
// ensure dir did not exist before restore run // ensure dir did not exist before restore run
err := os.Mkdir(dir, 0777) err := os.Mkdir(dir, 0777)
exc.Raiseif(err) exc.Raiseif(err)
// files // files
lstree := xgit("ls-tree", "--full-tree", "-r", "-z", "--", HEAD, prefix, RunWith{raw: true}) lstree := xgit("ls-tree", "--full-tree", "-r", "-z", "--", HEAD, prefix, RunWith{raw: true})
repos_seen := StrSet{} // dirs of *.git seen while restoring files repos_seen := StrSet{} // dirs of *.git seen while restoring files
for _, __ := range xstrings.SplitLines(lstree, "\x00") { for _, __ := range xstrings.SplitLines(lstree, "\x00") {
mode, type_, sha1, filename, err := parse_lstree_entry(__) mode, type_, sha1, filename, err := parse_lstree_entry(__)
// NOTE // NOTE
// - `ls-tree -r` shows only leaf objects // - `ls-tree -r` shows only leaf objects
// - git-backup repository does not have submodules and the like // - git-backup repository does not have submodules and the like
// -> type should be "blob" only // -> type should be "blob" only
if err != nil || type_ != "blob" { if err != nil || type_ != "blob" {
exc.Raisef("%s: invalid/unexpected ls-tree entry %q", HEAD, __) exc.Raisef("%s: invalid/unexpected ls-tree entry %q", HEAD, __)
} }
filename = reprefix(prefix, dir, filename) filename = reprefix(prefix, dir, filename)
infof("# file %s\t-> %s", prefix, filename) infof("# file %s\t-> %s", prefix, filename)
blob_to_file(gb, sha1, mode, filename) blob_to_file(gb, sha1, mode, filename)
// make sure git will recognize *.git as repo: // make sure git will recognize *.git as repo:
// - it should have refs/{heads,tags}/ and objects/pack/ inside. // - it should have refs/{heads,tags}/ and objects/pack/ inside.
// //
// NOTE doing it while restoring files, because a repo could be // NOTE doing it while restoring files, because a repo could be
// empty - without refs at all, and thus next "git packs restore" // empty - without refs at all, and thus next "git packs restore"
// step will not be run for it. // step will not be run for it.
filedir := pathpkg.Dir(filename) filedir := pathpkg.Dir(filename)
if strings.HasSuffix(filedir, ".git") && !repos_seen.Contains(filedir) { if strings.HasSuffix(filedir, ".git") && !repos_seen.Contains(filedir) {
infof("# repo %s\t-> %s", prefix, filedir) infof("# repo %s\t-> %s", prefix, filedir)
for _, __ := range []string{"refs/heads", "refs/tags", "objects/pack"} { for _, __ := range []string{"refs/heads", "refs/tags", "objects/pack"} {
err := os.MkdirAll(filedir+"/"+__, 0777) err := os.MkdirAll(filedir+"/"+__, 0777)
exc.Raiseif(err) exc.Raiseif(err)
} }
repos_seen.Add(filedir) repos_seen.Add(filedir)
} }
} }
// git packs // git packs
for i := ByRepoPath(repov).Search(prefix); i < len(repov); i++ { for i := ByRepoPath(repov).Search(prefix); i < len(repov); i++ {
repo := repov[i] repo := repov[i]
if !strings.HasPrefix(repo.repopath, prefix) { if !strings.HasPrefix(repo.repopath, prefix) {
break // repov is sorted - end of repositories with prefix break // repov is sorted - end of repositories with prefix
} }
// make sure tag/tree/blob objects represented as commits are // make sure tag/tree/blob objects represented as commits are
// present, before we generate pack for restored repo. // present, before we generate pack for restored repo.
// ( such objects could be lost e.g. after backup repo repack as they // ( such objects could be lost e.g. after backup repo repack as they
// are not reachable from backup repo HEAD ) // are not reachable from backup repo HEAD )
for _, __ := range repo.refs { for _, __ := range repo.refs {
if __.sha1 != __.sha1_ { if __.sha1 != __.sha1_ {
obj_recreate_from_commit(gb, __.sha1_) obj_recreate_from_commit(gb, __.sha1_)
} }
} }
select { select {
case packxq <- PackExtractReq{refs: repo.refs, case packxq <- PackExtractReq{refs: repo.refs,
repopath: reprefix(prefix, dir, repo.repopath), repopath: reprefix(prefix, dir, repo.repopath),
prefix: prefix}: prefix: prefix}:
case <-stopch: case <-stopch:
break runloop break runloop
} }
} }
} }
}() }()
// pack workers: packxq -> extract packs // pack workers: packxq -> extract packs
for i := 0; i < njobs; i++ { for i := 0; i < njobs; i++ {
wg.Add(1) wg.Add(1)
go func() { go func() {
defer wg.Done() defer wg.Done()
// raised err -> errch // raised err -> errch
here := my.FuncName() here := my.FuncName()
defer exc.Catch(func(e *exc.Error) { defer exc.Catch(func(e *exc.Error) {
errch <- exc.Addcallingcontext(here, e) errch <- exc.Addcallingcontext(here, e)
}) })
runloop: runloop:
for { for {
select { select {
case <-stopch: case <-stopch:
break runloop break runloop
case p, ok := <-packxq: case p, ok := <-packxq:
if !ok { if !ok {
break runloop break runloop
} }
infof("# git %s\t-> %s", p.prefix, p.repopath) infof("# git %s\t-> %s", p.prefix, p.repopath)
// extract pack for that repo from big backup pack + decoded tags // extract pack for that repo from big backup pack + decoded tags
pack_argv := []string{ pack_argv := []string{
"-c", "pack.threads=1", // occupy only 1 CPU + it packs better "-c", "pack.threads=1", // occupy only 1 CPU + it packs better
"pack-objects", "pack-objects",
"--revs", // include all objects referencable from input sha1 list "--revs", // include all objects referencable from input sha1 list
"--reuse-object", "--reuse-delta", "--delta-base-offset", "--reuse-object", "--reuse-delta", "--delta-base-offset",
// use bitmap index from backup repo, if present (faster pack generation) // use bitmap index from backup repo, if present (faster pack generation)
// https://git.kernel.org/pub/scm/git/git.git/commit/?h=645c432d61 // https://git.kernel.org/pub/scm/git/git.git/commit/?h=645c432d61
"--use-bitmap-index", "--use-bitmap-index",
} }
if verbose <= 0 { if verbose <= 0 {
pack_argv = append(pack_argv, "-q") pack_argv = append(pack_argv, "-q")
} }
pack_argv = append(pack_argv, p.repopath+"/objects/pack/pack") pack_argv = append(pack_argv, p.repopath+"/objects/pack/pack")
xgit2(pack_argv, RunWith{stdin: p.refs.Sha1HeadsStr(), stderr: gitprogress()}) xgit2(pack_argv, RunWith{stdin: p.refs.Sha1HeadsStr(), stderr: gitprogress()})
// verify that extracted repo refs match backup.refs index after extraction // verify that extracted repo refs match backup.refs index after extraction
x_ref_list := xgit("--git-dir=" + p.repopath, x_ref_list := xgit("--git-dir="+p.repopath,
"for-each-ref", "--format=%(objectname) %(refname)") "for-each-ref", "--format=%(objectname) %(refname)")
repo_refs := p.refs.Values() repo_refs := p.refs.Values()
sort.Sort(ByRefname(repo_refs)) sort.Sort(ByRefname(repo_refs))
repo_ref_listv := make([]string, 0, len(repo_refs)) repo_ref_listv := make([]string, 0, len(repo_refs))
for _, ref := range repo_refs { for _, ref := range repo_refs {
repo_ref_listv = append(repo_ref_listv, fmt.Sprintf("%s refs/%s", ref.sha1, ref.name)) repo_ref_listv = append(repo_ref_listv, fmt.Sprintf("%s refs/%s", ref.sha1, ref.name))
} }
repo_ref_list := strings.Join(repo_ref_listv, "\n") repo_ref_list := strings.Join(repo_ref_listv, "\n")
if x_ref_list != repo_ref_list { if x_ref_list != repo_ref_list {
// TODO show refs diff, not 2 dumps // TODO show refs diff, not 2 dumps
exc.Raisef("E: extracted %s refs corrupt:\n\nwant:\n%s\n\nhave:\n%s", exc.Raisef("E: extracted %s refs corrupt:\n\nwant:\n%s\n\nhave:\n%s",
p.repopath, repo_ref_list, x_ref_list) p.repopath, repo_ref_list, x_ref_list)
} }
// check connectivity in recreated repository. // check connectivity in recreated repository.
// //
// This way we verify that extracted pack indeed contains all // This way we verify that extracted pack indeed contains all
// objects for all refs in the repo. // objects for all refs in the repo.
// //
// Compared to fsck we do not re-compute sha1 sum of objects which // Compared to fsck we do not re-compute sha1 sum of objects which
// is significantly faster. // is significantly faster.
gerr, _, _ := ggit("--git-dir=" + p.repopath, gerr, _, _ := ggit("--git-dir="+p.repopath,
"rev-list", "--objects", "--stdin", "--quiet", RunWith{stdin: p.refs.Sha1HeadsStr()}) "rev-list", "--objects", "--stdin", "--quiet", RunWith{stdin: p.refs.Sha1HeadsStr()})
if gerr != nil { if gerr != nil {
fmt.Fprintln(os.Stderr, "E: Problem while checking connectivity of extracted repo:") fmt.Fprintln(os.Stderr, "E: Problem while checking connectivity of extracted repo:")
exc.Raise(gerr) exc.Raise(gerr)
} }
// XXX disabled because it is slow // XXX disabled because it is slow
// // NOTE progress goes to stderr, problems go to stdout // // NOTE progress goes to stderr, problems go to stdout
// xgit("--git-dir=" + p.repopath, "fsck", // xgit("--git-dir=" + p.repopath, "fsck",
// # only check that traversal from refs is ok: this unpacks // # only check that traversal from refs is ok: this unpacks
// # commits and trees and verifies blob objects are there, // # commits and trees and verifies blob objects are there,
// # but do _not_ unpack blobs =fast. // # but do _not_ unpack blobs =fast.
// "--connectivity-only", // "--connectivity-only",
// RunWith{stdout: gitprogress(), stderr: gitprogress()}) // RunWith{stdout: gitprogress(), stderr: gitprogress()})
} }
} }
}() }()
} }
// wait for workers to finish & collect/reraise their errors // wait for workers to finish & collect/reraise their errors
go func() { go func() {
wg.Wait() wg.Wait()
close(errch) close(errch)
}() }()
ev := xerr.Errorv{} ev := xerr.Errorv{}
for e := range errch { for e := range errch {
// tell everything to stop on first error // tell everything to stop on first error
if len(ev) == 0 { if len(ev) == 0 {
close(stopch) close(stopch)
} }
ev = append(ev, e) ev = append(ev, e)
} }
if len(ev) != 0 { if len(ev) != 0 {
exc.Raise(ev) exc.Raise(ev)
} }
} }
// loadBackupRefs loads 'backup.ref' content from a git object. // loadBackupRefs loads 'backup.ref' content from a git object.
// //
// an example of object is e.g. "HEAD:backup.ref". // an example of object is e.g. "HEAD:backup.ref".
func loadBackupRefs(object string) (repotab map[string]*BackupRepo, err error) { func loadBackupRefs(object string) (repotab map[string]*BackupRepo, err error) {
defer xerr.Contextf(&err, "load backup.refs %q", object) defer xerr.Contextf(&err, "load backup.refs %q", object)
gerr, backup_refs, _ := ggit("cat-file", "blob", object) gerr, backup_refs, _ := ggit("cat-file", "blob", object)
if gerr != nil { if gerr != nil {
return nil, gerr return nil, gerr
} }
repotab = make(map[string]*BackupRepo) repotab = make(map[string]*BackupRepo)
for _, refentry := range xstrings.SplitLines(backup_refs, "\n") { for _, refentry := range xstrings.SplitLines(backup_refs, "\n") {
// sha1 prefix+refname (sha1_) // sha1 prefix+refname (sha1_)
badentry := func() error { return fmt.Errorf("invalid entry: %q", refentry) } badentry := func() error { return fmt.Errorf("invalid entry: %q", refentry) }
refentryv := strings.Fields(refentry) refentryv := strings.Fields(refentry)
if !(2 <= len(refentryv) && len(refentryv) <= 3) { if !(2 <= len(refentryv) && len(refentryv) <= 3) {
return nil, badentry() return nil, badentry()
} }
sha1, err := Sha1Parse(refentryv[0]) sha1, err := Sha1Parse(refentryv[0])
sha1_, err_ := sha1, err sha1_, err_ := sha1, err
if len(refentryv) == 3 { if len(refentryv) == 3 {
sha1_, err_ = Sha1Parse(refentryv[2]) sha1_, err_ = Sha1Parse(refentryv[2])
} }
if err != nil || err_ != nil { if err != nil || err_ != nil {
return nil, badentry() return nil, badentry()
} }
reporef := refentryv[1] reporef := refentryv[1]
repopath, ref := reporef_split(reporef) repopath, ref := reporef_split(reporef)
repo := repotab[repopath] repo := repotab[repopath]
if repo == nil { if repo == nil {
repo = &BackupRepo{repopath, RefMap{}} repo = &BackupRepo{repopath, RefMap{}}
repotab[repopath] = repo repotab[repopath] = repo
} }
if _, alreadyin := repo.refs[ref]; alreadyin { if _, alreadyin := repo.refs[ref]; alreadyin {
return nil, fmt.Errorf("duplicate ref %q", ref) return nil, fmt.Errorf("duplicate ref %q", ref)
} }
repo.refs[ref] = BackupRefSha1{sha1, sha1_} repo.refs[ref] = BackupRefSha1{sha1, sha1_}
} }
return repotab, nil return repotab, nil
} }
var commands = map[string]func(*git.Repository, []string){ var commands = map[string]func(*git.Repository, []string){
"pull": cmd_pull, "pull": cmd_pull,
"restore": cmd_restore, "restore": cmd_restore,
} }
func usage() { func usage() {
fmt.Fprintf(os.Stderr, fmt.Fprintf(os.Stderr,
`git-backup [options] <command> `git-backup [options] <command>
pull pull git-repositories and files to backup pull pull git-repositories and files to backup
...@@ -1222,44 +1221,44 @@ func usage() { ...@@ -1222,44 +1221,44 @@ func usage() {
} }
func main() { func main() {
flag.Usage = usage flag.Usage = usage
quiet := 0 quiet := 0
flag.Var((*xflag.Count)(&verbose), "v", "verbosity level") flag.Var((*xflag.Count)(&verbose), "v", "verbosity level")
flag.Var((*xflag.Count)(&quiet), "q", "decrease verbosity") flag.Var((*xflag.Count)(&quiet), "q", "decrease verbosity")
flag.IntVar(&njobs, "j", njobs, "allow max N jobs to spawn") flag.IntVar(&njobs, "j", njobs, "allow max N jobs to spawn")
flag.Parse() flag.Parse()
verbose -= quiet verbose -= quiet
argv := flag.Args() argv := flag.Args()
if len(argv) == 0 { if len(argv) == 0 {
usage() usage()
os.Exit(1) os.Exit(1)
} }
cmd := commands[argv[0]] cmd := commands[argv[0]]
if cmd == nil { if cmd == nil {
fmt.Fprintf(os.Stderr, "E: unknown command %q", argv[0]) fmt.Fprintf(os.Stderr, "E: unknown command %q", argv[0])
os.Exit(1) os.Exit(1)
} }
// catch Error and report info from it // catch Error and report info from it
here := my.FuncName() here := my.FuncName()
defer exc.Catch(func(e *exc.Error) { defer exc.Catch(func(e *exc.Error) {
e = exc.Addcallingcontext(here, e) e = exc.Addcallingcontext(here, e)
fmt.Fprintln(os.Stderr, e) fmt.Fprintln(os.Stderr, e)
// also show traceback if debug // also show traceback if debug
if verbose > 2 { if verbose > 2 {
fmt.Fprint(os.Stderr, "\n") fmt.Fprint(os.Stderr, "\n")
debug.PrintStack() debug.PrintStack()
} }
os.Exit(1) os.Exit(1)
}) })
// backup repository // backup repository
gb, err := git.OpenRepository(".") gb, err := git.OpenRepository(".")
exc.Raiseif(err) exc.Raiseif(err)
cmd(gb, argv[1:]) cmd(gb, argv[1:])
} }
git-backup_test.go
View file @ f3f694b9
...@@ -20,355 +20,355 @@ ...@@ -20,355 +20,355 @@
package main package main
import ( import (
"fmt" "fmt"
"io/ioutil" "io/ioutil"
"os" "os"
"path/filepath" "path/filepath"
"regexp" "regexp"
"strings" "strings"
"syscall" "syscall"
"testing" "testing"
"lab.nexedi.com/kirr/go123/exc" "lab.nexedi.com/kirr/go123/exc"
"lab.nexedi.com/kirr/go123/my" "lab.nexedi.com/kirr/go123/my"
"lab.nexedi.com/kirr/go123/xruntime" "lab.nexedi.com/kirr/go123/xruntime"
"lab.nexedi.com/kirr/go123/xstrings" "lab.nexedi.com/kirr/go123/xstrings"
git "github.com/libgit2/git2go" git "github.com/libgit2/git2go"
) )
func xgetcwd(t *testing.T) string { func xgetcwd(t *testing.T) string {
cwd, err := os.Getwd() cwd, err := os.Getwd()
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
return cwd return cwd
} }
func xchdir(t *testing.T, dir string) { func xchdir(t *testing.T, dir string) {
err := os.Chdir(dir) err := os.Chdir(dir)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
} }
func XSha1(s string) Sha1 { func XSha1(s string) Sha1 {
sha1, err := Sha1Parse(s) sha1, err := Sha1Parse(s)
if err != nil { if err != nil {
panic(err) panic(err)
} }
return sha1 return sha1
} }
func xgittype(s string) git.ObjectType { func xgittype(s string) git.ObjectType {
type_, ok := gittype(s) type_, ok := gittype(s)
if !ok { if !ok {
exc.Raisef("unknown git type %q", s) exc.Raisef("unknown git type %q", s)
} }
return type_ return type_
} }
// verify end-to-end pull-restore // verify end-to-end pull-restore
func TestPullRestore(t *testing.T) { func TestPullRestore(t *testing.T) {
// if something raises -> don't let testing panic - report it as proper error with context. // if something raises -> don't let testing panic - report it as proper error with context.
here := my.FuncName() here := my.FuncName()
defer exc.Catch(func(e *exc.Error) { defer exc.Catch(func(e *exc.Error) {
e = exc.Addcallingcontext(here, e) e = exc.Addcallingcontext(here, e)
// add file:line for failing code inside testing function - so we have exact context to debug // add file:line for failing code inside testing function - so we have exact context to debug
failedat := "" failedat := ""
for _, f := range xruntime.Traceback(1) { for _, f := range xruntime.Traceback(1) {
if f.Function == here { if f.Function == here {
failedat = fmt.Sprintf("%s:%d", filepath.Base(f.File), f.Line) failedat = fmt.Sprintf("%s:%d", filepath.Base(f.File), f.Line)
break break
} }
} }
if failedat == "" { if failedat == "" {
panic(fmt.Errorf("cannot lookup failedat for %s", here)) panic(fmt.Errorf("cannot lookup failedat for %s", here))
} }
t.Errorf("%s: %v", failedat, e) t.Errorf("%s: %v", failedat, e)
}) })
workdir, err := ioutil.TempDir("", "t-git-backup") workdir, err := ioutil.TempDir("", "t-git-backup")
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
defer os.RemoveAll(workdir) defer os.RemoveAll(workdir)
mydir := xgetcwd(t) mydir := xgetcwd(t)
xchdir(t, workdir) xchdir(t, workdir)
defer xchdir(t, mydir) defer xchdir(t, mydir)
// -test.v -> verbosity of git-backup // -test.v -> verbosity of git-backup
if testing.Verbose() { if testing.Verbose() {
verbose = 1 verbose = 1
} else { } else {
verbose = 0 verbose = 0
} }
// init backup repository // init backup repository
xgit("init", "--bare", "backup.git") xgit("init", "--bare", "backup.git")
xchdir(t, "backup.git") xchdir(t, "backup.git")
gb, err := git.OpenRepository(".") gb, err := git.OpenRepository(".")
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
// pull from testdata // pull from testdata
my0 := mydir + "/testdata/0" my0 := mydir + "/testdata/0"
cmd_pull(gb, []string{my0+":b0"}) // only empty repo in testdata/0 cmd_pull(gb, []string{my0 + ":b0"}) // only empty repo in testdata/0
my1 := mydir + "/testdata/1" my1 := mydir + "/testdata/1"
cmd_pull(gb, []string{my1+":b1"}) cmd_pull(gb, []string{my1 + ":b1"})
// verify tag/tree/blob encoding is 1) consistent and 2) always the same. // verify tag/tree/blob encoding is 1) consistent and 2) always the same.
// we need it be always the same so different git-backup versions can // we need it be always the same so different git-backup versions can
// interoperate with each other. // interoperate with each other.
var noncommitv = []struct{ var noncommitv = []struct {
sha1 Sha1 // original sha1 Sha1 // original
sha1_ Sha1 // encoded sha1_ Sha1 // encoded
istag bool // is original object a tag object istag bool // is original object a tag object
}{ }{
{XSha1("f735011c9fcece41219729a33f7876cd8791f659"), XSha1("4f2486e99ff9744751e0756b155e57bb24c453dd"), true}, // tag-to-commit {XSha1("f735011c9fcece41219729a33f7876cd8791f659"), XSha1("4f2486e99ff9744751e0756b155e57bb24c453dd"), true}, // tag-to-commit
{XSha1("7124713e403925bc772cd252b0dec099f3ced9c5"), XSha1("6b3beabee3e0704fa3269558deab01e9d5d7764e"), true}, // tag-to-tag {XSha1("7124713e403925bc772cd252b0dec099f3ced9c5"), XSha1("6b3beabee3e0704fa3269558deab01e9d5d7764e"), true}, // tag-to-tag
{XSha1("11e67095628aa17b03436850e690faea3006c25d"), XSha1("89ad5fbeb9d3f0c7bc6366855a09484819289911"), true}, // tag-to-blob {XSha1("11e67095628aa17b03436850e690faea3006c25d"), XSha1("89ad5fbeb9d3f0c7bc6366855a09484819289911"), true}, // tag-to-blob
{XSha1("ba899e5639273a6fa4d50d684af8db1ae070351e"), XSha1("68ad6a7c31042e53201e47aee6096ed081b6fdb9"), true}, // tag-to-tree {XSha1("ba899e5639273a6fa4d50d684af8db1ae070351e"), XSha1("68ad6a7c31042e53201e47aee6096ed081b6fdb9"), true}, // tag-to-tree
{XSha1("61882eb85774ed4401681d800bb9c638031375e2"), XSha1("761f55bcdf119ced3fcf23b69fdc169cbb5fc143"), false}, // ref-to-tree {XSha1("61882eb85774ed4401681d800bb9c638031375e2"), XSha1("761f55bcdf119ced3fcf23b69fdc169cbb5fc143"), false}, // ref-to-tree
{XSha1("7a3343f584218e973165d943d7c0af47a52ca477"), XSha1("366f3598d662909e2537481852e42b775c7eb837"), false}, // ref-to-blob {XSha1("7a3343f584218e973165d943d7c0af47a52ca477"), XSha1("366f3598d662909e2537481852e42b775c7eb837"), false}, // ref-to-blob
} }
for _, nc := range noncommitv { for _, nc := range noncommitv {
// encoded object should be already present // encoded object should be already present
_, err := ReadObject2(gb, nc.sha1_) _, err := ReadObject2(gb, nc.sha1_)
if err != nil { if err != nil {
t.Fatalf("encode %s should give %s but expected encoded object not found: %s", nc.sha1, nc.sha1_, err) t.Fatalf("encode %s should give %s but expected encoded object not found: %s", nc.sha1, nc.sha1_, err)
} }
// decoding encoded object should give original sha1, if it was tag // decoding encoded object should give original sha1, if it was tag
sha1 := obj_recreate_from_commit(gb, nc.sha1_) sha1 := obj_recreate_from_commit(gb, nc.sha1_)
if nc.istag && sha1 != nc.sha1 { if nc.istag && sha1 != nc.sha1 {
t.Fatalf("decode %s -> %s ; want %s", nc.sha1_, sha1, nc.sha1) t.Fatalf("decode %s -> %s ; want %s", nc.sha1_, sha1, nc.sha1)
} }
// encoding original object should give sha1_ // encoding original object should give sha1_
obj_type := xgit("cat-file", "-t", nc.sha1) obj_type := xgit("cat-file", "-t", nc.sha1)
sha1_ := obj_represent_as_commit(gb, nc.sha1, xgittype(obj_type)) sha1_ := obj_represent_as_commit(gb, nc.sha1, xgittype(obj_type))
if sha1_ != nc.sha1_ { if sha1_ != nc.sha1_ {
t.Fatalf("encode %s -> %s ; want %s", sha1, sha1_, nc.sha1_) t.Fatalf("encode %s -> %s ; want %s", sha1, sha1_, nc.sha1_)
} }
} }
// checks / cleanups after cmd_pull // checks / cleanups after cmd_pull
afterPull := func() { afterPull := func() {
// verify no garbage is left under refs/backup/ // verify no garbage is left under refs/backup/
dentryv, err := ioutil.ReadDir("refs/backup/") dentryv, err := ioutil.ReadDir("refs/backup/")
if err != nil && !os.IsNotExist(err) { if err != nil && !os.IsNotExist(err) {
t.Fatal(err) t.Fatal(err)
} }
if len(dentryv) != 0 { if len(dentryv) != 0 {
namev := []string{} namev := []string{}
for _, fi := range dentryv { for _, fi := range dentryv {
namev = append(namev, fi.Name()) namev = append(namev, fi.Name())
} }
t.Fatalf("refs/backup/ not empty after pull: %v", namev) t.Fatalf("refs/backup/ not empty after pull: %v", namev)
} }
// prune all non-reachable objects (e.g. tags just pulled - they were encoded as commits) // prune all non-reachable objects (e.g. tags just pulled - they were encoded as commits)
xgit("prune") xgit("prune")
// verify backup repo is all ok // verify backup repo is all ok
xgit("fsck") xgit("fsck")
// verify that just pulled tag objects are now gone after pruning - // verify that just pulled tag objects are now gone after pruning -
// - they become not directly git-present. The only possibility to // - they become not directly git-present. The only possibility to
// get them back is via recreating from encoded commit objects. // get them back is via recreating from encoded commit objects.
for _, nc := range noncommitv { for _, nc := range noncommitv {
if !nc.istag { if !nc.istag {
continue continue
} }
gerr, _, _ := ggit("cat-file", "-p", nc.sha1) gerr, _, _ := ggit("cat-file", "-p", nc.sha1)
if gerr == nil { if gerr == nil {
t.Fatalf("tag %s still present in backup.git after git-prune", nc.sha1) t.Fatalf("tag %s still present in backup.git after git-prune", nc.sha1)
} }
} }
// reopen backup repository - to avoid having stale cache with present // reopen backup repository - to avoid having stale cache with present
// objects we deleted above with `git prune` // objects we deleted above with `git prune`
gb, err = git.OpenRepository(".") gb, err = git.OpenRepository(".")
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
} }
afterPull() afterPull()
// pull again - it should be noop // pull again - it should be noop
h1 := xgitSha1("rev-parse", "HEAD") h1 := xgitSha1("rev-parse", "HEAD")
cmd_pull(gb, []string{my1+":b1"}) cmd_pull(gb, []string{my1 + ":b1"})
afterPull() afterPull()
h2 := xgitSha1("rev-parse", "HEAD") h2 := xgitSha1("rev-parse", "HEAD")
if h1 == h2 { if h1 == h2 {
t.Fatal("pull: second run did not ajusted HEAD") t.Fatal("pull: second run did not ajusted HEAD")
} }
δ12 := xgit("diff", h1, h2) δ12 := xgit("diff", h1, h2)
if δ12 != "" { if δ12 != "" {
t.Fatalf("pull: second run was not noop: δ:\n%s", δ12) t.Fatalf("pull: second run was not noop: δ:\n%s", δ12)
} }
// restore backup // restore backup
work1 := workdir + "/1" work1 := workdir + "/1"
cmd_restore(gb, []string{"HEAD", "b1:"+work1}) cmd_restore(gb, []string{"HEAD", "b1:" + work1})
// verify files restored to the same as original // verify files restored to the same as original
gerr, diff, _ := ggit("diff", "--no-index", "--raw", "--exit-code", my1, work1) gerr, diff, _ := ggit("diff", "--no-index", "--raw", "--exit-code", my1, work1)
// 0 - no diff, 1 - has diff, 2 - problem // 0 - no diff, 1 - has diff, 2 - problem
if gerr != nil && gerr.Sys().(syscall.WaitStatus).ExitStatus() > 1 { if gerr != nil && gerr.Sys().(syscall.WaitStatus).ExitStatus() > 1 {
t.Fatal(gerr) t.Fatal(gerr)
} }
gitObjectsRe := regexp.MustCompile(`\.git/objects/`) gitObjectsRe := regexp.MustCompile(`\.git/objects/`)
for _, diffline := range strings.Split(diff, "\n") { for _, diffline := range strings.Split(diff, "\n") {
// :srcmode dstmode srcsha1 dstsha1 status\tpath // :srcmode dstmode srcsha1 dstsha1 status\tpath
_, path, err := xstrings.HeadTail(diffline, "\t") _, path, err := xstrings.HeadTail(diffline, "\t")
if err != nil { if err != nil {
t.Fatalf("restorecheck: cannot parse diff line %q", diffline) t.Fatalf("restorecheck: cannot parse diff line %q", diffline)
} }
// git objects can be represented differently (we check them later) // git objects can be represented differently (we check them later)
if gitObjectsRe.FindString(path) != "" { if gitObjectsRe.FindString(path) != "" {
continue continue
} }
t.Fatal("restorecheck: unexpected diff:", diffline) t.Fatal("restorecheck: unexpected diff:", diffline)
} }
// verify git objects restored to the same as original // verify git objects restored to the same as original
err = filepath.Walk(my1, func(path string, info os.FileInfo, err error) error { err = filepath.Walk(my1, func(path string, info os.FileInfo, err error) error {
// any error -> stop // any error -> stop
if err != nil { if err != nil {
return err return err
} }
// non *.git/ -- not interesting // non *.git/ -- not interesting
if !(info.IsDir() && strings.HasSuffix(path, ".git")) { if !(info.IsDir() && strings.HasSuffix(path, ".git")) {
return nil return nil
} }
// found git repo - check refs & objects in original and restored are exactly the same, // found git repo - check refs & objects in original and restored are exactly the same,
var R = [2]struct{ path, reflist, revlist string }{ var R = [2]struct{ path, reflist, revlist string }{
{path: path}, // original {path: path}, // original
{path: reprefix(my1, work1, path)}, // restored {path: reprefix(my1, work1, path)}, // restored
} }
for _, repo := range R { for _, repo := range R {
// fsck just in case // fsck just in case
xgit("--git-dir=" + repo.path, "fsck") xgit("--git-dir="+repo.path, "fsck")
// NOTE for-each-ref sorts output by refname // NOTE for-each-ref sorts output by refname
repo.reflist = xgit("--git-dir=" + repo.path, "for-each-ref") repo.reflist = xgit("--git-dir="+repo.path, "for-each-ref")
// NOTE rev-list emits objects in reverse chronological order, // NOTE rev-list emits objects in reverse chronological order,
// starting from refs roots which are also ordered by refname // starting from refs roots which are also ordered by refname
repo.revlist = xgit("--git-dir=" + repo.path, "rev-list", "--all", "--objects") repo.revlist = xgit("--git-dir="+repo.path, "rev-list", "--all", "--objects")
} }
if R[0].reflist != R[1].reflist { if R[0].reflist != R[1].reflist {
t.Fatalf("restorecheck: %q restored with different reflist (in %q)", R[0].path, R[1].path) t.Fatalf("restorecheck: %q restored with different reflist (in %q)", R[0].path, R[1].path)
} }
if R[0].revlist != R[1].revlist { if R[0].revlist != R[1].revlist {
t.Fatalf("restorecheck: %q restored with differrent objects (in %q)", R[0].path, R[1].path) t.Fatalf("restorecheck: %q restored with differrent objects (in %q)", R[0].path, R[1].path)
} }
// .git verified - no need to recurse // .git verified - no need to recurse
return filepath.SkipDir return filepath.SkipDir
}) })
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
// now try to pull corrupt repo - pull should refuse if transferred pack contains bad objects // now try to pull corrupt repo - pull should refuse if transferred pack contains bad objects
my2 := mydir + "/testdata/2" my2 := mydir + "/testdata/2"
func() { func() {
defer exc.Catch(func(e *exc.Error) { defer exc.Catch(func(e *exc.Error) {
// it ok - pull should raise // it ok - pull should raise
// git-backup leaves backup repo locked on error // git-backup leaves backup repo locked on error
xgit("update-ref", "-d", "refs/backup.locked") xgit("update-ref", "-d", "refs/backup.locked")
}) })
cmd_pull(gb, []string{my2+":b2"}) cmd_pull(gb, []string{my2 + ":b2"})
t.Fatal("pull corrupt.git: did not complain") t.Fatal("pull corrupt.git: did not complain")
}() }()
// now try to pull repo where `git pack-objects` misbehaves // now try to pull repo where `git pack-objects` misbehaves
my3 := mydir + "/testdata/3" my3 := mydir + "/testdata/3"
checkIncompletePack := func(kind, errExpect string) { checkIncompletePack := func(kind, errExpect string) {
defer exc.Catch(func(e *exc.Error) { defer exc.Catch(func(e *exc.Error) {
estr := e.Error() estr := e.Error()
bad := "" bad := ""
badf := func(format string, argv ...interface{}) { badf := func(format string, argv ...interface{}) {
bad += fmt.Sprintf(format+"\n", argv...) bad += fmt.Sprintf(format+"\n", argv...)
} }
if !strings.Contains(estr, errExpect) { if !strings.Contains(estr, errExpect) {
badf("- no %q", errExpect) badf("- no %q", errExpect)
} }
if bad != "" { if bad != "" {
t.Fatalf("pull incomplete-send-pack.git/%s: complained, but error is wrong:\n%s\nerror: %s", kind, bad, estr) t.Fatalf("pull incomplete-send-pack.git/%s: complained, but error is wrong:\n%s\nerror: %s", kind, bad, estr)
} }
// git-backup leaves backup repo locked on error // git-backup leaves backup repo locked on error
xgit("update-ref", "-d", "refs/backup.locked") xgit("update-ref", "-d", "refs/backup.locked")
}) })
// for incomplete-send-pack.git to indeed send incomplete pack, its git // for incomplete-send-pack.git to indeed send incomplete pack, its git
// config has to be activated via tweaked $HOME. // config has to be activated via tweaked $HOME.
home, ok := os.LookupEnv("HOME") home, ok := os.LookupEnv("HOME")
defer func() { defer func() {
if ok { if ok {
err = os.Setenv("HOME", home) err = os.Setenv("HOME", home)
} else { } else {
err = os.Unsetenv("HOME") err = os.Unsetenv("HOME")
} }
exc.Raiseif(err) exc.Raiseif(err)
}() }()
err = os.Setenv("HOME", my3+"/incomplete-send-pack.git/"+kind) err = os.Setenv("HOME", my3+"/incomplete-send-pack.git/"+kind)
exc.Raiseif(err) exc.Raiseif(err)
cmd_pull(gb, []string{my3+":b3"}) cmd_pull(gb, []string{my3 + ":b3"})
t.Fatalf("pull incomplete-send-pack.git/%s: did not complain", kind) t.Fatalf("pull incomplete-send-pack.git/%s: did not complain", kind)
} }
// missing blob: should be caught by git itself, because unpack-objects // missing blob: should be caught by git itself, because unpack-objects
// performs full reachability checks of fetched tips. // performs full reachability checks of fetched tips.
checkIncompletePack("x-missing-blob", "fatal: unpack-objects") checkIncompletePack("x-missing-blob", "fatal: unpack-objects")
// missing commit: remote sends a pack that is closed under reachability, // missing commit: remote sends a pack that is closed under reachability,
// but it has objects starting from only parent of requested tip. This way // but it has objects starting from only parent of requested tip. This way
// e.g. commit at tip itself is not sent and the fact that it is missing in // e.g. commit at tip itself is not sent and the fact that it is missing in
// the pack is not caught by fetch-pack. git-backup has to detect the // the pack is not caught by fetch-pack. git-backup has to detect the
// problem itself. // problem itself.
checkIncompletePack("x-commit-send-parent", "remote did not send all neccessary objects") checkIncompletePack("x-commit-send-parent", "remote did not send all neccessary objects")
// pulling incomplete-send-pack.git without pack-objects hook must succeed: // pulling incomplete-send-pack.git without pack-objects hook must succeed:
// without $HOME tweaks full and complete pack is sent. // without $HOME tweaks full and complete pack is sent.
cmd_pull(gb, []string{my3+":b3"}) cmd_pull(gb, []string{my3 + ":b3"})
} }
func TestRepoRefSplit(t *testing.T) { func TestRepoRefSplit(t *testing.T) {
var tests = []struct{ reporef, repo, ref string }{ var tests = []struct{ reporef, repo, ref string }{
{"kirr/wendelin.core.git/heads/master", "kirr/wendelin.core.git", "heads/master"}, {"kirr/wendelin.core.git/heads/master", "kirr/wendelin.core.git", "heads/master"},
{"kirr/erp5.git/backup/x/master+erp5-data-notebook", "kirr/erp5.git", "backup/x/master+erp5-data-notebook"}, {"kirr/erp5.git/backup/x/master+erp5-data-notebook", "kirr/erp5.git", "backup/x/master+erp5-data-notebook"},
{"tiwariayush/Discussion%20Forum%20.git/...", "tiwariayush/Discussion Forum .git", "..."}, {"tiwariayush/Discussion%20Forum%20.git/...", "tiwariayush/Discussion Forum .git", "..."},
{"tiwariayush/Discussion%20Forum+.git/...", "tiwariayush/Discussion Forum+.git", "..."}, {"tiwariayush/Discussion%20Forum+.git/...", "tiwariayush/Discussion Forum+.git", "..."},
{"tiwariayush/Discussion%2BForum+.git/...", "tiwariayush/Discussion+Forum+.git", "..."}, {"tiwariayush/Discussion%2BForum+.git/...", "tiwariayush/Discussion+Forum+.git", "..."},
} }
for _, tt := range tests { for _, tt := range tests {
repo, ref := reporef_split(tt.reporef) repo, ref := reporef_split(tt.reporef)
if repo != tt.repo || ref != tt.ref { if repo != tt.repo || ref != tt.ref {
t.Errorf("reporef_split(%q) -> %q %q ; want %q %q", tt.reporef, repo, ref, tt.repo, tt.ref) t.Errorf("reporef_split(%q) -> %q %q ; want %q %q", tt.reporef, repo, ref, tt.repo, tt.ref)
} }
} }
} }
git.go
View file @ f3f694b9
...@@ -21,142 +21,142 @@ package main ...@@ -21,142 +21,142 @@ package main
// Git-backup | Run git subprocess // Git-backup | Run git subprocess
import ( import (
"bytes" "bytes"
"fmt" "fmt"
"os" "os"
"os/exec" "os/exec"
"strings" "strings"
"lab.nexedi.com/kirr/go123/exc" "lab.nexedi.com/kirr/go123/exc"
"lab.nexedi.com/kirr/go123/mem" "lab.nexedi.com/kirr/go123/mem"
) )
// how/whether to redirect stdio of spawned process // how/whether to redirect stdio of spawned process
type StdioRedirect int type StdioRedirect int
const ( const (
PIPE StdioRedirect = iota // connect stdio channel via PIPE to parent (default value) PIPE StdioRedirect = iota // connect stdio channel via PIPE to parent (default value)
DontRedirect DontRedirect
) )
type RunWith struct { type RunWith struct {
stdin string stdin string
stdout StdioRedirect // PIPE | DontRedirect stdout StdioRedirect // PIPE | DontRedirect
stderr StdioRedirect // PIPE | DontRedirect stderr StdioRedirect // PIPE | DontRedirect
raw bool // !raw -> stdout, stderr are stripped raw bool // !raw -> stdout, stderr are stripped
env map[string]string // !nil -> subprocess environment setup from env env map[string]string // !nil -> subprocess environment setup from env
} }
// run `git *argv` -> error, stdout, stderr // run `git *argv` -> error, stdout, stderr
func _git(argv []string, ctx RunWith) (err error, stdout, stderr string) { func _git(argv []string, ctx RunWith) (err error, stdout, stderr string) {
debugf("git %s", strings.Join(argv, " ")) debugf("git %s", strings.Join(argv, " "))
cmd := exec.Command("git", argv...) cmd := exec.Command("git", argv...)
stdoutBuf := bytes.Buffer{} stdoutBuf := bytes.Buffer{}
stderrBuf := bytes.Buffer{} stderrBuf := bytes.Buffer{}
if ctx.stdin != "" { if ctx.stdin != "" {
cmd.Stdin = strings.NewReader(ctx.stdin) cmd.Stdin = strings.NewReader(ctx.stdin)
} }
switch ctx.stdout { switch ctx.stdout {
case PIPE: case PIPE:
cmd.Stdout = &stdoutBuf cmd.Stdout = &stdoutBuf
case DontRedirect: case DontRedirect:
cmd.Stdout = os.Stdout cmd.Stdout = os.Stdout
default: default:
panic("git: stdout redirect mode invalid") panic("git: stdout redirect mode invalid")
} }
switch ctx.stderr { switch ctx.stderr {
case PIPE: case PIPE:
cmd.Stderr = &stderrBuf cmd.Stderr = &stderrBuf
case DontRedirect: case DontRedirect:
cmd.Stderr = os.Stderr cmd.Stderr = os.Stderr
default: default:
panic("git: stderr redirect mode invalid") panic("git: stderr redirect mode invalid")
} }
if ctx.env != nil { if ctx.env != nil {
env := []string{} env := []string{}
for k, v := range ctx.env { for k, v := range ctx.env {
env = append(env, k+"="+v) env = append(env, k+"="+v)
} }
cmd.Env = env cmd.Env = env
} }
err = cmd.Run() err = cmd.Run()
stdout = mem.String(stdoutBuf.Bytes()) stdout = mem.String(stdoutBuf.Bytes())
stderr = mem.String(stderrBuf.Bytes()) stderr = mem.String(stderrBuf.Bytes())
if !ctx.raw { if !ctx.raw {
// prettify stdout (e.g. so that 'sha1\n' becomes 'sha1' and can be used directly // prettify stdout (e.g. so that 'sha1\n' becomes 'sha1' and can be used directly
stdout = strings.TrimSpace(stdout) stdout = strings.TrimSpace(stdout)
stderr = strings.TrimSpace(stderr) stderr = strings.TrimSpace(stderr)
} }
return err, stdout, stderr return err, stdout, stderr
} }
// error a git command returned // error a git command returned
type GitError struct { type GitError struct {
GitErrContext GitErrContext
*exec.ExitError *exec.ExitError
} }
type GitErrContext struct { type GitErrContext struct {
argv []string argv []string
stdin string stdin string
stdout string stdout string
stderr string stderr string
} }
func (e *GitError) Error() string { func (e *GitError) Error() string {
msg := e.GitErrContext.Error() msg := e.GitErrContext.Error()
if e.stderr == "" { if e.stderr == "" {
msg += "(failed)\n" msg += "(failed)\n"
} }
return msg return msg
} }
func (e *GitErrContext) Error() string { func (e *GitErrContext) Error() string {
msg := "git " + strings.Join(e.argv, " ") msg := "git " + strings.Join(e.argv, " ")
if e.stdin == "" { if e.stdin == "" {
msg += " </dev/null\n" msg += " </dev/null\n"
} else { } else {
msg += " <<EOF\n" + e.stdin msg += " <<EOF\n" + e.stdin
if !strings.HasSuffix(msg, "\n") { if !strings.HasSuffix(msg, "\n") {
msg += "\n" msg += "\n"
} }
msg += "EOF\n" msg += "EOF\n"
} }
msg += e.stderr msg += e.stderr
if !strings.HasSuffix(msg, "\n") { if !strings.HasSuffix(msg, "\n") {
msg += "\n" msg += "\n"
} }
return msg return msg
} }
// argv -> []string, ctx (for passing argv + RunWith handy - see ggit() for details) // argv -> []string, ctx (for passing argv + RunWith handy - see ggit() for details)
func _gitargv(argv ...interface{}) (argvs []string, ctx RunWith) { func _gitargv(argv ...interface{}) (argvs []string, ctx RunWith) {
ctx_seen := false ctx_seen := false
for _, arg := range argv { for _, arg := range argv {
switch arg := arg.(type) { switch arg := arg.(type) {
case string: case string:
argvs = append(argvs, arg) argvs = append(argvs, arg)
default: default:
argvs = append(argvs, fmt.Sprint(arg)) argvs = append(argvs, fmt.Sprint(arg))
case RunWith: case RunWith:
if ctx_seen { if ctx_seen {
panic("git: multiple RunWith contexts") panic("git: multiple RunWith contexts")
} }
ctx, ctx_seen = arg, true ctx, ctx_seen = arg, true
} }
} }
return argvs, ctx return argvs, ctx
} }
// run `git *argv` -> err, stdout, stderr // run `git *argv` -> err, stdout, stderr
...@@ -167,59 +167,59 @@ func _gitargv(argv ...interface{}) (argvs []string, ctx RunWith) { ...@@ -167,59 +167,59 @@ func _gitargv(argv ...interface{}) (argvs []string, ctx RunWith) {
// //
// NOTE err is concrete *GitError, not error // NOTE err is concrete *GitError, not error
func ggit(argv ...interface{}) (err *GitError, stdout, stderr string) { func ggit(argv ...interface{}) (err *GitError, stdout, stderr string) {
return ggit2(_gitargv(argv...)) return ggit2(_gitargv(argv...))
} }
func ggit2(argv []string, ctx RunWith) (err *GitError, stdout, stderr string) { func ggit2(argv []string, ctx RunWith) (err *GitError, stdout, stderr string) {
e, stdout, stderr := _git(argv, ctx) e, stdout, stderr := _git(argv, ctx)
eexec, _ := e.(*exec.ExitError) eexec, _ := e.(*exec.ExitError)
if e != nil && eexec == nil { if e != nil && eexec == nil {
exc.Raisef("git %s : ", strings.Join(argv, " "), e) exc.Raisef("git %s : ", strings.Join(argv, " "), e)
} }
if eexec != nil { if eexec != nil {
err = &GitError{GitErrContext{argv, ctx.stdin, stdout, stderr}, eexec} err = &GitError{GitErrContext{argv, ctx.stdin, stdout, stderr}, eexec}
} }
return err, stdout, stderr return err, stdout, stderr
} }
// run `git *argv` -> stdout // run `git *argv` -> stdout
// on error - raise exception // on error - raise exception
func xgit(argv ...interface{}) string { func xgit(argv ...interface{}) string {
return xgit2(_gitargv(argv...)) return xgit2(_gitargv(argv...))
} }
func xgit2(argv []string, ctx RunWith) string { func xgit2(argv []string, ctx RunWith) string {
gerr, stdout, _ := ggit2(argv, ctx) gerr, stdout, _ := ggit2(argv, ctx)
if gerr != nil { if gerr != nil {
exc.Raise(gerr) exc.Raise(gerr)
} }
return stdout return stdout
} }
// like xgit(), but automatically parse stdout to Sha1 // like xgit(), but automatically parse stdout to Sha1
func xgitSha1(argv ...interface{}) Sha1 { func xgitSha1(argv ...interface{}) Sha1 {
return xgit2Sha1(_gitargv(argv...)) return xgit2Sha1(_gitargv(argv...))
} }
// error when git output is not valid sha1 // error when git output is not valid sha1
type GitSha1Error struct { type GitSha1Error struct {
GitErrContext GitErrContext
} }
func (e *GitSha1Error) Error() string { func (e *GitSha1Error) Error() string {
msg := e.GitErrContext.Error() msg := e.GitErrContext.Error()
msg += fmt.Sprintf("expected valid sha1 (got %q)\n", e.stdout) msg += fmt.Sprintf("expected valid sha1 (got %q)\n", e.stdout)
return msg return msg
} }
func xgit2Sha1(argv []string, ctx RunWith) Sha1 { func xgit2Sha1(argv []string, ctx RunWith) Sha1 {
gerr, stdout, stderr := ggit2(argv, ctx) gerr, stdout, stderr := ggit2(argv, ctx)
if gerr != nil { if gerr != nil {
exc.Raise(gerr) exc.Raise(gerr)
} }
sha1, err := Sha1Parse(stdout) sha1, err := Sha1Parse(stdout)
if err != nil { if err != nil {
exc.Raise(&GitSha1Error{GitErrContext{argv, ctx.stdin, stdout, stderr}}) exc.Raise(&GitSha1Error{GitErrContext{argv, ctx.stdin, stdout, stderr}})
} }
return sha1 return sha1
} }
gitobjects.go
View file @ f3f694b9
...@@ -21,80 +21,80 @@ package main ...@@ -21,80 +21,80 @@ package main
// Git-backup | Git object: Blob Tree Commit Tag // Git-backup | Git object: Blob Tree Commit Tag
import ( import (
"errors" "errors"
"fmt" "fmt"
"os" "os"
"os/user" "os/user"
"sync" "sync"
"time" "time"
"lab.nexedi.com/kirr/go123/exc" "lab.nexedi.com/kirr/go123/exc"
"lab.nexedi.com/kirr/go123/mem" "lab.nexedi.com/kirr/go123/mem"
"lab.nexedi.com/kirr/go123/xstrings" "lab.nexedi.com/kirr/go123/xstrings"
git "github.com/libgit2/git2go" git "github.com/libgit2/git2go"
) )
// read/write raw objects // read/write raw objects
func ReadObject(g *git.Repository, sha1 Sha1, objtype git.ObjectType) (*git.OdbObject, error) { func ReadObject(g *git.Repository, sha1 Sha1, objtype git.ObjectType) (*git.OdbObject, error) {
obj, err := ReadObject2(g, sha1) obj, err := ReadObject2(g, sha1)
if err != nil { if err != nil {
return nil, err return nil, err
} }
if objtype != obj.Type() { if objtype != obj.Type() {
return nil, &UnexpectedObjType{obj, objtype} return nil, &UnexpectedObjType{obj, objtype}
} }
return obj, nil return obj, nil
} }
func ReadObject2(g *git.Repository, sha1 Sha1) (*git.OdbObject, error) { func ReadObject2(g *git.Repository, sha1 Sha1) (*git.OdbObject, error) {
odb, err := g.Odb() odb, err := g.Odb()
if err != nil { if err != nil {
return nil, &OdbNotReady{g, err} return nil, &OdbNotReady{g, err}
} }
obj, err := odb.Read(sha1.AsOid()) obj, err := odb.Read(sha1.AsOid())
if err != nil { if err != nil {
return nil, err return nil, err
} }
return obj, nil return obj, nil
} }
func WriteObject(g *git.Repository, content []byte, objtype git.ObjectType) (Sha1, error) { func WriteObject(g *git.Repository, content []byte, objtype git.ObjectType) (Sha1, error) {
odb, err := g.Odb() odb, err := g.Odb()
if err != nil { if err != nil {
return Sha1{}, &OdbNotReady{g, err} return Sha1{}, &OdbNotReady{g, err}
} }
oid, err := odb.Write(content, objtype) oid, err := odb.Write(content, objtype)
if err != nil { if err != nil {
// err is e.g. "Failed to create temporary file '.../objects/tmp_object_git2_G045iN': Permission denied" // err is e.g. "Failed to create temporary file '.../objects/tmp_object_git2_G045iN': Permission denied"
return Sha1{}, err return Sha1{}, err
} }
return Sha1FromOid(oid), nil return Sha1FromOid(oid), nil
} }
type OdbNotReady struct { type OdbNotReady struct {
g *git.Repository g *git.Repository
err error err error
} }
func (e *OdbNotReady) Error() string { func (e *OdbNotReady) Error() string {
return fmt.Sprintf("git(%q): odb not ready: %s", e.g.Path(), e.err) return fmt.Sprintf("git(%q): odb not ready: %s", e.g.Path(), e.err)
} }
type UnexpectedObjType struct { type UnexpectedObjType struct {
obj *git.OdbObject obj *git.OdbObject
wantType git.ObjectType wantType git.ObjectType
} }
func (e *UnexpectedObjType) Error() string { func (e *UnexpectedObjType) Error() string {
return fmt.Sprintf("%s: type is %s (expected %s)", e.obj.Id(), e.obj.Type(), e.wantType) return fmt.Sprintf("%s: type is %s (expected %s)", e.obj.Id(), e.obj.Type(), e.wantType)
} }
type Tag struct { type Tag struct {
tagged_type git.ObjectType tagged_type git.ObjectType
tagged_sha1 Sha1 tagged_sha1 Sha1
// TODO msg // TODO msg
} }
// load/parse Tag // load/parse Tag
...@@ -105,69 +105,69 @@ type Tag struct { ...@@ -105,69 +105,69 @@ type Tag struct {
// - we need to have tag_parse() -- a way to parse object from a buffer // - we need to have tag_parse() -- a way to parse object from a buffer
// (libgit2 does not provide such functionality at all) // (libgit2 does not provide such functionality at all)
func xload_tag(g *git.Repository, tag_sha1 Sha1) (tag *Tag, tag_obj *git.OdbObject) { func xload_tag(g *git.Repository, tag_sha1 Sha1) (tag *Tag, tag_obj *git.OdbObject) {
tag_obj, err := ReadObject(g, tag_sha1, git.ObjectTag) tag_obj, err := ReadObject(g, tag_sha1, git.ObjectTag)
exc.Raiseif(err) exc.Raiseif(err)
tag, err = tag_parse(mem.String(tag_obj.Data())) tag, err = tag_parse(mem.String(tag_obj.Data()))
if err != nil { if err != nil {
exc.Raise(&TagLoadError{tag_sha1, err}) exc.Raise(&TagLoadError{tag_sha1, err})
} }
return tag, tag_obj return tag, tag_obj
} }
type TagLoadError struct { type TagLoadError struct {
tag_sha1 Sha1 tag_sha1 Sha1
err error err error
} }
func (e *TagLoadError) Error() string { func (e *TagLoadError) Error() string {
return fmt.Sprintf("tag %s: %s", e.tag_sha1, e.err) return fmt.Sprintf("tag %s: %s", e.tag_sha1, e.err)
} }
func tag_parse(tag_raw string) (*Tag, error) { func tag_parse(tag_raw string) (*Tag, error) {
t := Tag{} t := Tag{}
tagged_type := "" tagged_type := ""
_, err := fmt.Sscanf(tag_raw, "object %s\ntype %s\n", &t.tagged_sha1, &tagged_type) _, err := fmt.Sscanf(tag_raw, "object %s\ntype %s\n", &t.tagged_sha1, &tagged_type)
if err != nil { if err != nil {
return nil, errors.New("invalid header") return nil, errors.New("invalid header")
} }
var ok bool var ok bool
t.tagged_type, ok = gittype(tagged_type) t.tagged_type, ok = gittype(tagged_type)
if !ok { if !ok {
return nil, fmt.Errorf("invalid tagged type %q", tagged_type) return nil, fmt.Errorf("invalid tagged type %q", tagged_type)
} }
return &t, nil return &t, nil
} }
// parse lstree entry // parse lstree entry
func parse_lstree_entry(lsentry string) (mode uint32, type_ string, sha1 Sha1, filename string, err error) { func parse_lstree_entry(lsentry string) (mode uint32, type_ string, sha1 Sha1, filename string, err error) {
// <mode> SP <type> SP <object> TAB <file> # NOTE file can contain spaces // <mode> SP <type> SP <object> TAB <file> # NOTE file can contain spaces
__, filename, err1 := xstrings.HeadTail(lsentry, "\t") __, filename, err1 := xstrings.HeadTail(lsentry, "\t")
_, err2 := fmt.Sscanf(__, "%o %s %s\n", &mode, &type_, &sha1) _, err2 := fmt.Sscanf(__, "%o %s %s\n", &mode, &type_, &sha1)
if err1 != nil || err2 != nil { if err1 != nil || err2 != nil {
return 0, "", Sha1{}, "", &InvalidLstreeEntry{lsentry} return 0, "", Sha1{}, "", &InvalidLstreeEntry{lsentry}
} }
// parsed ok // parsed ok
return return
} }
type InvalidLstreeEntry struct { type InvalidLstreeEntry struct {
lsentry string lsentry string
} }
func (e *InvalidLstreeEntry) Error() string { func (e *InvalidLstreeEntry) Error() string {
return fmt.Sprintf("invalid ls-tree entry %q", e.lsentry) return fmt.Sprintf("invalid ls-tree entry %q", e.lsentry)
} }
// create empty git tree -> tree sha1 // create empty git tree -> tree sha1
var tree_empty Sha1 var tree_empty Sha1
func mktree_empty() Sha1 { func mktree_empty() Sha1 {
if tree_empty.IsNull() { if tree_empty.IsNull() {
tree_empty = xgitSha1("mktree", RunWith{stdin: ""}) tree_empty = xgitSha1("mktree", RunWith{stdin: ""})
} }
return tree_empty return tree_empty
} }
// commit tree // commit tree
...@@ -178,84 +178,84 @@ func mktree_empty() Sha1 { ...@@ -178,84 +178,84 @@ func mktree_empty() Sha1 {
type AuthorInfo git.Signature type AuthorInfo git.Signature
func (ai *AuthorInfo) String() string { func (ai *AuthorInfo) String() string {
_, toffset := ai.When.Zone() _, toffset := ai.When.Zone()
// offset: Git wants in minutes, .Zone() gives in seconds // offset: Git wants in minutes, .Zone() gives in seconds
return fmt.Sprintf("%s <%s> %d %+05d", ai.Name, ai.Email, ai.When.Unix(), toffset / 60) return fmt.Sprintf("%s <%s> %d %+05d", ai.Name, ai.Email, ai.When.Unix(), toffset/60)
} }
var ( var (
defaultIdent AuthorInfo // default ident without date defaultIdent AuthorInfo // default ident without date
defaultIdentOnce sync.Once defaultIdentOnce sync.Once
) )
func getDefaultIdent(g *git.Repository) AuthorInfo { func getDefaultIdent(g *git.Repository) AuthorInfo {
sig, err := g.DefaultSignature() sig, err := g.DefaultSignature()
if err == nil { if err == nil {
return AuthorInfo(*sig) return AuthorInfo(*sig)
} }
// libgit2 failed for some reason (i.e. user.name config not set). Let's cook ident ourselves // libgit2 failed for some reason (i.e. user.name config not set). Let's cook ident ourselves
defaultIdentOnce.Do(func() { defaultIdentOnce.Do(func() {
var username, name string var username, name string
u, _ := user.Current() u, _ := user.Current()
if u != nil { if u != nil {
username = u.Username username = u.Username
name = u.Name name = u.Name
} else { } else {
username = "?" username = "?"
name = "?" name = "?"
} }
// XXX it is better to get hostname as fqdn // XXX it is better to get hostname as fqdn
hostname, _ := os.Hostname() hostname, _ := os.Hostname()
if hostname == "" { if hostname == "" {
hostname = "?" hostname = "?"
} }
defaultIdent.Name = name defaultIdent.Name = name
defaultIdent.Email = fmt.Sprintf("%s@%s", username, hostname) defaultIdent.Email = fmt.Sprintf("%s@%s", username, hostname)
}) })
ident := defaultIdent ident := defaultIdent
ident.When = time.Now() ident.When = time.Now()
return ident return ident
} }
// mkref creates a git reference. // mkref creates a git reference.
// //
// it is an error if the reference already exists. // it is an error if the reference already exists.
func mkref(g *git.Repository, name string, sha1 Sha1) error { func mkref(g *git.Repository, name string, sha1 Sha1) error {
_, err := g.References.Create(name, sha1.AsOid(), false, "") _, err := g.References.Create(name, sha1.AsOid(), false, "")
return err return err
} }
// `git commit-tree` -> commit_sha1, raise on error // `git commit-tree` -> commit_sha1, raise on error
func xcommit_tree2(g *git.Repository, tree Sha1, parents []Sha1, msg string, author AuthorInfo, committer AuthorInfo) Sha1 { func xcommit_tree2(g *git.Repository, tree Sha1, parents []Sha1, msg string, author AuthorInfo, committer AuthorInfo) Sha1 {
ident := getDefaultIdent(g) ident := getDefaultIdent(g)
if author.Name == "" { author.Name = ident.Name } if author.Name == "" { author.Name = ident.Name }
if author.Email == "" { author.Email = ident.Email } if author.Email == "" { author.Email = ident.Email }
if author.When.IsZero() { author.When = ident.When } if author.When.IsZero() { author.When = ident.When }
if committer.Name == "" { committer.Name = ident.Name } if committer.Name == "" { committer.Name = ident.Name }
if committer.Email == "" { committer.Email = ident.Email } if committer.Email == "" { committer.Email = ident.Email }
if committer.When.IsZero() { committer.When = ident.When } if committer.When.IsZero() { committer.When = ident.When }
commit := fmt.Sprintf("tree %s\n", tree) commit := fmt.Sprintf("tree %s\n", tree)
for _, p := range parents { for _, p := range parents {
commit += fmt.Sprintf("parent %s\n", p) commit += fmt.Sprintf("parent %s\n", p)
} }
commit += fmt.Sprintf("author %s\n", &author) commit += fmt.Sprintf("author %s\n", &author)
commit += fmt.Sprintf("committer %s\n", &committer) commit += fmt.Sprintf("committer %s\n", &committer)
commit += fmt.Sprintf("\n%s", msg) commit += fmt.Sprintf("\n%s", msg)
sha1, err := WriteObject(g, mem.Bytes(commit), git.ObjectCommit) sha1, err := WriteObject(g, mem.Bytes(commit), git.ObjectCommit)
exc.Raiseif(err) exc.Raiseif(err)
return sha1 return sha1
} }
func xcommit_tree(g *git.Repository, tree Sha1, parents []Sha1, msg string) Sha1 { func xcommit_tree(g *git.Repository, tree Sha1, parents []Sha1, msg string) Sha1 {
return xcommit_tree2(g, tree, parents, msg, AuthorInfo{}, AuthorInfo{}) return xcommit_tree2(g, tree, parents, msg, AuthorInfo{}, AuthorInfo{})
} }
...@@ -263,14 +263,14 @@ func xcommit_tree(g *git.Repository, tree Sha1, parents []Sha1, msg string) Sha1 ...@@ -263,14 +263,14 @@ func xcommit_tree(g *git.Repository, tree Sha1, parents []Sha1, msg string) Sha1
// //
// Only valid concrete git types are converted successfully. // Only valid concrete git types are converted successfully.
func gittype(typ string) (git.ObjectType, bool) { func gittype(typ string) (git.ObjectType, bool) {
switch typ { switch typ {
case "commit": return git.ObjectCommit, true case "commit": return git.ObjectCommit, true
case "tree": return git.ObjectTree, true case "tree": return git.ObjectTree, true
case "blob": return git.ObjectBlob, true case "blob": return git.ObjectBlob, true
case "tag": return git.ObjectTag, true case "tag": return git.ObjectTag, true
} }
return git.ObjectBad, false return git.ObjectBad, false
} }
...@@ -282,12 +282,12 @@ func gittype(typ string) (git.ObjectType, bool) { ...@@ -282,12 +282,12 @@ func gittype(typ string) (git.ObjectType, bool) {
// //
// gittypestr expects the type to be valid and concrete - else it panics. // gittypestr expects the type to be valid and concrete - else it panics.
func gittypestr(typ git.ObjectType) string { func gittypestr(typ git.ObjectType) string {
switch typ { switch typ {
case git.ObjectCommit: return "commit" case git.ObjectCommit: return "commit"
case git.ObjectTree: return "tree" case git.ObjectTree: return "tree"
case git.ObjectBlob: return "blob" case git.ObjectBlob: return "blob"
case git.ObjectTag: return "tag" case git.ObjectTag: return "tag"
} }
panic(fmt.Sprintf("git type %#v invalid", typ)) panic(fmt.Sprintf("git type %#v invalid", typ))
} }
set.go
View file @ f3f694b9
...@@ -25,44 +25,44 @@ package main ...@@ -25,44 +25,44 @@ package main
type Sha1Set map[Sha1]struct{} type Sha1Set map[Sha1]struct{}
func (s Sha1Set) Add(v Sha1) { func (s Sha1Set) Add(v Sha1) {
s[v] = struct{}{} s[v] = struct{}{}
} }
func (s Sha1Set) Contains(v Sha1) bool { func (s Sha1Set) Contains(v Sha1) bool {
_, ok := s[v] _, ok := s[v]
return ok return ok
} }
// all elements of set as slice // all elements of set as slice
func (s Sha1Set) Elements() []Sha1 { func (s Sha1Set) Elements() []Sha1 {
ev := make([]Sha1, len(s)) ev := make([]Sha1, len(s))
i := 0 i := 0
for e := range s { for e := range s {
ev[i] = e ev[i] = e
i++ i++
} }
return ev return ev
} }
// Set<string> // Set<string>
type StrSet map[string]struct{} type StrSet map[string]struct{}
func (s StrSet) Add(v string) { func (s StrSet) Add(v string) {
s[v] = struct{}{} s[v] = struct{}{}
} }
func (s StrSet) Contains(v string) bool { func (s StrSet) Contains(v string) bool {
_, ok := s[v] _, ok := s[v]
return ok return ok
} }
// all elements of set as slice // all elements of set as slice
func (s StrSet) Elements() []string { func (s StrSet) Elements() []string {
ev := make([]string, len(s)) ev := make([]string, len(s))
i := 0 i := 0
for e := range s { for e := range s {
ev[i] = e ev[i] = e
i++ i++
} }
return ev return ev
} }
sha1.go
View file @ f3f694b9
...@@ -21,13 +21,13 @@ package main ...@@ -21,13 +21,13 @@ package main
// Git-backup | Sha1 type to work with SHA1 oids // Git-backup | Sha1 type to work with SHA1 oids
import ( import (
"bytes" "bytes"
"encoding/hex" "encoding/hex"
"fmt" "fmt"
"lab.nexedi.com/kirr/go123/mem" "lab.nexedi.com/kirr/go123/mem"
git "github.com/libgit2/git2go" git "github.com/libgit2/git2go"
) )
const SHA1_RAWSIZE = 20 const SHA1_RAWSIZE = 20
...@@ -39,51 +39,51 @@ const SHA1_RAWSIZE = 20 ...@@ -39,51 +39,51 @@ const SHA1_RAWSIZE = 20
// - slice size = 24 bytes // - slice size = 24 bytes
// -> so it is reasonable to pass Sha1 not by reference // -> so it is reasonable to pass Sha1 not by reference
type Sha1 struct { type Sha1 struct {
sha1 [SHA1_RAWSIZE]byte sha1 [SHA1_RAWSIZE]byte
} }
// fmt.Stringer // fmt.Stringer
var _ fmt.Stringer = Sha1{} var _ fmt.Stringer = Sha1{}
func (sha1 Sha1) String() string { func (sha1 Sha1) String() string {
return hex.EncodeToString(sha1.sha1[:]) return hex.EncodeToString(sha1.sha1[:])
} }
func Sha1Parse(sha1str string) (Sha1, error) { func Sha1Parse(sha1str string) (Sha1, error) {
sha1 := Sha1{} sha1 := Sha1{}
if hex.DecodedLen(len(sha1str)) != SHA1_RAWSIZE { if hex.DecodedLen(len(sha1str)) != SHA1_RAWSIZE {
return Sha1{}, fmt.Errorf("sha1parse: %q invalid", sha1str) return Sha1{}, fmt.Errorf("sha1parse: %q invalid", sha1str)
} }
_, err := hex.Decode(sha1.sha1[:], mem.Bytes(sha1str)) _, err := hex.Decode(sha1.sha1[:], mem.Bytes(sha1str))
if err != nil { if err != nil {
return Sha1{}, fmt.Errorf("sha1parse: %q invalid: %s", sha1str, err) return Sha1{}, fmt.Errorf("sha1parse: %q invalid: %s", sha1str, err)
} }
return sha1, nil return sha1, nil
} }
// fmt.Scanner // fmt.Scanner
var _ fmt.Scanner = (*Sha1)(nil) var _ fmt.Scanner = (*Sha1)(nil)
func (sha1 *Sha1) Scan(s fmt.ScanState, ch rune) error { func (sha1 *Sha1) Scan(s fmt.ScanState, ch rune) error {
switch ch { switch ch {
case 's', 'v': case 's', 'v':
default: default:
return fmt.Errorf("Sha1.Scan: invalid verb %q", ch) return fmt.Errorf("Sha1.Scan: invalid verb %q", ch)
} }
tok, err := s.Token(true, nil) tok, err := s.Token(true, nil)
if err != nil { if err != nil {
return err return err
} }
*sha1, err = Sha1Parse(mem.String(tok)) *sha1, err = Sha1Parse(mem.String(tok))
return err return err
} }
// check whether sha1 is null // check whether sha1 is null
func (sha1 *Sha1) IsNull() bool { func (sha1 *Sha1) IsNull() bool {
return *sha1 == Sha1{} return *sha1 == Sha1{}
} }
// for sorting by Sha1 // for sorting by Sha1
...@@ -95,9 +95,9 @@ func (p BySha1) Less(i, j int) bool { return bytes.Compare(p[i].sha1[:], p[j].sh ...@@ -95,9 +95,9 @@ func (p BySha1) Less(i, j int) bool { return bytes.Compare(p[i].sha1[:], p[j].sh
// interoperability with git2go // interoperability with git2go
func (sha1 *Sha1) AsOid() *git.Oid { func (sha1 *Sha1) AsOid() *git.Oid {
return (*git.Oid)(&sha1.sha1) return (*git.Oid)(&sha1.sha1)
} }
func Sha1FromOid(oid *git.Oid) Sha1 { func Sha1FromOid(oid *git.Oid) Sha1 {
return Sha1{*oid} return Sha1{*oid}
} }
util.go
View file @ f3f694b9
...@@ -21,161 +21,161 @@ package main ...@@ -21,161 +21,161 @@ package main
// Git-backup | Miscellaneous utilities // Git-backup | Miscellaneous utilities
import ( import (
"encoding/hex" "encoding/hex"
"fmt" "fmt"
"os" "os"
"strings" "strings"
"syscall" "syscall"
"unicode" "unicode"
"unicode/utf8" "unicode/utf8"
"lab.nexedi.com/kirr/go123/mem" "lab.nexedi.com/kirr/go123/mem"
) )
// strip_prefix("/a/b", "/a/b/c/d/e") -> "c/d/e" (without leading /) // strip_prefix("/a/b", "/a/b/c/d/e") -> "c/d/e" (without leading /)
// path must start with prefix // path must start with prefix
func strip_prefix(prefix, path string) string { func strip_prefix(prefix, path string) string {
if !strings.HasPrefix(path, prefix) { if !strings.HasPrefix(path, prefix) {
panic(fmt.Errorf("strip_prefix: %q has no prefix %q", path, prefix)) panic(fmt.Errorf("strip_prefix: %q has no prefix %q", path, prefix))
} }
path = path[len(prefix):] path = path[len(prefix):]
for strings.HasPrefix(path, "/") { for strings.HasPrefix(path, "/") {
path = path[1:] // strip leading / path = path[1:] // strip leading /
} }
return path return path
} }
// reprefix("/a", "/b", "/a/str") -> "/b/str" // reprefix("/a", "/b", "/a/str") -> "/b/str"
// path must start with prefix_from // path must start with prefix_from
func reprefix(prefix_from, prefix_to, path string) string { func reprefix(prefix_from, prefix_to, path string) string {
path = strip_prefix(prefix_from, path) path = strip_prefix(prefix_from, path)
return fmt.Sprintf("%s/%s", prefix_to, path) return fmt.Sprintf("%s/%s", prefix_to, path)
} }
// like ioutil.WriteFile() but takes native mode/perm // like ioutil.WriteFile() but takes native mode/perm
func writefile(path string, data []byte, perm uint32) error { func writefile(path string, data []byte, perm uint32) error {
fd, err := syscall.Open(path, syscall.O_WRONLY|syscall.O_CREAT|syscall.O_TRUNC, perm) fd, err := syscall.Open(path, syscall.O_WRONLY|syscall.O_CREAT|syscall.O_TRUNC, perm)
if err != nil { if err != nil {
return &os.PathError{"open", path, err} return &os.PathError{"open", path, err}
} }
f := os.NewFile(uintptr(fd), path) f := os.NewFile(uintptr(fd), path)
_, err = f.Write(data) _, err = f.Write(data)
err2 := f.Close() err2 := f.Close()
if err == nil { if err == nil {
err = err2 err = err2
} }
return err return err
} }
// escape path so that git is happy to use it as ref // escape path so that git is happy to use it as ref
// https://git.kernel.org/cgit/git/git.git/tree/refs.c?h=v2.9.0-37-g6d523a3#n34 // https://git.kernel.org/cgit/git/git.git/tree/refs.c?h=v2.9.0-37-g6d523a3#n34
// XXX very suboptimal // XXX very suboptimal
func path_refescape(path string) string { func path_refescape(path string) string {
outv := []string{} outv := []string{}
for _, component := range strings.Split(path, "/") { for _, component := range strings.Split(path, "/") {
out := "" out := ""
dots := 0 // number of seen consecutive dots dots := 0 // number of seen consecutive dots
for len(component) > 0 { for len(component) > 0 {
r, size := utf8.DecodeRuneInString(component) r, size := utf8.DecodeRuneInString(component)
// no ".." anywhere - we replace dots run to %46%46... with trailing "." // no ".." anywhere - we replace dots run to %46%46... with trailing "."
// this way for single "." case we'll have it intact and avoid .. anywhere // this way for single "." case we'll have it intact and avoid .. anywhere
// also this way: trailing .git is always encoded as ".git" // also this way: trailing .git is always encoded as ".git"
if r == '.' { if r == '.' {
dots += 1 dots += 1
component = component[size:] component = component[size:]
continue continue
} }
if dots != 0 { if dots != 0 {
out += strings.Repeat(escape("."), dots-1) out += strings.Repeat(escape("."), dots-1)
out += "." out += "."
dots = 0 dots = 0
} }
rbytes := component[:size] rbytes := component[:size]
if shouldEscape(r) { if shouldEscape(r) {
rbytes = escape(rbytes) rbytes = escape(rbytes)
} }
out += rbytes out += rbytes
component = component[size:] component = component[size:]
} }
// handle trailing dots // handle trailing dots
if dots != 0 { if dots != 0 {
out += strings.Repeat(escape("."), dots-1) out += strings.Repeat(escape("."), dots-1)
out += "." out += "."
} }
if len(out) > 0 { if len(out) > 0 {
// ^. not allowed // ^. not allowed
if out[0] == '.' { if out[0] == '.' {
out = escape(".") + out[1:] out = escape(".") + out[1:]
} }
// .lock$ not allowed // .lock$ not allowed
if strings.HasSuffix(out, ".lock") { if strings.HasSuffix(out, ".lock") {
out = out[:len(out)-5] + escape(".") + "lock" out = out[:len(out)-5] + escape(".") + "lock"
} }
} }
outv = append(outv, out) outv = append(outv, out)
} }
// strip trailing / // strip trailing /
for len(outv) > 0 { for len(outv) > 0 {
if len(outv[len(outv)-1]) != 0 { if len(outv[len(outv)-1]) != 0 {
break break
} }
outv = outv[:len(outv)-1] outv = outv[:len(outv)-1]
} }
return strings.Join(outv, "/") return strings.Join(outv, "/")
} }
func shouldEscape(r rune) bool { func shouldEscape(r rune) bool {
if unicode.IsSpace(r) || unicode.IsControl(r) { if unicode.IsSpace(r) || unicode.IsControl(r) {
return true return true
} }
switch r { switch r {
// NOTE RuneError is for always escaping non-valid UTF-8 // NOTE RuneError is for always escaping non-valid UTF-8
case ':', '?', '[', '\\', '^', '~', '*', '@', '%', utf8.RuneError: case ':', '?', '[', '\\', '^', '~', '*', '@', '%', utf8.RuneError:
return true return true
} }
return false return false
} }
func escape(s string) string { func escape(s string) string {
out := "" out := ""
for i := 0; i < len(s); i++ { for i := 0; i < len(s); i++ {
out += fmt.Sprintf("%%%02X", s[i]) out += fmt.Sprintf("%%%02X", s[i])
} }
return out return out
} }
// unescape path encoded by path_refescape() // unescape path encoded by path_refescape()
// decoding is permissive - any byte can be %-encoded, not only special cases // decoding is permissive - any byte can be %-encoded, not only special cases
// XXX very suboptimal // XXX very suboptimal
func path_refunescape(s string) (string, error) { func path_refunescape(s string) (string, error) {
l := len(s) l := len(s)
out := make([]byte, 0, len(s)) out := make([]byte, 0, len(s))
for i := 0; i < l; i++ { for i := 0; i < l; i++ {
c := s[i] c := s[i]
if c == '%' { if c == '%' {
if i+2 >= l { if i+2 >= l {
return "", EscapeError(s) return "", EscapeError(s)
} }
b, err := hex.DecodeString(s[i+1:i+3]) b, err := hex.DecodeString(s[i+1 : i+3])
if err != nil { if err != nil {
return "", EscapeError(s) return "", EscapeError(s)
} }
c = b[0] c = b[0]
i += 2 i += 2
} }
out = append(out, c) out = append(out, c)
} }
return mem.String(out), nil return mem.String(out), nil
} }
type EscapeError string type EscapeError string
func (e EscapeError) Error() string { func (e EscapeError) Error() string {
return fmt.Sprintf("%q: invalid escape format", string(e)) return fmt.Sprintf("%q: invalid escape format", string(e))
} }
util_test.go
View file @ f3f694b9
...@@ -20,81 +20,81 @@ ...@@ -20,81 +20,81 @@
package main package main
import ( import (
"strings" "strings"
"testing" "testing"
) )
func TestPathEscapeUnescape(t *testing.T) { func TestPathEscapeUnescape(t *testing.T) {
type TestEntry struct { path string; escapedv []string } type TestEntry struct { path string; escapedv []string }
te := func(path string, escaped ...string) TestEntry { te := func(path string, escaped ...string) TestEntry {
return TestEntry{path, escaped} return TestEntry{path, escaped}
} }
var tests = []TestEntry{ var tests = []TestEntry{
// path escaped non-canonical escapes // path escaped non-canonical escapes
te("hello/world", "hello/world", "%68%65%6c%6c%6f%2f%77%6f%72%6c%64"), te("hello/world", "hello/world", "%68%65%6c%6c%6f%2f%77%6f%72%6c%64"),
te("hello/мир", "hello/мир"), te("hello/мир", "hello/мир"),
te("hello/ мир", "hello/%20мир"), te("hello/ мир", "hello/%20мир"),
te("hel%lo/мир", "hel%25lo/мир"), te("hel%lo/мир", "hel%25lo/мир"),
te(".hello/.world", "%2Ehello/%2Eworld"), te(".hello/.world", "%2Ehello/%2Eworld"),
te("..hello/world.loc", "%2E.hello/world.loc"), te("..hello/world.loc", "%2E.hello/world.loc"),
te("..hello/world.lock", "%2E.hello/world%2Elock"), te("..hello/world.lock", "%2E.hello/world%2Elock"),
// leading / // leading /
te("/hello/world", "/hello/world"), te("/hello/world", "/hello/world"),
te("//hello///world", "//hello///world"), te("//hello///world", "//hello///world"),
// trailing / // trailing /
te("/hello/world/", "/hello/world"), te("/hello/world/", "/hello/world"),
te("/hello/world//", "/hello/world"), te("/hello/world//", "/hello/world"),
// trailing ... // trailing ...
te("/hello/world.", "/hello/world."), te("/hello/world.", "/hello/world."),
te("/hello/world..", "/hello/world%2E."), te("/hello/world..", "/hello/world%2E."),
te("/hello/world...", "/hello/world%2E%2E."), te("/hello/world...", "/hello/world%2E%2E."),
te("/hello/world...git", "/hello/world%2E%2E.git"), te("/hello/world...git", "/hello/world%2E%2E.git"),
// .. anywhere // .. anywhere
te("/hello/./world", "/hello/%2E/world"), te("/hello/./world", "/hello/%2E/world"),
te("/hello/.a/world", "/hello/%2Ea/world"), te("/hello/.a/world", "/hello/%2Ea/world"),
te("/hello/a./world", "/hello/a./world"), te("/hello/a./world", "/hello/a./world"),
te("/hello/../world", "/hello/%2E./world"), te("/hello/../world", "/hello/%2E./world"),
te("/hello/a..b/world", "/hello/a%2E.b/world"), te("/hello/a..b/world", "/hello/a%2E.b/world"),
te("/hello/a.c.b/world", "/hello/a.c.b/world"), te("/hello/a.c.b/world", "/hello/a.c.b/world"),
te("/hello/a.c..b/world", "/hello/a.c%2E.b/world"), te("/hello/a.c..b/world", "/hello/a.c%2E.b/world"),
// special & control characters // special & control characters
te("/hel lo/wor\tld/a:?[\\^~*@%b/\001\004\n\xc2\xa0", "/hel%20lo/wor%09ld/a%3A%3F%5B%5C%5E%7E%2A%40%25b/%01%04%0A%C2%A0"), te("/hel lo/wor\tld/a:?[\\^~*@%b/\001\004\n\xc2\xa0", "/hel%20lo/wor%09ld/a%3A%3F%5B%5C%5E%7E%2A%40%25b/%01%04%0A%C2%A0"),
// utf8 error // utf8 error
te("a\xc5z", "a%C5z"), te("a\xc5z", "a%C5z"),
} }
for _, tt := range tests { for _, tt := range tests {
escaped := path_refescape(tt.path) escaped := path_refescape(tt.path)
if escaped != tt.escapedv[0] { if escaped != tt.escapedv[0] {
t.Errorf("path_refescape(%q) -> %q ; want %q", tt.path, escaped, tt.escapedv[0]) t.Errorf("path_refescape(%q) -> %q ; want %q", tt.path, escaped, tt.escapedv[0])
} }
// also check the decoding // also check the decoding
pathok := strings.TrimRight(tt.path, "/") pathok := strings.TrimRight(tt.path, "/")
for _, escaped := range tt.escapedv { for _, escaped := range tt.escapedv {
unescaped, err := path_refunescape(escaped) unescaped, err := path_refunescape(escaped)
if unescaped != pathok || err != nil { if unescaped != pathok || err != nil {
t.Errorf("path_refunescape(%q) -> %q %v ; want %q nil", escaped, unescaped, err, tt.path) t.Errorf("path_refunescape(%q) -> %q %v ; want %q nil", escaped, unescaped, err, tt.path)
} }
} }
} }
} }
func TestPathUnescapeErr(t *testing.T) { func TestPathUnescapeErr(t *testing.T) {
var tests = []struct{ escaped string }{ var tests = []struct{ escaped string }{
{"%"}, {"%"},
{"%2"}, {"%2"},
{"%2q"}, {"%2q"},
{"hell%2q/world"}, {"hell%2q/world"},
} }
for _, tt := range tests { for _, tt := range tests {
unescaped, err := path_refunescape(tt.escaped) unescaped, err := path_refunescape(tt.escaped)
if err == nil || unescaped != "" { if err == nil || unescaped != "" {
t.Errorf("path_refunescape(%q) -> %q %v ; want \"\" err", tt.escaped, unescaped, err) t.Errorf("path_refunescape(%q) -> %q %v ; want \"\" err", tt.escaped, unescaped, err)
} }
} }
} }
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7