Add splice library from termite.

splice/copy.go

+package splice
+
+import (
+	"io"
+	"os"
+)
+
+func SpliceCopy(dst *os.File, src *os.File, p *Pair) (int64, error) {
+	total := int64(0)
+
+	for {
+		n, err := p.LoadFrom(src.Fd(), p.size)
+		if err != nil {
+			return total, err
+		}
+		if n == 0 {
+			break
+		}
+		m, err := p.WriteTo(dst.Fd(), n)
+		total += int64(m)
+		if err != nil {
+			return total, err
+		}
+		if m < n {
+			panic("m<n")
+		}
+		if int(n) < p.size {
+			break
+		}
+	}
+
+	return total, nil
+}
+
+// Argument ordering follows io.Copy.
+func CopyFile(dstName string, srcName string, mode int) error {
+	src, err := os.Open(srcName)
+	if err != nil {
+		return err
+	}
+	defer src.Close()
+
+	dst, err := os.OpenFile(dstName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, os.FileMode(mode))
+	if err != nil {
+		return err
+	}
+	defer dst.Close()
+
+	return CopyFds(dst, src)
+}
+
+func CopyFds(dst *os.File, src *os.File) (err error) {
+	p, err := splicePool.get()
+	if p != nil {
+		p.Grow(256 * 1024)
+		_, err := SpliceCopy(dst, src, p)
+		splicePool.done(p)
+		return err
+	} else {
+		_, err = io.Copy(dst, src)
+	}
+	if err == io.EOF {
+		err = nil
+	}
+	return err
+}

splice/copy_test.go

+package splice
+
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+)
+
+func check(err error) {
+	if err != nil {
+		panic(err)
+	}
+}
+
+func TestCopyFile(t *testing.T) {
+	src, _ := ioutil.TempFile("", "termite")
+	err := ioutil.WriteFile(src.Name(), []byte("hello"), 0644)
+	if err != nil {
+		t.Error(err)
+	}
+	dst, _ := ioutil.TempFile("", "termite")
+	err = CopyFile(dst.Name(), src.Name(), 0755)
+	if err != nil {
+		t.Error(err)
+	}
+
+	c, err := ioutil.ReadFile(dst.Name())
+	if err != nil {
+		t.Error(err)
+	}
+	if string(c) != "hello" {
+		t.Error("mismatch", string(c))
+	}
+}
+
+func TestSpliceCopy(t *testing.T) {
+	src, err := ioutil.TempFile("", "termite")
+	check(err)
+	bs := make([]byte, 2*1024*1024)
+	for i := range bs {
+		bs[i] = byte(i % 256)
+	}
+	_, err = src.Write(bs)
+	check(err)
+	err = src.Close()
+	check(err)
+	src, err = os.Open(src.Name())
+	check(err)
+	dst, err := ioutil.TempFile("", "termite")
+	check(err)
+
+	if maxPipeSize%4096 != 0 || maxPipeSize < 4096 {
+		t.Error("pipe size should be page size multiple", maxPipeSize)
+	}
+	pool := newSplicePairPool()
+	p, err := pool.get()
+	if p != nil {
+		p.MaxGrow()
+		t.Logf("Splice size %d", p.size)
+		SpliceCopy(dst, src, p)
+		dst.Close()
+		src.Close()
+		p.Close()
+	} else {
+		t.Error("Could not open splice: ", err)
+	}
+}

splice/pair.go

+package splice
+
+import (
+	"fmt"
+	"os"
+	"syscall"
+)
+
+type Pair struct {
+	r, w *os.File
+	size int
+}
+
+func (p *Pair) MaxGrow() {
+	for p.Grow(2 * p.size) {
+	}
+}
+
+func (p *Pair) Grow(n int) bool {
+	if !resizable {
+		return false
+	}
+	if n > maxPipeSize {
+		return false
+	}
+	if n <= p.size {
+		return true
+	}
+
+	newsize, errNo := fcntl(p.r.Fd(), F_SETPIPE_SZ, n)
+	if errNo != 0 {
+		return false
+	}
+	p.size = newsize
+	return true
+}
+
+func (p *Pair) Cap() int {
+	return p.size
+}
+
+func (p *Pair) Close() error {
+	err1 := p.r.Close()
+	err2 := p.w.Close()
+	if err1 != nil {
+		return err1
+	}
+	return err2
+}
+
+func (p *Pair) Read(d []byte) (n int, err error) {
+	return p.r.Read(d)
+}
+
+func (p *Pair) Write(d []byte) (n int, err error) {
+	return p.w.Write(d)
+}
+
+func (p *Pair) LoadFromAt(fd uintptr, sz int, off int64) (int, error) {
+	n, err := syscall.Splice(int(fd), &off, int(p.w.Fd()), nil, sz, 0)
+	return int(n), err
+}
+
+func (p *Pair) LoadFrom(fd uintptr, sz int) (int, error) {
+	if sz > p.size {
+		return 0, fmt.Errorf("LoadFrom: not enough space %d, %d",
+			sz, p.size)
+	}
+
+	n, err := syscall.Splice(int(fd), nil, int(p.w.Fd()), nil, sz, 0)
+	if err != nil {
+		err = os.NewSyscallError("Splice load from", err)
+	}
+	return int(n), err
+}
+
+func (p *Pair) WriteTo(fd uintptr, n int) (int, error) {
+	m, err := syscall.Splice(int(p.r.Fd()), nil, int(fd), nil, int(n), 0)
+	if err != nil {
+		err = os.NewSyscallError("Splice write to: ", err)
+	}
+	return int(m), err
+}

splice/pool.go

+package splice
+import (
+	"sync"
+)
+
+var splicePool *pairPool
+
+type pairPool struct {
+	sync.Mutex
+	unused    []*Pair
+	usedCount int
+}
+
+func ClearSplicePool() {
+	splicePool.clear()
+}
+
+func Get() (*Pair, error) {
+	return splicePool.get()
+}
+
+func Used() int {
+	return splicePool.used()
+}
+
+func Done(p *Pair) {
+	splicePool.done(p)
+}
+
+func newSplicePairPool() *pairPool {
+	return &pairPool{}
+}
+
+func (me *pairPool) clear() {
+	me.Lock()
+	defer me.Unlock()
+	for _, p := range me.unused {
+		p.Close()
+	}
+	me.unused = me.unused[:0]
+}
+
+func (me *pairPool) used() int {
+	me.Lock()
+	defer me.Unlock()
+	return me.usedCount
+}
+
+
+func (me *pairPool) get() (p *Pair, err error) {
+	me.Lock()
+	defer me.Unlock()
+
+	me.usedCount++
+	l := len(me.unused)
+	if l > 0 {
+		p := me.unused[l-1]
+		me.unused = me.unused[:l-1]
+		return p, nil
+	}
+	
+	return newSplicePair()
+}
+
+func (me *pairPool) done(p *Pair) {
+	me.Lock()
+	defer me.Unlock()
+
+	me.usedCount--
+	me.unused = append(me.unused, p)
+}
+
+func init() {
+	splicePool = newSplicePairPool()
+}

splice/splice.go

+package splice
+
+// Routines for efficient file to file copying.
+
+import (
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"syscall"
+)
+
+var _ = log.Println
+
+
+var maxPipeSize int
+var resizable bool 
+
+func Resizable() bool {
+	return resizable
+}
+
+func MaxPipeSize() int {
+	return maxPipeSize
+}
+
+// From manpage on ubuntu Lucid:
+//
+// Since Linux 2.6.11, the pipe capacity is 65536 bytes.
+const DefaultPipeSize = 16 * 4096
+
+func init() {
+	content, err := ioutil.ReadFile("/proc/sys/fs/pipe-max-size")
+	if err != nil {
+		maxPipeSize = DefaultPipeSize
+	}
+	fmt.Sscan(string(content), &maxPipeSize)
+
+	r, w, err := os.Pipe()
+	if err != nil {
+		log.Panicf("cannot create pipe: %v", err)
+	}
+	sz, errNo := fcntl(r.Fd(), F_GETPIPE_SZ, 0)
+	resizable = (errNo == 0)
+	_, errNo = fcntl(r.Fd(), F_SETPIPE_SZ, 2*sz)
+	resizable = resizable && (errNo == 0)
+	r.Close()
+	w.Close()
+}
+
+// copy & paste from syscall.
+func fcntl(fd uintptr, cmd int, arg int) (val int, errno syscall.Errno) {
+	r0, _, e1 := syscall.Syscall(syscall.SYS_FCNTL, fd, uintptr(cmd), uintptr(arg))
+	val = int(r0)
+	errno = syscall.Errno(e1)
+	return
+}
+
+const F_SETPIPE_SZ = 1031
+const F_GETPIPE_SZ = 1032
+
+func newSplicePair() (p *Pair, err error) {
+	p = &Pair{}
+	p.r, p.w, err = os.Pipe()
+	if err != nil {
+		return nil, err
+	}
+
+	errNo := syscall.Errno(0)
+	for _, f := range []*os.File{p.r, p.w} {
+		_, errNo = fcntl(f.Fd(), syscall.F_SETFL, syscall.O_NONBLOCK)
+		if errNo != 0 {
+			p.Close()
+			return nil, os.NewSyscallError("fcntl setfl", errNo)
+		}
+	}
+
+	p.size, errNo = fcntl(p.r.Fd(), F_GETPIPE_SZ, 0)
+	if errNo == syscall.EINVAL {
+		p.size = DefaultPipeSize
+		return p, nil
+	}
+	if errNo != 0 {
+		p.Close()
+		return nil, os.NewSyscallError("fcntl getsize", errNo)
+	}
+	return p, nil
+}
+

splice/splice_test.go

+package splice
+
+import (
+	"io/ioutil"
+	"testing"
+)
+
+func TestPairSize(t *testing.T) {
+	p, _ := Get()
+	defer Done(p)
+	p.MaxGrow()
+	b := make([]byte, p.Cap() + 100)
+	for i := range b {
+		b[i] = byte(i)
+	}
+
+	f, _ := ioutil.TempFile("", "splice")
+	err := ioutil.WriteFile(f.Name(), b, 0644)
+	if err != nil {
+		t.Fatalf("WriteFile: %v", err)
+	}
+
+	_, err = p.LoadFrom(f.Fd(), len(b))
+	if err == nil {
+		t.Fatalf("should give error on exceeding capacity")
+	}
+	
+}
+
+
+
+