Skip to content

G
go-fuse
Commit eac61bbc authored by Han-Wen Nienhuys's avatar Han-Wen Nienhuys
Browse files
Options

zipfs: rewrite using new nodefs API. 

Remove memtree.
parent dddd77b8
Show whitespace changes
Inline Side-by-side
Showing with 219 additions and 369 deletions
nodefs/README.md
View file @ eac61bbc
......@@ -60,3 +60,5 @@ To do/To decide
* decide on a final package name
* handle less open/create.
* Symlink []byte vs string.
zipfs/memtree.go deleted 100644 → 0
View file @ dddd77b8
// Copyright 2016 the Go-FUSE Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package zipfs
import (
"fmt"
"strings"
"github.com/hanwen/go-fuse/fuse"
"github.com/hanwen/go-fuse/fuse/nodefs"
)
type MemFile interface {
Stat(out *fuse.Attr)
Data() []byte
}
type memNode struct {
nodefs.Node
file MemFile
fs *MemTreeFs
}
// memTreeFs creates a tree of internal Inodes. Since the tree is
// loaded in memory completely at startup, it does not need inode
// discovery through Lookup() at serve time.
type MemTreeFs struct {
root *memNode
files map[string]MemFile
Name string
}
func NewMemTreeFs(files map[string]MemFile) *MemTreeFs {
fs := &MemTreeFs{
root: &memNode{Node: nodefs.NewDefaultNode()},
files: files,
}
fs.root.fs = fs
return fs
}
func (fs *MemTreeFs) String() string {
return fs.Name
}
func (fs *MemTreeFs) Root() nodefs.Node {
return fs.root
}
func (fs *MemTreeFs) onMount() {
for k, v := range fs.files {
fs.addFile(k, v)
}
fs.files = nil
}
func (n *memNode) OnMount(c *nodefs.FileSystemConnector) {
n.fs.onMount()
}
func (n *memNode) Print(indent int) {
s := ""
for i := 0; i < indent; i++ {
s = s + " "
}
children := n.Inode().Children()
for k, v := range children {
if v.IsDir() {
fmt.Println(s + k + ":")
mn, ok := v.Node().(*memNode)
if ok {
mn.Print(indent + 2)
}
} else {
fmt.Println(s + k)
}
}
}
func (n *memNode) OpenDir(context *fuse.Context) (stream []fuse.DirEntry, code fuse.Status) {
children := n.Inode().Children()
stream = make([]fuse.DirEntry, 0, len(children))
for k, v := range children {
mode := fuse.S_IFREG | 0666
if v.IsDir() {
mode = fuse.S_IFDIR | 0777
}
stream = append(stream, fuse.DirEntry{
Name: k,
Mode: uint32(mode),
})
}
return stream, fuse.OK
}
func (n *memNode) Open(flags uint32, context *fuse.Context) (fuseFile nodefs.File, code fuse.Status) {
if flags&fuse.O_ANYWRITE != 0 {
return nil, fuse.EPERM
}
return nodefs.NewDataFile(n.file.Data()), fuse.OK
}
func (n *memNode) Deletable() bool {
return false
}
func (n *memNode) GetAttr(out *fuse.Attr, file nodefs.File, context *fuse.Context) fuse.Status {
if n.Inode().IsDir() {
out.Mode = fuse.S_IFDIR | 0777
return fuse.OK
}
n.file.Stat(out)
out.Blocks = (out.Size + 511) / 512
return fuse.OK
}
func (n *MemTreeFs) addFile(name string, f MemFile) {
comps := strings.Split(name, "/")
node := n.root.Inode()
for i, c := range comps {
child := node.GetChild(c)
if child == nil {
fsnode := &memNode{
Node: nodefs.NewDefaultNode(),
fs: n,
}
if i == len(comps)-1 {
fsnode.file = f
}
child = node.NewChild(c, fsnode.file == nil, fsnode)
}
node = child
}
}
zipfs/multizip.go
View file @ eac61bbc
......@@ -15,191 +15,69 @@ symlinking path/to/zipfile to /config/zipmount
*/
import (
"context"
"log"
"path/filepath"
"sync"
"syscall"
"github.com/hanwen/go-fuse/fuse"
"github.com/hanwen/go-fuse/fuse/nodefs"
"github.com/hanwen/go-fuse/fuse/pathfs"
"github.com/hanwen/go-fuse/nodefs"
)
const (
CONFIG_PREFIX = "config/"
)
////////////////////////////////////////////////////////////////
// MultiZipFs is a path filesystem that mounts zipfiles.
// MultiZipFs is a filesystem that mounts zipfiles.
type MultiZipFs struct {
lock sync.RWMutex
zips map[string]nodefs.Node
dirZipFileMap map[string]string
// zip files that we are in the process of unmounting.
zombie map[string]bool
nodeFs *pathfs.PathNodeFs
pathfs.FileSystem
nodefs.Inode
}
func NewMultiZipFs() *MultiZipFs {
m := &MultiZipFs{
zips: make(map[string]nodefs.Node),
zombie: make(map[string]bool),
dirZipFileMap: make(map[string]string),
FileSystem: pathfs.NewDefaultFileSystem(),
}
return m
}
func (fs *MultiZipFs) OnAdd(ctx context.Context) {
n := fs.NewPersistentInode(ctx, &configRoot{}, nodefs.NodeAttr{Mode: syscall.S_IFDIR})
func (fs *MultiZipFs) String() string {
return "MultiZipFs"
fs.AddChild("config", n, false)
}
func (fs *MultiZipFs) OnMount(nodeFs *pathfs.PathNodeFs) {
fs.nodeFs = nodeFs
}
func (fs *MultiZipFs) OpenDir(name string, context *fuse.Context) (stream []fuse.DirEntry, code fuse.Status) {
fs.lock.RLock()
defer fs.lock.RUnlock()
stream = make([]fuse.DirEntry, 0, len(fs.zips)+2)
if name == "" {
var d fuse.DirEntry
d.Name = "config"
d.Mode = fuse.S_IFDIR | 0700
stream = append(stream, fuse.DirEntry(d))
}
if name == "config" {
for k := range fs.zips {
var d fuse.DirEntry
d.Name = k
d.Mode = fuse.S_IFLNK
stream = append(stream, fuse.DirEntry(d))
}
}
return stream, fuse.OK
}
func (fs *MultiZipFs) GetAttr(name string, context *fuse.Context) (*fuse.Attr, fuse.Status) {
a := &fuse.Attr{}
a.Owner = *fuse.CurrentOwner()
if name == "" {
// Should not write in top dir.
a.Mode = fuse.S_IFDIR | 0500
return a, fuse.OK
}
if name == "config" {
a.Mode = fuse.S_IFDIR | 0700
return a, fuse.OK
}
dir, base := filepath.Split(name)
if dir != "" && dir != CONFIG_PREFIX {
return nil, fuse.ENOENT
}
submode := uint32(fuse.S_IFDIR | 0700)
if dir == CONFIG_PREFIX {
submode = fuse.S_IFLNK | 0600
}
fs.lock.RLock()
defer fs.lock.RUnlock()
a.Mode = submode
_, hasDir := fs.zips[base]
if hasDir {
return a, fuse.OK
}
return nil, fuse.ENOENT
type configRoot struct {
nodefs.Inode
}
func (fs *MultiZipFs) Unlink(name string, context *fuse.Context) (code fuse.Status) {
dir, basename := filepath.Split(name)
if dir == CONFIG_PREFIX {
fs.lock.Lock()
defer fs.lock.Unlock()
if fs.zombie[basename] {
return fuse.ENOENT
}
root, ok := fs.zips[basename]
if !ok {
return fuse.ENOENT
}
var _ = (nodefs.Unlinker)((*configRoot)(nil))
var _ = (nodefs.Symlinker)((*configRoot)(nil))
name := fs.dirZipFileMap[basename]
fs.zombie[basename] = true
delete(fs.zips, basename)
delete(fs.dirZipFileMap, basename)
// Drop the lock to ensure that notify doesn't cause a deadlock.
fs.lock.Unlock()
code = fs.nodeFs.UnmountNode(root.Inode())
fs.lock.Lock()
delete(fs.zombie, basename)
if !code.Ok() {
// Failed: reinstate
fs.zips[basename] = root
fs.dirZipFileMap[basename] = name
func (r *configRoot) Unlink(ctx context.Context, basename string) syscall.Errno {
if r.GetChild(basename) == nil {
return syscall.ENOENT
}
return code
}
return fuse.EPERM
}
func (fs *MultiZipFs) Readlink(path string, context *fuse.Context) (val string, code fuse.Status) {
dir, base := filepath.Split(path)
if dir != CONFIG_PREFIX {
return "", fuse.ENOENT
}
// XXX RmChild should return Inode?
fs.lock.Lock()
defer fs.lock.Unlock()
if fs.zombie[base] {
return "", fuse.ENOENT
_, parent := r.Parent()
ch := parent.GetChild(basename)
if ch == nil {
return syscall.ENOENT
}
zipfile, ok := fs.dirZipFileMap[base]
if !ok {
return "", fuse.ENOENT
success, _ := parent.RmChild(basename)
if !success {
return syscall.EIO
}
return zipfile, fuse.OK
ch.RmAllChildren()
parent.RmChild(basename)
parent.NotifyEntry(basename)
return 0
}
func (fs *MultiZipFs) Symlink(value string, linkName string, context *fuse.Context) (code fuse.Status) {
dir, base := filepath.Split(linkName)
if dir != CONFIG_PREFIX {
return fuse.EPERM
}
fs.lock.Lock()
defer fs.lock.Unlock()
if fs.zombie[base] {
return fuse.EBUSY
}
_, ok := fs.dirZipFileMap[base]
if ok {
return fuse.EBUSY
}
root, err := NewArchiveFileSystem(value)
func (r *configRoot) Symlink(ctx context.Context, target string, base string, out *fuse.EntryOut) (*nodefs.Inode, syscall.Errno) {
root, err := NewArchiveFileSystem(target)
if err != nil {
log.Println("NewZipArchiveFileSystem failed.", err)
return fuse.EINVAL
return nil, syscall.EINVAL
}
code = fs.nodeFs.Mount(base, root, nil)
if !code.Ok() {
return code
}
_, parent := r.Parent()
ch := r.NewPersistentInode(ctx, root, nodefs.NodeAttr{Mode: syscall.S_IFDIR})
parent.AddChild(base, ch, false)
fs.dirZipFileMap[base] = value
fs.zips[base] = root
return fuse.OK
link := r.NewPersistentInode(ctx, &nodefs.MemSymlink{
Data: []byte(target),
}, nodefs.NodeAttr{Mode: syscall.S_IFLNK})
r.AddChild(base, link, false)
return link, 0
}
zipfs/multizip_test.go
View file @ eac61bbc
......@@ -11,31 +11,29 @@ import (
"time"
"github.com/hanwen/go-fuse/fuse"
"github.com/hanwen/go-fuse/fuse/nodefs"
"github.com/hanwen/go-fuse/fuse/pathfs"
"github.com/hanwen/go-fuse/internal/testutil"
"github.com/hanwen/go-fuse/nodefs"
)
const testTtl = 100 * time.Millisecond
func setupMzfs(t *testing.T) (mountPoint string, state *fuse.Server, cleanup func()) {
fs := NewMultiZipFs()
root := &MultiZipFs{}
mountPoint = testutil.TempDir()
nfs := pathfs.NewPathNodeFs(fs, nil)
state, _, err := nodefs.MountRoot(mountPoint, nfs.Root(), &nodefs.Options{
EntryTimeout: testTtl,
AttrTimeout: testTtl,
NegativeTimeout: 0.0,
Debug: testutil.VerboseTest(),
})
dt := testTtl
opts := &nodefs.Options{
EntryTimeout: &dt,
AttrTimeout: &dt,
}
opts.Debug = testutil.VerboseTest()
server, err := nodefs.Mount(mountPoint, root, opts)
if err != nil {
t.Fatalf("MountNodeFileSystem failed: %v", err)
}
go state.Serve()
state.WaitMount()
return mountPoint, state, func() {
state.Unmount()
return mountPoint, server, func() {
server.Unmount()
os.RemoveAll(mountPoint)
}
}
......
zipfs/tarfs.go
View file @ eac61bbc
......@@ -9,11 +9,15 @@ import (
"bytes"
"compress/bzip2"
"compress/gzip"
"github.com/hanwen/go-fuse/fuse"
"context"
"io"
"os"
"path/filepath"
"strings"
"syscall"
"github.com/hanwen/go-fuse/fuse"
"github.com/hanwen/go-fuse/nodefs"
)
// TODO - handle symlinks.
......@@ -40,9 +44,14 @@ func (f *TarFile) Data() []byte {
return f.data
}
func NewTarTree(r io.Reader) map[string]MemFile {
files := map[string]MemFile{}
tr := tar.NewReader(r)
type tarRoot struct {
nodefs.Inode
rc io.ReadCloser
}
func (r *tarRoot) OnAdd(ctx context.Context) {
tr := tar.NewReader(r.rc)
defer r.rc.Close()
var longName *string
for {
......@@ -74,35 +83,64 @@ func NewTarTree(r io.Reader) map[string]MemFile {
buf := bytes.NewBuffer(make([]byte, 0, hdr.Size))
io.Copy(buf, tr)
df := &nodefs.MemRegularFile{
Data: buf.Bytes(),
}
dir, base := filepath.Split(filepath.Clean(hdr.Name))
files[hdr.Name] = &TarFile{
Header: *hdr,
data: buf.Bytes(),
p := r.EmbeddedInode()
for _, comp := range strings.Split(dir, "/") {
if len(comp) == 0 {
continue
}
ch := p.GetChild(comp)
if ch == nil {
p.AddChild(comp, p.NewPersistentInode(ctx,
&nodefs.Inode{},
nodefs.NodeAttr{Mode: syscall.S_IFDIR}), false)
}
p = ch
}
return files
HeaderToFileInfo(&df.Attr, hdr)
p.AddChild(base, r.NewPersistentInode(ctx, df, nodefs.NodeAttr{}), false)
}
}
type readCloser struct {
io.Reader
close func() error
}
func NewTarCompressedTree(name string, format string) (map[string]MemFile, error) {
func (rc *readCloser) Close() error {
return rc.close()
}
func NewTarCompressedTree(name string, format string) (nodefs.InodeEmbedder, error) {
f, err := os.Open(name)
if err != nil {
return nil, err
}
defer f.Close()
var stream io.Reader
var stream io.ReadCloser
switch format {
case "gz":
unzip, err := gzip.NewReader(f)
if err != nil {
return nil, err
}
defer unzip.Close()
stream = unzip
stream = &readCloser{
unzip,
f.Close,
}
case "bz2":
unzip := bzip2.NewReader(f)
stream = unzip
stream = &readCloser{
unzip,
f.Close,
}
}
return NewTarTree(stream), nil
return &tarRoot{rc: stream}, nil
}
zipfs/zipfs.go
View file @ eac61bbc
......@@ -7,14 +7,18 @@ package zipfs
import (
"archive/zip"
"bytes"
"context"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"strings"
"sync"
"syscall"
"github.com/hanwen/go-fuse/fuse"
"github.com/hanwen/go-fuse/fuse/nodefs"
"github.com/hanwen/go-fuse/nodefs"
)
type ZipFile struct {
......@@ -22,11 +26,6 @@ type ZipFile struct {
}
func (f *ZipFile) Stat(out *fuse.Attr) {
out.Mode = fuse.S_IFREG | uint32(f.File.Mode())
out.Size = uint64(f.File.UncompressedSize)
out.Mtime = uint64(f.File.ModTime().Unix())
out.Atime = out.Mtime
out.Ctime = out.Mtime
}
func (f *ZipFile) Data() []byte {
......@@ -44,41 +43,123 @@ func (f *ZipFile) Data() []byte {
return dest.Bytes()
}
type zipRoot struct {
nodefs.Inode
zr *zip.ReadCloser
}
var _ = (nodefs.OnAdder)((*zipRoot)(nil))
func (zr *zipRoot) OnAdd(ctx context.Context) {
for _, f := range zr.zr.File {
if f.FileInfo().IsDir() {
continue
}
dir, base := filepath.Split(filepath.Clean(f.Name))
p := &zr.Inode
for _, component := range strings.Split(dir, "/") {
if len(component) == 0 {
continue
}
ch := p.GetChild(component)
if ch == nil {
ch = p.NewPersistentInode(ctx, &nodefs.Inode{},
nodefs.NodeAttr{Mode: fuse.S_IFDIR})
p.AddChild(component, ch, true)
}
p = ch
}
ch := p.NewPersistentInode(ctx, &zipFile{file: f}, nodefs.NodeAttr{})
p.AddChild(base, ch, true)
}
}
// NewZipTree creates a new file-system for the zip file named name.
func NewZipTree(name string) (map[string]MemFile, error) {
func NewZipTree(name string) (nodefs.InodeEmbedder, error) {
r, err := zip.OpenReader(name)
if err != nil {
return nil, err
}
out := map[string]MemFile{}
for _, f := range r.File {
if strings.HasSuffix(f.Name, "/") {
continue
return &zipRoot{zr: r}, nil
}
// zipFile is a file read from a zip archive.
type zipFile struct {
nodefs.Inode
file *zip.File
mu sync.Mutex
data []byte
}
var _ = (nodefs.Opener)((*zipFile)(nil))
var _ = (nodefs.Getattrer)((*zipFile)(nil))
// Getattr sets the minimum, which is the size. A more full-featured
// FS would also set timestamps and permissions.
func (zf *zipFile) Getattr(ctx context.Context, f nodefs.FileHandle, out *fuse.AttrOut) syscall.Errno {
out.Mode = uint32(zf.file.Mode()) & 07777
out.Nlink = 1
out.Mtime = uint64(zf.file.ModTime().Unix())
out.Atime = out.Mtime
out.Ctime = out.Mtime
out.Size = zf.file.UncompressedSize64
out.Blocks = (out.Size + 511) / 512
return 0
}
// Open lazily unpacks zip data
func (zf *zipFile) Open(ctx context.Context, flags uint32) (nodefs.FileHandle, uint32, syscall.Errno) {
zf.mu.Lock()
defer zf.mu.Unlock()
if zf.data == nil {
rc, err := zf.file.Open()
if err != nil {
return nil, 0, syscall.EIO
}
content, err := ioutil.ReadAll(rc)
if err != nil {
return nil, 0, syscall.EIO
}
n := filepath.Clean(f.Name)
zf := &ZipFile{f}
out[n] = zf
zf.data = content
}
return out, nil
// We don't return a filehandle since we don't really need
// one. The file content is immutable, so hint the kernel to
// cache the data.
return nil, fuse.FOPEN_KEEP_CACHE, 0
}
func NewArchiveFileSystem(name string) (root nodefs.Node, err error) {
var files map[string]MemFile
// Read simply returns the data that was already unpacked in the Open call
func (zf *zipFile) Read(ctx context.Context, f nodefs.FileHandle, dest []byte, off int64) (fuse.ReadResult, syscall.Errno) {
end := int(off) + len(dest)
if end > len(zf.data) {
end = len(zf.data)
}
return fuse.ReadResultData(zf.data[off:end]), 0
}
var _ = (nodefs.OnAdder)((*zipRoot)(nil))
func NewArchiveFileSystem(name string) (root nodefs.InodeEmbedder, err error) {
switch {
case strings.HasSuffix(name, ".zip"):
files, err = NewZipTree(name)
root, err = NewZipTree(name)
case strings.HasSuffix(name, ".tar.gz"):
files, err = NewTarCompressedTree(name, "gz")
root, err = NewTarCompressedTree(name, "gz")
case strings.HasSuffix(name, ".tar.bz2"):
files, err = NewTarCompressedTree(name, "bz2")
root, err = NewTarCompressedTree(name, "bz2")
case strings.HasSuffix(name, ".tar"):
f, err := os.Open(name)
if err != nil {
return nil, err
}
files = NewTarTree(f)
root = &tarRoot{rc: f}
default:
return nil, fmt.Errorf("unknown archive format %q", name)
}
......@@ -87,7 +168,5 @@ func NewArchiveFileSystem(name string) (root nodefs.Node, err error) {
return nil, err
}
mfs := NewMemTreeFs(files)
mfs.Name = fmt.Sprintf("fs(%s)", name)
return mfs.Root(), nil
return root, nil
}
zipfs/zipfs_test.go
View file @ eac61bbc
......@@ -14,8 +14,8 @@ import (
"time"
"github.com/hanwen/go-fuse/fuse"
"github.com/hanwen/go-fuse/fuse/nodefs"
"github.com/hanwen/go-fuse/internal/testutil"
"github.com/hanwen/go-fuse/nodefs"
)
func testZipFile() string {
......@@ -34,15 +34,12 @@ func setupZipfs(t *testing.T) (mountPoint string, cleanup func()) {
}
mountPoint = testutil.TempDir()
state, _, err := nodefs.MountRoot(mountPoint, root, &nodefs.Options{
Debug: testutil.VerboseTest(),
})
go state.Serve()
state.WaitMount()
opts := &nodefs.Options{}
opts.Debug = testutil.VerboseTest()
server, err := nodefs.Mount(mountPoint, root, opts)
return mountPoint, func() {
state.Unmount()
server.Unmount()
os.RemoveAll(mountPoint)
}
}
......@@ -70,19 +67,17 @@ func TestZipFs(t *testing.T) {
if err != nil {
t.Fatalf("Stat failed: %v", err)
}
if fi.Mode() != 0664 {
t.Fatalf("File mode 0%o != 0664", fi.Mode())
if got, want := fi.Mode(), 0664; int(got) != want {
t.Fatalf("File mode: got 0%o want 0%o", got, want)
}
if st := fi.Sys().(*syscall.Stat_t); st.Blocks != 1 {
t.Errorf("got block count %d, want 1", st.Blocks)
}
mtime, err := time.Parse(time.RFC3339, "2011-02-22T12:56:12Z")
if err != nil {
if want, err := time.Parse(time.RFC3339, "2011-02-22T12:56:12Z"); err != nil {
panic(err)
}
if !fi.ModTime().Equal(mtime) {
t.Fatalf("File mtime %v != %v", fi.ModTime(), mtime)
} else if !fi.ModTime().Equal(want) {
t.Fatalf("File mtime got %v, want %v", fi.ModTime(), want)
}
if fi.IsDir() {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7