Commit 0f728ba1 authored by Tommy Lindgren's avatar Tommy Lindgren

fuse: increase max readers for better throughput

Previously maximum number of requests readers was hard-coded to 2.
Setting it to max(2, min(16, GOMAXPROCS)) improves the throughput on
systems with more cores available. Going beyond 16 readers seems to hurt
performance even if there are more cores available.

Benchmark GoFuseRead can be used for demonstrating the effects of this
variable. The benchmark reads 32 streams in parallel from a dummy
filesystem (read operations immediately return zeros). Example results
from an i7-8550U (8 cores):

| Max readers | Total throughput |
| ----------: | ---------------: |
|           2 |       13217 MB/s |
|           4 |       19202 MB/s |
|           8 |       19973 MB/s |
|          16 |       18994 MB/s |

On a 96 core system:

| Max readers | Total throughput |
| ----------: | ---------------: |
|           2 |       11490 MB/s |
|           4 |       16129 MB/s |
|           8 |       24263 MB/s |
|          16 |       29568 MB/s |
|          32 |       28262 MB/s |

Note that improvements won't be as dramatic for real filesytem
implementations. In benchmarks for a filesystem doing real work I see a
30-40% improvement (8.3 -> 11.4 GB/s) on the 96 core system.

Also tweaked some of the other benchmarks so they don't leave behind
mountpoints.

Fixes #388.

Change-Id: Ibff17d7fc92195f078a9ccff818a31f3a58873f2
parent 8e0bbdb1
...@@ -16,5 +16,6 @@ Patrick Crosby <pcrosby@gmail.com> ...@@ -16,5 +16,6 @@ Patrick Crosby <pcrosby@gmail.com>
Paul Jolly <paul@myitcv.org.uk> Paul Jolly <paul@myitcv.org.uk>
Paul Warren <paul.warren@emc.com> Paul Warren <paul.warren@emc.com>
Shayan Pooya <shayan@arista.com> Shayan Pooya <shayan@arista.com>
Tommy Lindgren <tommy.lindgren@gmail.com>
Valient Gough <vgough@pobox.com> Valient Gough <vgough@pobox.com>
Yongwoo Park <nnnlife@gmail.com> Yongwoo Park <nnnlife@gmail.com>
// Copyright 2021 the Go-FUSE Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package benchmark
import (
"fmt"
"os"
"os/exec"
"testing"
"github.com/hanwen/go-fuse/v2/internal/testutil"
"golang.org/x/sync/errgroup"
)
func BenchmarkGoFuseRead(b *testing.B) {
fs := &readFS{}
wd, clean := setupFs(fs, b.N)
defer clean()
jobs := 32
blockSize := 64 * 1024
cmds := make([]*exec.Cmd, jobs)
for i := 0; i < jobs; i++ {
cmds[i] = exec.Command("dd",
fmt.Sprintf("if=%s/foo.txt", wd),
"iflag=direct",
"of=/dev/null",
fmt.Sprintf("bs=%d", blockSize),
fmt.Sprintf("count=%d", b.N))
if testutil.VerboseTest() {
cmds[i].Stdout = os.Stdout
cmds[i].Stderr = os.Stderr
}
}
b.SetBytes(int64(jobs * blockSize))
b.ReportAllocs()
b.ResetTimer()
var eg errgroup.Group
for i := 0; i < jobs; i++ {
i := i
eg.Go(func() error {
return cmds[i].Run()
})
}
if err := eg.Wait(); err != nil {
b.Fatalf("dd failed: %v", err)
}
b.StopTimer()
}
// Copyright 2021 the Go-FUSE Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package benchmark
import (
"context"
"syscall"
"time"
"github.com/hanwen/go-fuse/v2/fs"
"github.com/hanwen/go-fuse/v2/fuse"
)
const fileSize = 2 << 60
// readFS is a filesystem that always and immediately returns zeros on read
// operations. Useful when benchmarking the raw throughput with go-fuse.
type readFS struct {
fs.Inode
}
var _ = (fs.NodeLookuper)((*readFS)(nil))
func (n *readFS) Lookup(ctx context.Context, name string, out *fuse.EntryOut) (*fs.Inode, syscall.Errno) {
sattr := fs.StableAttr{Mode: fuse.S_IFREG}
return n.NewInode(ctx, &readFS{}, sattr), fs.OK
}
var _ = (fs.NodeGetattrer)((*readFS)(nil))
func (n *readFS) Getattr(ctx context.Context, f fs.FileHandle, out *fuse.AttrOut) syscall.Errno {
out.Size = fileSize
out.SetTimeout(time.Hour)
return fs.OK
}
var _ = (fs.NodeOpener)((*readFS)(nil))
func (n *readFS) Open(ctx context.Context, flags uint32) (fh fs.FileHandle, fuseFlags uint32, errno syscall.Errno) {
return &readFS{}, fuse.FOPEN_DIRECT_IO, fs.OK
}
var _ = (fs.FileReader)((*readFS)(nil))
func (n *readFS) Read(ctx context.Context, dest []byte, offset int64) (fuse.ReadResult, syscall.Errno) {
return fuse.ReadResultData(dest), fs.OK
}
...@@ -129,7 +129,7 @@ func BenchmarkGoFuseStat(b *testing.B) { ...@@ -129,7 +129,7 @@ func BenchmarkGoFuseStat(b *testing.B) {
threads := runtime.GOMAXPROCS(0) threads := runtime.GOMAXPROCS(0)
if err := TestingBOnePass(b, threads, fileList, wd); err != nil { if err := TestingBOnePass(b, threads, fileList, wd); err != nil {
log.Fatalf("TestingBOnePass %v8", err) b.Fatalf("TestingBOnePass %v8", err)
} }
} }
...@@ -252,6 +252,8 @@ func BenchmarkCFuseThreadedStat(b *testing.B) { ...@@ -252,6 +252,8 @@ func BenchmarkCFuseThreadedStat(b *testing.B) {
f.Close() f.Close()
mountPoint := testutil.TempDir() mountPoint := testutil.TempDir()
defer os.RemoveAll(mountPoint)
cmd := exec.Command(wd+"/cstatfs", cmd := exec.Command(wd+"/cstatfs",
"-o", "-o",
"entry_timeout=0.0,attr_timeout=0.0,ac_attr_timeout=0.0,negative_timeout=0.0", "entry_timeout=0.0,attr_timeout=0.0,ac_attr_timeout=0.0,negative_timeout=0.0",
...@@ -274,6 +276,6 @@ func BenchmarkCFuseThreadedStat(b *testing.B) { ...@@ -274,6 +276,6 @@ func BenchmarkCFuseThreadedStat(b *testing.B) {
os.Lstat(mountPoint) os.Lstat(mountPoint)
threads := runtime.GOMAXPROCS(0) threads := runtime.GOMAXPROCS(0)
if err := TestingBOnePass(b, threads, fileList, mountPoint); err != nil { if err := TestingBOnePass(b, threads, fileList, mountPoint); err != nil {
log.Fatalf("TestingBOnePass %v", err) b.Fatalf("TestingBOnePass %v", err)
} }
} }
...@@ -21,6 +21,9 @@ import ( ...@@ -21,6 +21,9 @@ import (
const ( const (
// The kernel caps writes at 128k. // The kernel caps writes at 128k.
MAX_KERNEL_WRITE = 128 * 1024 MAX_KERNEL_WRITE = 128 * 1024
minMaxReaders = 2
maxMaxReaders = 16
) )
// Server contains the logic for reading from the FUSE device and // Server contains the logic for reading from the FUSE device and
...@@ -40,6 +43,9 @@ type Server struct { ...@@ -40,6 +43,9 @@ type Server struct {
opts *MountOptions opts *MountOptions
// maxReaders is the maximum number of goroutines reading requests
maxReaders int
// Pools for []byte // Pools for []byte
buffers bufferPool buffers bufferPool
...@@ -161,9 +167,17 @@ func NewServer(fs RawFileSystem, mountPoint string, opts *MountOptions) (*Server ...@@ -161,9 +167,17 @@ func NewServer(fs RawFileSystem, mountPoint string, opts *MountOptions) (*Server
} }
} }
maxReaders := runtime.GOMAXPROCS(0)
if maxReaders < minMaxReaders {
maxReaders = minMaxReaders
} else if maxReaders > maxMaxReaders {
maxReaders = maxMaxReaders
}
ms := &Server{ ms := &Server{
fileSystem: fs, fileSystem: fs,
opts: &o, opts: &o,
maxReaders: maxReaders,
retrieveTab: make(map[uint64]*retrieveCacheRequest), retrieveTab: make(map[uint64]*retrieveCacheRequest),
// OSX has races when multiple routines read from the // OSX has races when multiple routines read from the
// FUSE device: on unmount, sometime some reads do not // FUSE device: on unmount, sometime some reads do not
...@@ -238,9 +252,6 @@ func (ms *Server) DebugData() string { ...@@ -238,9 +252,6 @@ func (ms *Server) DebugData() string {
return fmt.Sprintf("readers: %d", r) return fmt.Sprintf("readers: %d", r)
} }
// What is a good number? Maybe the number of CPUs?
const _MAX_READERS = 2
// handleEINTR retries the given function until it doesn't return syscall.EINTR. // handleEINTR retries the given function until it doesn't return syscall.EINTR.
// This is similar to the HANDLE_EINTR() macro from Chromium ( see // This is similar to the HANDLE_EINTR() macro from Chromium ( see
// https://code.google.com/p/chromium/codesearch#chromium/src/base/posix/eintr_wrapper.h // https://code.google.com/p/chromium/codesearch#chromium/src/base/posix/eintr_wrapper.h
...@@ -267,7 +278,7 @@ func (ms *Server) readRequest(exitIdle bool) (req *request, code Status) { ...@@ -267,7 +278,7 @@ func (ms *Server) readRequest(exitIdle bool) (req *request, code Status) {
dest := ms.readPool.Get().([]byte) dest := ms.readPool.Get().([]byte)
ms.reqMu.Lock() ms.reqMu.Lock()
if ms.reqReaders > _MAX_READERS { if ms.reqReaders > ms.maxReaders {
ms.reqMu.Unlock() ms.reqMu.Unlock()
return nil, OK return nil, OK
} }
......
...@@ -3,6 +3,7 @@ module github.com/hanwen/go-fuse/v2 ...@@ -3,6 +3,7 @@ module github.com/hanwen/go-fuse/v2
require ( require (
github.com/hanwen/go-fuse v1.0.0 github.com/hanwen/go-fuse v1.0.0
github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522
) )
......
...@@ -2,5 +2,7 @@ github.com/hanwen/go-fuse v1.0.0 h1:GxS9Zrn6c35/BnfiVsZVWmsG803xwE7eVRDvcf/BEVc= ...@@ -2,5 +2,7 @@ github.com/hanwen/go-fuse v1.0.0 h1:GxS9Zrn6c35/BnfiVsZVWmsG803xwE7eVRDvcf/BEVc=
github.com/hanwen/go-fuse v1.0.0/go.mod h1:unqXarDXqzAk0rt98O2tVndEPIpUgLD9+rwFisZH3Ok= github.com/hanwen/go-fuse v1.0.0/go.mod h1:unqXarDXqzAk0rt98O2tVndEPIpUgLD9+rwFisZH3Ok=
github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 h1:MtvEpTB6LX3vkb4ax0b5D2DHbNAUsen0Gx5wZoq3lV4= github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 h1:MtvEpTB6LX3vkb4ax0b5D2DHbNAUsen0Gx5wZoq3lV4=
github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a h1:DcqTD9SDLc+1P/r1EmRBwnVsrOwW+kk2vWf9n+1sGhs=
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522 h1:Ve1ORMCxvRmSXBwJK+t3Oy+V2vRW2OetUQBq4rJIkZE= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522 h1:Ve1ORMCxvRmSXBwJK+t3Oy+V2vRW2OetUQBq4rJIkZE=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment