Commit 23362204 authored by Kirill Smelkov's avatar Kirill Smelkov

bigfile/py: Allow PyBigFile backend to expose "mmap overlay" functionality

This patch logically continues previous change `bigfile/virtmem:
Introduce "mmap overlay" mode` and exposes mmap-overlay functionality to
Python: if PyBigFile backend provides .blkmmapper PyCapsule the
mmap-related methods will be extracted from it and passed on through to
virtmem - see _bigfile.h for details.

ZBigFile will use this to hook into using WCFS.
parent fae045cc
...@@ -425,6 +425,19 @@ PyFunc(pyfileh_invalidate_page, "invalidate_page(pgoffset) - invalidate fileh pa ...@@ -425,6 +425,19 @@ PyFunc(pyfileh_invalidate_page, "invalidate_page(pgoffset) - invalidate fileh pa
} }
PyFunc(pyfileh_uses_mmap_overlay, "uses_mmap_overlay() - whether base data for all VMAs"
" of this fileh are taken as base-layer mmap")
(PyObject *pyfileh0, PyObject *args)
{
PyBigFileH *pyfileh = container_of(pyfileh0, PyBigFileH, pyobj);
BigFileH *fileh = &pyfileh->fileh;
if (!PyArg_ParseTuple(args, ""))
return NULL;
return PyBool_FromLong(fileh->mmap_overlay);
}
/* pyfileh vs cyclic GC */ /* pyfileh vs cyclic GC */
static int static int
...@@ -509,7 +522,8 @@ static /*const*/ PyMethodDef pyfileh_methods[] = { ...@@ -509,7 +522,8 @@ static /*const*/ PyMethodDef pyfileh_methods[] = {
{"dirty_writeout", pyfileh_dirty_writeout, METH_VARARGS, pyfileh_dirty_writeout_doc}, {"dirty_writeout", pyfileh_dirty_writeout, METH_VARARGS, pyfileh_dirty_writeout_doc},
{"dirty_discard", pyfileh_dirty_discard, METH_VARARGS, pyfileh_dirty_discard_doc}, {"dirty_discard", pyfileh_dirty_discard, METH_VARARGS, pyfileh_dirty_discard_doc},
{"isdirty", pyfileh_isdirty, METH_VARARGS, pyfileh_isdirty_doc}, {"isdirty", pyfileh_isdirty, METH_VARARGS, pyfileh_isdirty_doc},
{"invalidate_page", pyfileh_invalidate_page,METH_VARARGS, pyfileh_invalidate_page_doc}, {"invalidate_page", pyfileh_invalidate_page, METH_VARARGS, pyfileh_invalidate_page_doc},
{"uses_mmap_overlay", pyfileh_uses_mmap_overlay, METH_VARARGS, pyfileh_uses_mmap_overlay_doc},
{NULL} {NULL}
}; };
...@@ -955,10 +969,42 @@ out: ...@@ -955,10 +969,42 @@ out:
} }
/* PyBigFile: mmap methods.
* They redirect op X to type.blkmmapper.X without going to Python level */
static int
pybigfile_mmap_setup_read(VMA *vma, BigFile *file0, blk_t blk, size_t blklen)
{
PyBigFile *file = container_of(file0, PyBigFile, file);
ASSERT(file->blkmmap_ops != NULL);
return file->blkmmap_ops->mmap_setup_read(vma, file0, blk, blklen);
}
static int
pybigfile_remmap_blk_read(VMA *vma, BigFile *file0, blk_t blk)
{
PyBigFile *file = container_of(file0, PyBigFile, file);
ASSERT(file->blkmmap_ops != NULL);
return file->blkmmap_ops->remmap_blk_read(vma, file0, blk);
}
static int
pybigfile_munmap(VMA *vma, BigFile *file0)
{
PyBigFile *file = container_of(file0, PyBigFile, file);
ASSERT(file->blkmmap_ops != NULL);
return file->blkmmap_ops->munmap(vma, file0);
}
static const struct bigfile_ops pybigfile_ops = { static const struct bigfile_ops pybigfile_ops = {
.loadblk = pybigfile_loadblk, .loadblk = pybigfile_loadblk,
.storeblk = pybigfile_storeblk, .storeblk = pybigfile_storeblk,
//.release = //.release =
.mmap_setup_read = pybigfile_mmap_setup_read,
.remmap_blk_read = pybigfile_remmap_blk_read,
.munmap = pybigfile_munmap,
}; };
...@@ -972,16 +1018,23 @@ pyfileh_open(PyObject *pyfile0, PyObject *args) ...@@ -972,16 +1018,23 @@ pyfileh_open(PyObject *pyfile0, PyObject *args)
RAM *ram = ram_get_default(NULL); // TODO get ram from args RAM *ram = ram_get_default(NULL); // TODO get ram from args
int err; int err;
int mmap_overlay = -1; /* -1 means None; https://bugs.python.org/issue14705 */
if (!PyArg_ParseTuple(args, "")) if (!PyArg_ParseTuple(args, "|i", &mmap_overlay))
return NULL; return NULL;
if (mmap_overlay == -1)
mmap_overlay = (pyfile->blkmmap_ops != NULL ? 1 : 0);
if (mmap_overlay && pyfile->blkmmap_ops == NULL)
return PyErr_Format(PyExc_TypeError,
"%s type does not provide blkmmapper", pyfile0->ob_type->tp_name);
pyfileh = PyType_New(PyBigFileH, &PyBigFileH_Type, NULL); pyfileh = PyType_New(PyBigFileH, &PyBigFileH_Type, NULL);
if (!pyfileh) if (!pyfileh)
return NULL; return NULL;
Py_INCREF(pyfile); Py_INCREF(pyfile);
err = fileh_open(&pyfileh->fileh, &pyfile->file, ram, DONT_MMAP_OVERLAY); err = fileh_open(&pyfileh->fileh, &pyfile->file, ram,
mmap_overlay ? MMAP_OVERLAY : DONT_MMAP_OVERLAY);
if (err) { if (err) {
XPyErr_SetFromErrno(); XPyErr_SetFromErrno();
Py_DECREF(pyfile); Py_DECREF(pyfile);
...@@ -998,6 +1051,7 @@ pyfile_traverse(PyObject *pyfile0, visitproc visit, void *arg) ...@@ -998,6 +1051,7 @@ pyfile_traverse(PyObject *pyfile0, visitproc visit, void *arg)
{ {
PyBigFile *pyfile = container_of(pyfile0, PyBigFile, pyobj); PyBigFile *pyfile = container_of(pyfile0, PyBigFile, pyobj);
Py_VISIT(pyfile->blkmmapper);
return 0; return 0;
} }
...@@ -1006,6 +1060,7 @@ pyfile_clear(PyObject *pyfile0) ...@@ -1006,6 +1060,7 @@ pyfile_clear(PyObject *pyfile0)
{ {
PyBigFile *pyfile = container_of(pyfile0, PyBigFile, pyobj); PyBigFile *pyfile = container_of(pyfile0, PyBigFile, pyobj);
Py_CLEAR(pyfile->blkmmapper);
return 0; return 0;
} }
...@@ -1022,6 +1077,8 @@ pyfile_dealloc(PyObject *pyfile0) ...@@ -1022,6 +1077,8 @@ pyfile_dealloc(PyObject *pyfile0)
PyBigFile *pyfile = container_of(pyfile0, PyBigFile, pyobj); PyBigFile *pyfile = container_of(pyfile0, PyBigFile, pyobj);
pyfile->blkmmap_ops = NULL;
pyfile_clear(&pyfile->pyobj); pyfile_clear(&pyfile->pyobj);
pyfile->pyobj.ob_type->tp_free(&pyfile->pyobj); pyfile->pyobj.ob_type->tp_free(&pyfile->pyobj);
} }
...@@ -1031,10 +1088,51 @@ static PyObject * ...@@ -1031,10 +1088,51 @@ static PyObject *
pyfile_new(PyTypeObject *type, PyObject *args, PyObject *kw) pyfile_new(PyTypeObject *type, PyObject *args, PyObject *kw)
{ {
PyBigFile *self; PyBigFile *self;
PyObject *blkmmapper;
bigfile_ops *blkmmap_ops = NULL;
/* try to get type.blkmmapper and verify it provides IBlkMMapper interface */
blkmmapper = PyObject_GetAttrString((PyObject*)type, "blkmmapper");
PyErr_Clear(); /* GetAttr raises exception if there is no attribute */
if (blkmmapper) {
if (!PyCapsule_IsValid(blkmmapper, "wendelin.bigfile.IBlkMMapper")) {
Py_DECREF(blkmmapper);
return PyErr_Format(PyExc_TypeError,
"%s: .blkmmapper is not a valid pycapsule with mmap methods", type->tp_name);
}
blkmmap_ops = PyCapsule_GetPointer(blkmmapper, "wendelin.bigfile.IBlkMMapper");
if (blkmmap_ops == NULL) { /* just in case - must not fail */
Py_DECREF(blkmmapper);
return NULL;
}
if (blkmmap_ops->loadblk ||
blkmmap_ops->storeblk)
{
Py_DECREF(blkmmapper);
return PyErr_Format(PyExc_TypeError,
"%s: .blkmmapper: !mmap methods present", type->tp_name);
}
if (!(blkmmap_ops->mmap_setup_read &&
blkmmap_ops->remmap_blk_read &&
blkmmap_ops->munmap))
{
Py_DECREF(blkmmapper);
return PyErr_Format(PyExc_TypeError,
"%s: .blkmmapper: not all mmap methods present", type->tp_name);
}
}
self = (PyBigFile *)PyType_GenericNew(type, args, kw); self = (PyBigFile *)PyType_GenericNew(type, args, kw);
if (!self) if (!self) {
Py_XDECREF(blkmmapper);
return NULL; return NULL;
}
self->blkmmapper = blkmmapper;
self->blkmmap_ops = blkmmap_ops;
// FIXME "k" = unsigned long - we need size_t // FIXME "k" = unsigned long - we need size_t
static char *kw_list[] = {"blksize", NULL}; static char *kw_list[] = {"blksize", NULL};
...@@ -1054,7 +1152,7 @@ static PyMemberDef pyfile_members[] = { ...@@ -1054,7 +1152,7 @@ static PyMemberDef pyfile_members[] = {
}; };
static /*const*/ PyMethodDef pyfile_methods[] = { static /*const*/ PyMethodDef pyfile_methods[] = {
{"fileh_open", pyfileh_open, METH_VARARGS, "fileh_open(ram=None) -> new file handle"}, {"fileh_open", pyfileh_open, METH_VARARGS, "fileh_open(ram=None, mmap_overlay=None) -> new file handle"},
{NULL} {NULL}
}; };
......
...@@ -2,7 +2,7 @@ ...@@ -2,7 +2,7 @@
#define _WENDELIN_BIGFILE__BIGFILE_H #define _WENDELIN_BIGFILE__BIGFILE_H
/* Wendelin.bigfile | Python interface to memory/files /* Wendelin.bigfile | Python interface to memory/files
* Copyright (C) 2014-2019 Nexedi SA and Contributors. * Copyright (C) 2014-2021 Nexedi SA and Contributors.
* Kirill Smelkov <kirr@nexedi.com> * Kirill Smelkov <kirr@nexedi.com>
* *
* This program is free software: you can Use, Study, Modify and Redistribute * This program is free software: you can Use, Study, Modify and Redistribute
...@@ -26,11 +26,14 @@ ...@@ -26,11 +26,14 @@
* *
* - `BigFile` is base class that allows implementing BigFile backends in Python. * - `BigFile` is base class that allows implementing BigFile backends in Python.
* Users can inherit from BigFile, implement loadblk/storeblk and this way * Users can inherit from BigFile, implement loadblk/storeblk and this way
* provide access to data managed from Python to virtmem subsystem. * provide access to data managed from Python to virtmem subsystem(*).
* - `BigFileH` represents virtmem file handle for opened BigFile. * - `BigFileH` represents virtmem file handle for opened BigFile.
* It can be mmap'ed and provides writeout control. * It can be mmap'ed and provides writeout control.
* - `VMA` represents mmap'ed part of a BigFileH. * - `VMA` represents mmap'ed part of a BigFileH.
* It provides buffer/memoryview interface for data access. * It provides buffer/memoryview interface for data access.
*
* (*) A subclass may additionally provide functionality to map file data into
* memory. Please see BigFile documentation for details.
*/ */
#include <Python.h> #include <Python.h>
...@@ -97,8 +100,18 @@ typedef struct PyBigFileH PyBigFileH; ...@@ -97,8 +100,18 @@ typedef struct PyBigFileH PyBigFileH;
/* /*
* BigFile that can be implemented in python * BigFile that can be implemented in python
* *
* Allows subclasses to implement .loadblk() (& friends) in python. * Allows subclasses to implement .loadblk() and .storeblk() in python.
* For users .fileh_open() is exposed to get to file handles. * For users .fileh_open() is exposed to get to file handles.
*
* A subclass may additionally provide functionality to map file data into
* memory: if subclass provides .blkmmapper attribute, it is treated as
* pycapsule with type "wendelin.bigfile.IBlkMMapper" and C-level bigfile_ops
* struct that provides .mmap_setup_read and other operations related to
* mmapping data. To avoid deadlocks all mmap-related functionality must be
* nogil and so cannot be implemented in Python.
*
* The primary user of .blkmmapper functionality will be _ZBigFile which uses WCFS
* and mmaps files from it to provide memory mappings for ZBigFile data.
*/ */
struct PyBigFile { struct PyBigFile {
PyObject pyobj; PyObject pyobj;
...@@ -106,6 +119,11 @@ struct PyBigFile { ...@@ -106,6 +119,11 @@ struct PyBigFile {
* automatically adds support for weakrefs for in-python defined children */ * automatically adds support for weakrefs for in-python defined children */
BigFile file; BigFile file;
/* blkmmapper is PyCapsule object with type.blkmmapper if BigFile subclass has it | NULL */
PyObject *blkmmapper;
/* bigfile_ops extracted from ^^^ capsule | NULL */
bigfile_ops *blkmmap_ops;
}; };
typedef struct PyBigFile PyBigFile; typedef struct PyBigFile PyBigFile;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment