Skip to content

Z
ZODB
Commit 04581d75 authored by Jim Fulton's avatar Jim Fulton
Browse files
Options

Fixed a compiler warning. 

Cleaned up code formatting.
parent 86ec9190
Hide whitespace changes
Inline Side-by-side
Showing with 1698 additions and 1570 deletions
src/persistent/cPersistence.c
View file @ 04581d75
...@@ -10,17 +10,17 @@ ...@@ -10,17 +10,17 @@
WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
FOR A PARTICULAR PURPOSE FOR A PARTICULAR PURPOSE
****************************************************************************/ ****************************************************************************/
static char cPersistence_doc_string[] = static char cPersistence_doc_string[] =
"Defines Persistent mixin class for persistent objects.\n" "Defines Persistent mixin class for persistent objects.\n"
"\n" "\n"
"$Id$\n"; "$Id$\n";
#include "cPersistence.h" #include "cPersistence.h"
#include "structmember.h" #include "structmember.h"
struct ccobject_head_struct { struct ccobject_head_struct {
CACHE_HEAD CACHE_HEAD
}; };
/* These two objects are initialized when the module is loaded */ /* These two objects are initialized when the module is loaded */
...@@ -37,23 +37,23 @@ static PyObject *py___getnewargs__, *py___getstate__; ...@@ -37,23 +37,23 @@ static PyObject *py___getnewargs__, *py___getstate__;
static int static int
init_strings(void) init_strings(void)
{ {
#define INIT_STRING(S) \ #define INIT_STRING(S) \
if (!(py_ ## S = PyString_InternFromString(#S))) \ if (!(py_ ## S = PyString_InternFromString(#S))) \
return -1; return -1;
INIT_STRING(keys); INIT_STRING(keys);
INIT_STRING(setstate); INIT_STRING(setstate);
INIT_STRING(timeTime); INIT_STRING(timeTime);
INIT_STRING(__dict__); INIT_STRING(__dict__);
INIT_STRING(_p_changed); INIT_STRING(_p_changed);
INIT_STRING(_p_deactivate); INIT_STRING(_p_deactivate);
INIT_STRING(__getattr__); INIT_STRING(__getattr__);
INIT_STRING(__setattr__); INIT_STRING(__setattr__);
INIT_STRING(__delattr__); INIT_STRING(__delattr__);
INIT_STRING(__slotnames__); INIT_STRING(__slotnames__);
INIT_STRING(__getnewargs__); INIT_STRING(__getnewargs__);
INIT_STRING(__getstate__); INIT_STRING(__getstate__);
#undef INIT_STRING #undef INIT_STRING
return 0; return 0;
} }
#ifdef Py_DEBUG #ifdef Py_DEBUG
...@@ -63,13 +63,13 @@ fatal_1350(cPersistentObject *self, const char *caller, const char *detail) ...@@ -63,13 +63,13 @@ fatal_1350(cPersistentObject *self, const char *caller, const char *detail)
char buf[1000]; char buf[1000];
PyOS_snprintf(buf, sizeof(buf), PyOS_snprintf(buf, sizeof(buf),
"cPersistence.c %s(): object at %p with type %.200s\n" "cPersistence.c %s(): object at %p with type %.200s\n"
"%s.\n" "%s.\n"
"The only known cause is multiple threads trying to ghost and\n" "The only known cause is multiple threads trying to ghost and\n"
"unghost the object simultaneously.\n" "unghost the object simultaneously.\n"
"That's not legal, but ZODB can't stop it.\n" "That's not legal, but ZODB can't stop it.\n"
"See Collector #1350.\n", "See Collector #1350.\n",
caller, self, self->ob_type->tp_name, detail); caller, self, self->ob_type->tp_name, detail);
Py_FatalError(buf); Py_FatalError(buf);
} }
#endif #endif
...@@ -82,44 +82,48 @@ static void ghostify(cPersistentObject*); ...@@ -82,44 +82,48 @@ static void ghostify(cPersistentObject*);
static int static int
unghostify(cPersistentObject *self) unghostify(cPersistentObject *self)
{ {
if (self->state < 0 && self->jar) { if (self->state < 0 && self->jar)
PyObject *r; {
PyObject *r;
/* Is it ever possible to not have a cache? */
if (self->cache) { /* Is it ever possible to not have a cache? */
/* Create a node in the ring for this unghostified object. */ if (self->cache)
self->cache->non_ghost_count++; {
self->cache->total_estimated_size += /* Create a node in the ring for this unghostified object. */
_estimated_size_in_bytes(self->estimated_size); self->cache->non_ghost_count++;
ring_add(&self->cache->ring_home, &self->ring); self->cache->total_estimated_size +=
Py_INCREF(self); _estimated_size_in_bytes(self->estimated_size);
ring_add(&self->cache->ring_home, &self->ring);
Py_INCREF(self);
} }
/* set state to CHANGED while setstate() call is in progress /* set state to CHANGED while setstate() call is in progress
to prevent a recursive call to _PyPersist_Load(). to prevent a recursive call to _PyPersist_Load().
*/ */
self->state = cPersistent_CHANGED_STATE; self->state = cPersistent_CHANGED_STATE;
/* Call the object's __setstate__() */ /* Call the object's __setstate__() */
r = PyObject_CallMethod(self->jar, "setstate", "O", (PyObject *)self); r = PyObject_CallMethod(self->jar, "setstate", "O", (PyObject *)self);
if (r == NULL) { if (r == NULL)
ghostify(self); {
return -1; ghostify(self);
return -1;
} }
self->state = cPersistent_UPTODATE_STATE; self->state = cPersistent_UPTODATE_STATE;
Py_DECREF(r); Py_DECREF(r);
if (self->cache && self->ring.r_next == NULL) { if (self->cache && self->ring.r_next == NULL)
{
#ifdef Py_DEBUG #ifdef Py_DEBUG
fatal_1350(self, "unghostify", fatal_1350(self, "unghostify",
"is not in the cache despite that we just " "is not in the cache despite that we just "
"unghostified it"); "unghostified it");
#else #else
PyErr_Format(PyExc_SystemError, "object at %p with type " PyErr_Format(PyExc_SystemError, "object at %p with type "
"%.200s not in the cache despite that we just " "%.200s not in the cache despite that we just "
"unghostified it", self, self->ob_type->tp_name); "unghostified it", self, self->ob_type->tp_name);
return -1; return -1;
#endif #endif
} }
} }
return 1; return 1;
} }
/****************************************************************************/ /****************************************************************************/
...@@ -129,72 +133,58 @@ static PyTypeObject Pertype; ...@@ -129,72 +133,58 @@ static PyTypeObject Pertype;
static void static void
accessed(cPersistentObject *self) accessed(cPersistentObject *self)
{ {
/* Do nothing unless the object is in a cache and not a ghost. */ /* Do nothing unless the object is in a cache and not a ghost. */
if (self->cache && self->state >= 0 && self->ring.r_next) if (self->cache && self->state >= 0 && self->ring.r_next)
ring_move_to_head(&self->cache->ring_home, &self->ring); ring_move_to_head(&self->cache->ring_home, &self->ring);
}
static void
unlink_from_ring(cPersistentObject *self)
{
/* If the cache has been cleared, then a non-ghost object
isn't in the ring any longer.
*/
if (self->ring.r_next == NULL)
return;
/* if we're ghostifying an object, we better have some non-ghosts */
assert(self->cache->non_ghost_count > 0);
self->cache->non_ghost_count--;
self->cache->total_estimated_size -=
_estimated_size_in_bytes(self->estimated_size);
ring_del(&self->ring);
} }
static void static void
ghostify(cPersistentObject *self) ghostify(cPersistentObject *self)
{ {
PyObject **dictptr; PyObject **dictptr;
/* are we already a ghost? */ /* are we already a ghost? */
if (self->state == cPersistent_GHOST_STATE) if (self->state == cPersistent_GHOST_STATE)
return; return;
/* Is it ever possible to not have a cache? */ /* Is it ever possible to not have a cache? */
if (self->cache == NULL) { if (self->cache == NULL)
self->state = cPersistent_GHOST_STATE; {
return; self->state = cPersistent_GHOST_STATE;
return;
} }
if (self->ring.r_next == NULL) { if (self->ring.r_next == NULL)
/* There's no way to raise an error in this routine. */ {
/* There's no way to raise an error in this routine. */
#ifdef Py_DEBUG #ifdef Py_DEBUG
fatal_1350(self, "ghostify", "claims to be in a cache but isn't"); fatal_1350(self, "ghostify", "claims to be in a cache but isn't");
#else #else
return; return;
#endif #endif
} }
/* If we're ghostifying an object, we better have some non-ghosts. */ /* If we're ghostifying an object, we better have some non-ghosts. */
assert(self->cache->non_ghost_count > 0); assert(self->cache->non_ghost_count > 0);
self->cache->non_ghost_count--; self->cache->non_ghost_count--;
self->cache->total_estimated_size -= self->cache->total_estimated_size -=
_estimated_size_in_bytes(self->estimated_size); _estimated_size_in_bytes(self->estimated_size);
ring_del(&self->ring); ring_del(&self->ring);
self->state = cPersistent_GHOST_STATE; self->state = cPersistent_GHOST_STATE;
dictptr = _PyObject_GetDictPtr((PyObject *)self); dictptr = _PyObject_GetDictPtr((PyObject *)self);
if (dictptr && *dictptr) { if (dictptr && *dictptr)
Py_DECREF(*dictptr); {
*dictptr = NULL; Py_DECREF(*dictptr);
*dictptr = NULL;
} }
/* We remove the reference to the just ghosted object that the ring /* We remove the reference to the just ghosted object that the ring
* holds. Note that the dictionary of oids->objects has an uncounted * holds. Note that the dictionary of oids->objects has an uncounted
* reference, so if the ring's reference was the only one, this frees * reference, so if the ring's reference was the only one, this frees
* the ghost object. Note further that the object's dealloc knows to * the ghost object. Note further that the object's dealloc knows to
* inform the dictionary that it is going away. * inform the dictionary that it is going away.
*/ */
Py_DECREF(self); Py_DECREF(self);
} }
static int static int
...@@ -202,31 +192,32 @@ changed(cPersistentObject *self) ...@@ -202,31 +192,32 @@ changed(cPersistentObject *self)
{ {
if ((self->state == cPersistent_UPTODATE_STATE || if ((self->state == cPersistent_UPTODATE_STATE ||
self->state == cPersistent_STICKY_STATE) self->state == cPersistent_STICKY_STATE)
&& self->jar) && self->jar)
{ {
PyObject *meth, *arg, *result; PyObject *meth, *arg, *result;
static PyObject *s_register; static PyObject *s_register;
if (s_register == NULL) if (s_register == NULL)
s_register = PyString_InternFromString("register"); s_register = PyString_InternFromString("register");
meth = PyObject_GetAttr((PyObject *)self->jar, s_register); meth = PyObject_GetAttr((PyObject *)self->jar, s_register);
if (meth == NULL) if (meth == NULL)
return -1; return -1;
arg = PyTuple_New(1); arg = PyTuple_New(1);
if (arg == NULL) { if (arg == NULL)
Py_DECREF(meth); {
return -1; Py_DECREF(meth);
} return -1;
Py_INCREF(self); }
PyTuple_SET_ITEM(arg, 0, (PyObject *)self); Py_INCREF(self);
result = PyEval_CallObject(meth, arg); PyTuple_SET_ITEM(arg, 0, (PyObject *)self);
Py_DECREF(arg); result = PyEval_CallObject(meth, arg);
Py_DECREF(meth); Py_DECREF(arg);
if (result == NULL) Py_DECREF(meth);
return -1; if (result == NULL)
Py_DECREF(result); return -1;
Py_DECREF(result);
self->state = cPersistent_CHANGED_STATE;
self->state = cPersistent_CHANGED_STATE;
} }
return 0; return 0;
...@@ -235,30 +226,32 @@ changed(cPersistentObject *self) ...@@ -235,30 +226,32 @@ changed(cPersistentObject *self)
static PyObject * static PyObject *
Per__p_deactivate(cPersistentObject *self) Per__p_deactivate(cPersistentObject *self)
{ {
if (self->state == cPersistent_UPTODATE_STATE && self->jar) { if (self->state == cPersistent_UPTODATE_STATE && self->jar)
PyObject **dictptr = _PyObject_GetDictPtr((PyObject *)self); {
if (dictptr && *dictptr) { PyObject **dictptr = _PyObject_GetDictPtr((PyObject *)self);
Py_DECREF(*dictptr); if (dictptr && *dictptr)
*dictptr = NULL; {
} Py_DECREF(*dictptr);
/* Note that we need to set to ghost state unless we are *dictptr = NULL;
called directly. Methods that override this need to }
do the same! */ /* Note that we need to set to ghost state unless we are
ghostify(self); called directly. Methods that override this need to
do the same! */
ghostify(self);
} }
Py_INCREF(Py_None); Py_INCREF(Py_None);
return Py_None; return Py_None;
} }
static PyObject * static PyObject *
Per__p_activate(cPersistentObject *self) Per__p_activate(cPersistentObject *self)
{ {
if (unghostify(self) < 0) if (unghostify(self) < 0)
return NULL; return NULL;
Py_INCREF(Py_None); Py_INCREF(Py_None);
return Py_None; return Py_None;
} }
static int Per_set_changed(cPersistentObject *self, PyObject *v); static int Per_set_changed(cPersistentObject *self, PyObject *v);
...@@ -266,278 +259,298 @@ static int Per_set_changed(cPersistentObject *self, PyObject *v); ...@@ -266,278 +259,298 @@ static int Per_set_changed(cPersistentObject *self, PyObject *v);
static PyObject * static PyObject *
Per__p_invalidate(cPersistentObject *self) Per__p_invalidate(cPersistentObject *self)
{ {
signed char old_state = self->state; signed char old_state = self->state;
if (old_state != cPersistent_GHOST_STATE) { if (old_state != cPersistent_GHOST_STATE)
if (Per_set_changed(self, NULL) < 0) {
return NULL; if (Per_set_changed(self, NULL) < 0)
ghostify(self); return NULL;
ghostify(self);
} }
Py_INCREF(Py_None); Py_INCREF(Py_None);
return Py_None; return Py_None;
} }
static PyObject * static PyObject *
pickle_slotnames(PyTypeObject *cls) pickle_slotnames(PyTypeObject *cls)
{ {
PyObject *slotnames; PyObject *slotnames;
slotnames = PyDict_GetItem(cls->tp_dict, py___slotnames__); slotnames = PyDict_GetItem(cls->tp_dict, py___slotnames__);
if (slotnames) { if (slotnames)
Py_INCREF(slotnames); {
return slotnames; Py_INCREF(slotnames);
return slotnames;
} }
slotnames = PyObject_CallFunctionObjArgs(copy_reg_slotnames, slotnames = PyObject_CallFunctionObjArgs(copy_reg_slotnames,
(PyObject*)cls, NULL); (PyObject*)cls, NULL);
if (slotnames && !(slotnames == Py_None || PyList_Check(slotnames))) { if (slotnames && !(slotnames == Py_None || PyList_Check(slotnames)))
PyErr_SetString(PyExc_TypeError, {
"copy_reg._slotnames didn't return a list or None"); PyErr_SetString(PyExc_TypeError,
Py_DECREF(slotnames); "copy_reg._slotnames didn't return a list or None");
return NULL; Py_DECREF(slotnames);
return NULL;
} }
return slotnames; return slotnames;
} }
static PyObject * static PyObject *
pickle_copy_dict(PyObject *state) pickle_copy_dict(PyObject *state)
{ {
PyObject *copy, *key, *value; PyObject *copy, *key, *value;
char *ckey; char *ckey;
Py_ssize_t pos = 0; Py_ssize_t pos = 0;
copy = PyDict_New(); copy = PyDict_New();
if (!copy) if (!copy)
return NULL; return NULL;
if (!state) if (!state)
return copy; return copy;
while (PyDict_Next(state, &pos, &key, &value)) { while (PyDict_Next(state, &pos, &key, &value))
if (key && PyString_Check(key)) { {
ckey = PyString_AS_STRING(key); if (key && PyString_Check(key))
if (*ckey == '_' && {
(ckey[1] == 'v' || ckey[1] == 'p') && ckey = PyString_AS_STRING(key);
ckey[2] == '_') if (*ckey == '_' &&
/* skip volatile and persistent */ (ckey[1] == 'v' || ckey[1] == 'p') &&
continue; ckey[2] == '_')
/* skip volatile and persistent */
continue;
} }
if (PyObject_SetItem(copy, key, value) < 0) if (PyObject_SetItem(copy, key, value) < 0)
goto err; goto err;
} }
return copy; return copy;
err: err:
Py_DECREF(copy); Py_DECREF(copy);
return NULL; return NULL;
} }
static char pickle___getstate__doc[] = static char pickle___getstate__doc[] =
"Get the object serialization state\n" "Get the object serialization state\n"
"\n" "\n"
"If the object has no assigned slots and has no instance dictionary, then \n" "If the object has no assigned slots and has no instance dictionary, then \n"
"None is returned.\n" "None is returned.\n"
"\n" "\n"
"If the object has no assigned slots and has an instance dictionary, then \n" "If the object has no assigned slots and has an instance dictionary, then \n"
"the a copy of the instance dictionary is returned. The copy has any items \n" "the a copy of the instance dictionary is returned. The copy has any items \n"
"with names starting with '_v_' or '_p_' ommitted.\n" "with names starting with '_v_' or '_p_' ommitted.\n"
"\n" "\n"
"If the object has assigned slots, then a two-element tuple is returned. \n" "If the object has assigned slots, then a two-element tuple is returned. \n"
"The first element is either None or a copy of the instance dictionary, \n" "The first element is either None or a copy of the instance dictionary, \n"
"as described above. The second element is a dictionary with items \n" "as described above. The second element is a dictionary with items \n"
"for each of the assigned slots.\n" "for each of the assigned slots.\n"
; ;
static PyObject * static PyObject *
pickle___getstate__(PyObject *self) pickle___getstate__(PyObject *self)
{ {
PyObject *slotnames=NULL, *slots=NULL, *state=NULL; PyObject *slotnames=NULL, *slots=NULL, *state=NULL;
PyObject **dictp; PyObject **dictp;
int n=0; int n=0;
slotnames = pickle_slotnames(self->ob_type); slotnames = pickle_slotnames(self->ob_type);
if (!slotnames) if (!slotnames)
return NULL; return NULL;
dictp = _PyObject_GetDictPtr(self); dictp = _PyObject_GetDictPtr(self);
if (dictp) if (dictp)
state = pickle_copy_dict(*dictp); state = pickle_copy_dict(*dictp);
else { else
state = Py_None; {
Py_INCREF(state); state = Py_None;
Py_INCREF(state);
} }
if (slotnames != Py_None) { if (slotnames != Py_None)
int i; {
int i;
slots = PyDict_New();
if (!slots) slots = PyDict_New();
goto end; if (!slots)
goto end;
for (i = 0; i < PyList_GET_SIZE(slotnames); i++) {
PyObject *name, *value; for (i = 0; i < PyList_GET_SIZE(slotnames); i++)
char *cname; {
PyObject *name, *value;
name = PyList_GET_ITEM(slotnames, i); char *cname;
if (PyString_Check(name)) {
cname = PyString_AS_STRING(name); name = PyList_GET_ITEM(slotnames, i);
if (*cname == '_' && if (PyString_Check(name))
(cname[1] == 'v' || cname[1] == 'p') && {
cname[2] == '_') cname = PyString_AS_STRING(name);
/* skip volatile and persistent */ if (*cname == '_' &&
continue; (cname[1] == 'v' || cname[1] == 'p') &&
cname[2] == '_')
/* skip volatile and persistent */
continue;
} }
/* Unclear: Will this go through our getattr hook? */ /* Unclear: Will this go through our getattr hook? */
value = PyObject_GetAttr(self, name); value = PyObject_GetAttr(self, name);
if (value == NULL) if (value == NULL)
PyErr_Clear(); PyErr_Clear();
else { else
int err = PyDict_SetItem(slots, name, value); {
Py_DECREF(value); int err = PyDict_SetItem(slots, name, value);
if (err < 0) Py_DECREF(value);
goto end; if (err < 0)
n++; goto end;
n++;
} }
} }
} }
if (n) if (n)
state = Py_BuildValue("(NO)", state, slots); state = Py_BuildValue("(NO)", state, slots);
end: end:
Py_XDECREF(slotnames); Py_XDECREF(slotnames);
Py_XDECREF(slots); Py_XDECREF(slots);
return state; return state;
} }
static int static int
pickle_setattrs_from_dict(PyObject *self, PyObject *dict) pickle_setattrs_from_dict(PyObject *self, PyObject *dict)
{ {
PyObject *key, *value; PyObject *key, *value;
Py_ssize_t pos = 0; Py_ssize_t pos = 0;
if (!PyDict_Check(dict)) { if (!PyDict_Check(dict))
PyErr_SetString(PyExc_TypeError, "Expected dictionary"); {
return -1; PyErr_SetString(PyExc_TypeError, "Expected dictionary");
return -1;
} }
while (PyDict_Next(dict, &pos, &key, &value)) { while (PyDict_Next(dict, &pos, &key, &value))
if (PyObject_SetAttr(self, key, value) < 0) {
return -1; if (PyObject_SetAttr(self, key, value) < 0)
return -1;
} }
return 0; return 0;
} }
static char pickle___setstate__doc[] = static char pickle___setstate__doc[] =
"Set the object serialization state\n\n" "Set the object serialization state\n\n"
"The state should be in one of 3 forms:\n\n" "The state should be in one of 3 forms:\n\n"
"- None\n\n" "- None\n\n"
" Ignored\n\n" " Ignored\n\n"
"- A dictionary\n\n" "- A dictionary\n\n"
" In this case, the object's instance dictionary will be cleared and \n" " In this case, the object's instance dictionary will be cleared and \n"
" updated with the new state.\n\n" " updated with the new state.\n\n"
"- A two-tuple with a string as the first element. \n\n" "- A two-tuple with a string as the first element. \n\n"
" In this case, the method named by the string in the first element will be\n" " In this case, the method named by the string in the first element will\n"
" called with the second element.\n\n" " be called with the second element.\n\n"
" This form supports migration of data formats.\n\n" " This form supports migration of data formats.\n\n"
"- A two-tuple with None or a Dictionary as the first element and\n" "- A two-tuple with None or a Dictionary as the first element and\n"
" with a dictionary as the second element.\n\n" " with a dictionary as the second element.\n\n"
" If the first element is not None, then the object's instance dictionary \n" " If the first element is not None, then the object's instance dictionary \n"
" will be cleared and updated with the value.\n\n" " will be cleared and updated with the value.\n\n"
" The items in the second element will be assigned as attributes.\n" " The items in the second element will be assigned as attributes.\n"
; ;
static PyObject * static PyObject *
pickle___setstate__(PyObject *self, PyObject *state) pickle___setstate__(PyObject *self, PyObject *state)
{ {
PyObject *slots=NULL; PyObject *slots=NULL;
if (PyTuple_Check(state)) { if (PyTuple_Check(state))
if (!PyArg_ParseTuple(state, "OO:__setstate__", &state, &slots)) {
return NULL; if (!PyArg_ParseTuple(state, "OO:__setstate__", &state, &slots))
return NULL;
} }
if (state != Py_None) { if (state != Py_None)
PyObject **dict; {
PyObject **dict;
dict = _PyObject_GetDictPtr(self);
if (dict) { dict = _PyObject_GetDictPtr(self);
if (!*dict) { if (dict)
*dict = PyDict_New(); {
if (!*dict) if (!*dict)
return NULL; {
*dict = PyDict_New();
if (!*dict)
return NULL;
} }
} }
if (*dict) { if (*dict)
PyDict_Clear(*dict); {
if (PyDict_Update(*dict, state) < 0) PyDict_Clear(*dict);
return NULL; if (PyDict_Update(*dict, state) < 0)
return NULL;
} }
else if (pickle_setattrs_from_dict(self, state) < 0) else if (pickle_setattrs_from_dict(self, state) < 0)
return NULL; return NULL;
} }
if (slots && pickle_setattrs_from_dict(self, slots) < 0) if (slots && pickle_setattrs_from_dict(self, slots) < 0)
return NULL; return NULL;
Py_INCREF(Py_None); Py_INCREF(Py_None);
return Py_None; return Py_None;
} }
static char pickle___reduce__doc[] = static char pickle___reduce__doc[] =
"Reduce an object to contituent parts for serialization\n" "Reduce an object to contituent parts for serialization\n"
; ;
static PyObject * static PyObject *
pickle___reduce__(PyObject *self) pickle___reduce__(PyObject *self)
{ {
PyObject *args=NULL, *bargs=NULL, *state=NULL, *getnewargs=NULL; PyObject *args=NULL, *bargs=NULL, *state=NULL, *getnewargs=NULL;
int l, i; int l, i;
getnewargs = PyObject_GetAttr(self, py___getnewargs__); getnewargs = PyObject_GetAttr(self, py___getnewargs__);
if (getnewargs) { if (getnewargs)
bargs = PyObject_CallFunctionObjArgs(getnewargs, NULL); {
Py_DECREF(getnewargs); bargs = PyObject_CallFunctionObjArgs(getnewargs, NULL);
if (!bargs) Py_DECREF(getnewargs);
return NULL; if (!bargs)
l = PyTuple_Size(bargs); return NULL;
if (l < 0) l = PyTuple_Size(bargs);
goto end; if (l < 0)
goto end;
} }
else { else
PyErr_Clear(); {
l = 0; PyErr_Clear();
l = 0;
} }
args = PyTuple_New(l+1); args = PyTuple_New(l+1);
if (args == NULL) if (args == NULL)
goto end; goto end;
Py_INCREF(self->ob_type); Py_INCREF(self->ob_type);
PyTuple_SET_ITEM(args, 0, (PyObject*)(self->ob_type)); PyTuple_SET_ITEM(args, 0, (PyObject*)(self->ob_type));
for (i = 0; i < l; i++) { for (i = 0; i < l; i++)
Py_INCREF(PyTuple_GET_ITEM(bargs, i)); {
PyTuple_SET_ITEM(args, i+1, PyTuple_GET_ITEM(bargs, i)); Py_INCREF(PyTuple_GET_ITEM(bargs, i));
PyTuple_SET_ITEM(args, i+1, PyTuple_GET_ITEM(bargs, i));
} }
state = PyObject_CallMethodObjArgs(self, py___getstate__, NULL); state = PyObject_CallMethodObjArgs(self, py___getstate__, NULL);
if (!state) if (!state)
goto end; goto end;
state = Py_BuildValue("(OON)", __newobj__, args, state); state = Py_BuildValue("(OON)", __newobj__, args, state);
end: end:
Py_XDECREF(bargs); Py_XDECREF(bargs);
Py_XDECREF(args); Py_XDECREF(args);
return state; return state;
} }
...@@ -554,13 +567,13 @@ pickle___reduce__(PyObject *self) ...@@ -554,13 +567,13 @@ pickle___reduce__(PyObject *self)
static PyObject * static PyObject *
Per__getstate__(cPersistentObject *self) Per__getstate__(cPersistentObject *self)
{ {
/* TODO: Should it be an error to call __getstate__() on a ghost? */ /* TODO: Should it be an error to call __getstate__() on a ghost? */
if (unghostify(self) < 0) if (unghostify(self) < 0)
return NULL; return NULL;
/* TODO: should we increment stickyness? Tim doesn't understand that /* TODO: should we increment stickyness? Tim doesn't understand that
question. S*/ question. S*/
return pickle___getstate__((PyObject*)self); return pickle___getstate__((PyObject*)self);
} }
/* The Persistent base type provides a traverse function, but not a /* The Persistent base type provides a traverse function, but not a
...@@ -578,34 +591,48 @@ Per__getstate__(cPersistentObject *self) ...@@ -578,34 +591,48 @@ Per__getstate__(cPersistentObject *self)
static void static void
Per_dealloc(cPersistentObject *self) Per_dealloc(cPersistentObject *self)
{ {
if (self->state >= 0) if (self->state >= 0)
unlink_from_ring(self); {
if (self->cache) /* If the cache has been cleared, then a non-ghost object
cPersistenceCAPI->percachedel(self->cache, self->oid); isn't in the ring any longer.
Py_XDECREF(self->cache); */
Py_XDECREF(self->jar); if (self->ring.r_next != NULL)
Py_XDECREF(self->oid); {
self->ob_type->tp_free(self); /* if we're ghostifying an object, we better have some non-ghosts */
assert(self->cache->non_ghost_count > 0);
self->cache->non_ghost_count--;
self->cache->total_estimated_size -=
_estimated_size_in_bytes(self->estimated_size);
ring_del(&self->ring);
}
}
if (self->cache)
cPersistenceCAPI->percachedel(self->cache, self->oid);
Py_XDECREF(self->cache);
Py_XDECREF(self->jar);
Py_XDECREF(self->oid);
self->ob_type->tp_free(self);
} }
static int static int
Per_traverse(cPersistentObject *self, visitproc visit, void *arg) Per_traverse(cPersistentObject *self, visitproc visit, void *arg)
{ {
int err; int err;
#define VISIT(SLOT) \ #define VISIT(SLOT) \
if (SLOT) { \ if (SLOT) { \
err = visit((PyObject *)(SLOT), arg); \ err = visit((PyObject *)(SLOT), arg); \
if (err) \ if (err) \
return err; \ return err; \
} }
VISIT(self->jar); VISIT(self->jar);
VISIT(self->oid); VISIT(self->oid);
VISIT(self->cache); VISIT(self->cache);
#undef VISIT #undef VISIT
return 0; return 0;
} }
/* convert_name() returns a new reference to a string name /* convert_name() returns a new reference to a string name
...@@ -616,20 +643,23 @@ static PyObject * ...@@ -616,20 +643,23 @@ static PyObject *
convert_name(PyObject *name) convert_name(PyObject *name)
{ {
#ifdef Py_USING_UNICODE #ifdef Py_USING_UNICODE
/* The Unicode to string conversion is done here because the /* The Unicode to string conversion is done here because the
existing tp_setattro slots expect a string object as name existing tp_setattro slots expect a string object as name
and we wouldn't want to break those. */ and we wouldn't want to break those. */
if (PyUnicode_Check(name)) { if (PyUnicode_Check(name))
name = PyUnicode_AsEncodedString(name, NULL, NULL); {
name = PyUnicode_AsEncodedString(name, NULL, NULL);
} }
else else
#endif #endif
if (!PyString_Check(name)) { if (!PyString_Check(name))
PyErr_SetString(PyExc_TypeError, "attribute name must be a string"); {
return NULL; PyErr_SetString(PyExc_TypeError, "attribute name must be a string");
} else return NULL;
Py_INCREF(name); }
return name; else
Py_INCREF(name);
return name;
} }
/* Returns true if the object requires unghostification. /* Returns true if the object requires unghostification.
...@@ -646,375 +676,400 @@ convert_name(PyObject *name) ...@@ -646,375 +676,400 @@ convert_name(PyObject *name)
static int static int
unghost_getattr(const char *s) unghost_getattr(const char *s)
{ {
if (*s++ != '_') if (*s++ != '_')
return 1; return 1;
if (*s == 'p') { if (*s == 'p')
s++; {
if (*s == '_') s++;
return 0; /* _p_ */ if (*s == '_')
else return 0; /* _p_ */
return 1; else
return 1;
} }
else if (*s == '_') { else if (*s == '_')
s++; {
switch (*s) { s++;
case 'c': switch (*s)
return strcmp(s, "class__"); {
case 'd': case 'c':
s++; return strcmp(s, "class__");
if (!strcmp(s, "el__")) case 'd':
return 0; /* __del__ */ s++;
if (!strcmp(s, "ict__")) if (!strcmp(s, "el__"))
return 0; /* __dict__ */ return 0; /* __del__ */
return 1; if (!strcmp(s, "ict__"))
case 'o': return 0; /* __dict__ */
return strcmp(s, "of__"); return 1;
case 's': case 'o':
return strcmp(s, "setstate__"); return strcmp(s, "of__");
default: case 's':
return 1; return strcmp(s, "setstate__");
} default:
return 1;
}
} }
return 1; return 1;
} }
static PyObject* static PyObject*
Per_getattro(cPersistentObject *self, PyObject *name) Per_getattro(cPersistentObject *self, PyObject *name)
{ {
PyObject *result = NULL; /* guilty until proved innocent */ PyObject *result = NULL; /* guilty until proved innocent */
char *s; char *s;
name = convert_name(name); name = convert_name(name);
if (!name) if (!name)
goto Done; goto Done;
s = PyString_AS_STRING(name); s = PyString_AS_STRING(name);
if (unghost_getattr(s)) { if (unghost_getattr(s))
if (unghostify(self) < 0) {
goto Done; if (unghostify(self) < 0)
accessed(self); goto Done;
accessed(self);
} }
result = PyObject_GenericGetAttr((PyObject *)self, name); result = PyObject_GenericGetAttr((PyObject *)self, name);
Done: Done:
Py_XDECREF(name); Py_XDECREF(name);
return result; return result;
} }
/* Exposed as _p_getattr method. Test whether base getattr should be used */ /* Exposed as _p_getattr method. Test whether base getattr should be used */
static PyObject * static PyObject *
Per__p_getattr(cPersistentObject *self, PyObject *name) Per__p_getattr(cPersistentObject *self, PyObject *name)
{ {
PyObject *result = NULL; /* guilty until proved innocent */ PyObject *result = NULL; /* guilty until proved innocent */
char *s; char *s;
name = convert_name(name); name = convert_name(name);
if (!name) if (!name)
goto Done; goto Done;
s = PyString_AS_STRING(name); s = PyString_AS_STRING(name);
if (*s != '_' || unghost_getattr(s)) { if (*s != '_' || unghost_getattr(s))
if (unghostify(self) < 0) {
goto Done; if (unghostify(self) < 0)
accessed(self); goto Done;
result = Py_False; accessed(self);
result = Py_False;
} }
else else
result = Py_True; result = Py_True;
Py_INCREF(result); Py_INCREF(result);
Done: Done:
Py_XDECREF(name); Py_XDECREF(name);
return result; return result;
} }
/* /*
TODO: we should probably not allow assignment of __class__ and __dict__. TODO: we should probably not allow assignment of __class__ and __dict__.
*/ */
static int static int
Per_setattro(cPersistentObject *self, PyObject *name, PyObject *v) Per_setattro(cPersistentObject *self, PyObject *name, PyObject *v)
{ {
int result = -1; /* guilty until proved innocent */ int result = -1; /* guilty until proved innocent */
char *s; char *s;
name = convert_name(name); name = convert_name(name);
if (!name) if (!name)
goto Done; goto Done;
s = PyString_AS_STRING(name); s = PyString_AS_STRING(name);
if (strncmp(s, "_p_", 3) != 0) { if (strncmp(s, "_p_", 3) != 0)
if (unghostify(self) < 0) {
goto Done; if (unghostify(self) < 0)
accessed(self); goto Done;
if (strncmp(s, "_v_", 3) != 0 accessed(self);
&& self->state != cPersistent_CHANGED_STATE) { if (strncmp(s, "_v_", 3) != 0
if (changed(self) < 0) && self->state != cPersistent_CHANGED_STATE)
goto Done; {
} if (changed(self) < 0)
goto Done;
}
} }
result = PyObject_GenericSetAttr((PyObject *)self, name, v); result = PyObject_GenericSetAttr((PyObject *)self, name, v);
Done: Done:
Py_XDECREF(name); Py_XDECREF(name);
return result; return result;
} }
static int static int
Per_p_set_or_delattro(cPersistentObject *self, PyObject *name, PyObject *v) Per_p_set_or_delattro(cPersistentObject *self, PyObject *name, PyObject *v)
{ {
int result = -1; /* guilty until proved innocent */ int result = -1; /* guilty until proved innocent */
char *s; char *s;
name = convert_name(name); name = convert_name(name);
if (!name) if (!name)
goto Done; goto Done;
s = PyString_AS_STRING(name); s = PyString_AS_STRING(name);
if (strncmp(s, "_p_", 3)) { if (strncmp(s, "_p_", 3))
if (unghostify(self) < 0) {
goto Done; if (unghostify(self) < 0)
accessed(self); goto Done;
accessed(self);
result = 0; result = 0;
} }
else { else
if (PyObject_GenericSetAttr((PyObject *)self, name, v) < 0) {
goto Done; if (PyObject_GenericSetAttr((PyObject *)self, name, v) < 0)
result = 1; goto Done;
result = 1;
} }
Done: Done:
Py_XDECREF(name); Py_XDECREF(name);
return result; return result;
} }
static PyObject * static PyObject *
Per__p_setattr(cPersistentObject *self, PyObject *args) Per__p_setattr(cPersistentObject *self, PyObject *args)
{ {
PyObject *name, *v, *result; PyObject *name, *v, *result;
int r; int r;
if (!PyArg_ParseTuple(args, "OO:_p_setattr", &name, &v)) if (!PyArg_ParseTuple(args, "OO:_p_setattr", &name, &v))
return NULL; return NULL;
r = Per_p_set_or_delattro(self, name, v); r = Per_p_set_or_delattro(self, name, v);
if (r < 0) if (r < 0)
return NULL; return NULL;
result = r ? Py_True : Py_False; result = r ? Py_True : Py_False;
Py_INCREF(result); Py_INCREF(result);
return result; return result;
} }
static PyObject * static PyObject *
Per__p_delattr(cPersistentObject *self, PyObject *name) Per__p_delattr(cPersistentObject *self, PyObject *name)
{ {
int r; int r;
PyObject *result; PyObject *result;
r = Per_p_set_or_delattro(self, name, NULL); r = Per_p_set_or_delattro(self, name, NULL);
if (r < 0) if (r < 0)
return NULL; return NULL;
result = r ? Py_True : Py_False; result = r ? Py_True : Py_False;
Py_INCREF(result); Py_INCREF(result);
return result; return result;
} }
static PyObject * static PyObject *
Per_get_changed(cPersistentObject *self) Per_get_changed(cPersistentObject *self)
{ {
if (self->state < 0) { if (self->state < 0)
Py_INCREF(Py_None); {
return Py_None; Py_INCREF(Py_None);
return Py_None;
} }
return PyBool_FromLong(self->state == cPersistent_CHANGED_STATE); return PyBool_FromLong(self->state == cPersistent_CHANGED_STATE);
} }
static int static int
Per_set_changed(cPersistentObject *self, PyObject *v) Per_set_changed(cPersistentObject *self, PyObject *v)
{ {
int deactivate = 0; int deactivate = 0;
int true; int true;
if (!v) { if (!v)
/* delattr is used to invalidate an object even if it has changed. */ {
if (self->state != cPersistent_GHOST_STATE) /* delattr is used to invalidate an object even if it has changed. */
self->state = cPersistent_UPTODATE_STATE; if (self->state != cPersistent_GHOST_STATE)
deactivate = 1; self->state = cPersistent_UPTODATE_STATE;
deactivate = 1;
} }
else if (v == Py_None) else if (v == Py_None)
deactivate = 1; deactivate = 1;
if (deactivate) { if (deactivate)
PyObject *res, *meth; {
meth = PyObject_GetAttr((PyObject *)self, py__p_deactivate); PyObject *res, *meth;
if (meth == NULL) meth = PyObject_GetAttr((PyObject *)self, py__p_deactivate);
return -1; if (meth == NULL)
res = PyObject_CallObject(meth, NULL); return -1;
if (res) res = PyObject_CallObject(meth, NULL);
Py_DECREF(res); if (res)
else { Py_DECREF(res);
/* an error occured in _p_deactivate(). else
{
It's not clear what we should do here. The code is /* an error occured in _p_deactivate().
obviously ignoring the exception, but it shouldn't return
0 for a getattr and set an exception. The simplest change It's not clear what we should do here. The code is
is to clear the exception, but that simply masks the obviously ignoring the exception, but it shouldn't return
error. 0 for a getattr and set an exception. The simplest change
is to clear the exception, but that simply masks the
This prints an error to stderr just like exceptions in error.
__del__(). It would probably be better to log it but that
would be painful from C. This prints an error to stderr just like exceptions in
*/ __del__(). It would probably be better to log it but that
PyErr_WriteUnraisable(meth); would be painful from C.
} */
Py_DECREF(meth); PyErr_WriteUnraisable(meth);
return 0; }
Py_DECREF(meth);
return 0;
} }
/* !deactivate. If passed a true argument, mark self as changed (starting /* !deactivate. If passed a true argument, mark self as changed (starting
* with ZODB 3.6, that includes activating the object if it's a ghost). * with ZODB 3.6, that includes activating the object if it's a ghost).
* If passed a false argument, and the object isn't a ghost, set the * If passed a false argument, and the object isn't a ghost, set the
* state as up-to-date. * state as up-to-date.
*/ */
true = PyObject_IsTrue(v); true = PyObject_IsTrue(v);
if (true == -1) if (true == -1)
return -1; return -1;
if (true) { if (true)
if (self->state < 0) { {
if (self->state < 0)
{
if (unghostify(self) < 0) if (unghostify(self) < 0)
return -1; return -1;
} }
return changed(self); return changed(self);
} }
/* We were passed a false, non-None argument. If we're not a ghost, /* We were passed a false, non-None argument. If we're not a ghost,
* mark self as up-to-date. * mark self as up-to-date.
*/ */
if (self->state >= 0) if (self->state >= 0)
self->state = cPersistent_UPTODATE_STATE; self->state = cPersistent_UPTODATE_STATE;
return 0; return 0;
} }
static PyObject * static PyObject *
Per_get_oid(cPersistentObject *self) Per_get_oid(cPersistentObject *self)
{ {
PyObject *oid = self->oid ? self->oid : Py_None; PyObject *oid = self->oid ? self->oid : Py_None;
Py_INCREF(oid); Py_INCREF(oid);
return oid; return oid;
} }
static int static int
Per_set_oid(cPersistentObject *self, PyObject *v) Per_set_oid(cPersistentObject *self, PyObject *v)
{ {
if (self->cache) { if (self->cache)
int result; {
int result;
if (v == NULL) {
PyErr_SetString(PyExc_ValueError, if (v == NULL)
"can't delete _p_oid of cached object"); {
return -1; PyErr_SetString(PyExc_ValueError,
} "can't delete _p_oid of cached object");
if (PyObject_Cmp(self->oid, v, &result) < 0) return -1;
return -1; }
if (result) { if (PyObject_Cmp(self->oid, v, &result) < 0)
PyErr_SetString(PyExc_ValueError, return -1;
"can not change _p_oid of cached object"); if (result)
return -1; {
} PyErr_SetString(PyExc_ValueError,
"can not change _p_oid of cached object");
return -1;
}
} }
Py_XDECREF(self->oid); Py_XDECREF(self->oid);
Py_XINCREF(v); Py_XINCREF(v);
self->oid = v; self->oid = v;
return 0; return 0;
} }
static PyObject * static PyObject *
Per_get_jar(cPersistentObject *self) Per_get_jar(cPersistentObject *self)
{ {
PyObject *jar = self->jar ? self->jar : Py_None; PyObject *jar = self->jar ? self->jar : Py_None;
Py_INCREF(jar); Py_INCREF(jar);
return jar; return jar;
} }
static int static int
Per_set_jar(cPersistentObject *self, PyObject *v) Per_set_jar(cPersistentObject *self, PyObject *v)
{ {
if (self->cache) { if (self->cache)
int result; {
int result;
if (v == NULL) {
PyErr_SetString(PyExc_ValueError, if (v == NULL)
"can't delete _p_jar of cached object"); {
return -1; PyErr_SetString(PyExc_ValueError,
} "can't delete _p_jar of cached object");
if (PyObject_Cmp(self->jar, v, &result) < 0) return -1;
return -1; }
if (result) { if (PyObject_Cmp(self->jar, v, &result) < 0)
PyErr_SetString(PyExc_ValueError, return -1;
"can not change _p_jar of cached object"); if (result)
return -1; {
} PyErr_SetString(PyExc_ValueError,
"can not change _p_jar of cached object");
return -1;
}
} }
Py_XDECREF(self->jar); Py_XDECREF(self->jar);
Py_XINCREF(v); Py_XINCREF(v);
self->jar = v; self->jar = v;
return 0; return 0;
} }
static PyObject * static PyObject *
Per_get_serial(cPersistentObject *self) Per_get_serial(cPersistentObject *self)
{ {
return PyString_FromStringAndSize(self->serial, 8); return PyString_FromStringAndSize(self->serial, 8);
} }
static int static int
Per_set_serial(cPersistentObject *self, PyObject *v) Per_set_serial(cPersistentObject *self, PyObject *v)
{ {
if (v) { if (v)
if (PyString_Check(v) && PyString_GET_SIZE(v) == 8) {
memcpy(self->serial, PyString_AS_STRING(v), 8); if (PyString_Check(v) && PyString_GET_SIZE(v) == 8)
else { memcpy(self->serial, PyString_AS_STRING(v), 8);
PyErr_SetString(PyExc_ValueError, else
"_p_serial must be an 8-character string"); {
return -1; PyErr_SetString(PyExc_ValueError,
} "_p_serial must be an 8-character string");
} else return -1;
memset(self->serial, 0, 8); }
return 0; }
else
memset(self->serial, 0, 8);
return 0;
} }
static PyObject * static PyObject *
Per_get_mtime(cPersistentObject *self) Per_get_mtime(cPersistentObject *self)
{ {
PyObject *t, *v; PyObject *t, *v;
if (unghostify(self) < 0) if (unghostify(self) < 0)
return NULL; return NULL;
accessed(self); accessed(self);
if (memcmp(self->serial, "\0\0\0\0\0\0\0\0", 8) == 0) { if (memcmp(self->serial, "\0\0\0\0\0\0\0\0", 8) == 0)
Py_INCREF(Py_None); {
return Py_None; Py_INCREF(Py_None);
return Py_None;
} }
t = PyObject_CallFunction(TimeStamp, "s#", self->serial, 8); t = PyObject_CallFunction(TimeStamp, "s#", self->serial, 8);
if (!t) if (!t)
return NULL; return NULL;
v = PyObject_CallMethod(t, "timeTime", ""); v = PyObject_CallMethod(t, "timeTime", "");
Py_DECREF(t); Py_DECREF(t);
return v; return v;
} }
static PyObject * static PyObject *
Per_get_state(cPersistentObject *self) Per_get_state(cPersistentObject *self)
{ {
return PyInt_FromLong(self->state); return PyInt_FromLong(self->state);
} }
static PyObject * static PyObject *
...@@ -1026,37 +1081,42 @@ Per_get_estimated_size(cPersistentObject *self) ...@@ -1026,37 +1081,42 @@ Per_get_estimated_size(cPersistentObject *self)
static int static int
Per_set_estimated_size(cPersistentObject *self, PyObject *v) Per_set_estimated_size(cPersistentObject *self, PyObject *v)
{ {
if (v) { if (v)
if (PyInt_Check(v)) { {
long lv = PyInt_AS_LONG(v); if (PyInt_Check(v))
if (lv < 0) { {
PyErr_SetString(PyExc_ValueError, long lv = PyInt_AS_LONG(v);
"_p_estimated_size must not be negative"); if (lv < 0)
return -1; {
} PyErr_SetString(PyExc_ValueError,
self->estimated_size = _estimated_size_in_24_bits(lv); "_p_estimated_size must not be negative");
} return -1;
else { }
PyErr_SetString(PyExc_ValueError, self->estimated_size = _estimated_size_in_24_bits(lv);
"_p_estimated_size must be an integer"); }
return -1; else
{
PyErr_SetString(PyExc_ValueError,
"_p_estimated_size must be an integer");
return -1;
}
} }
} else else
self->estimated_size = 0; self->estimated_size = 0;
return 0; return 0;
} }
static PyGetSetDef Per_getsets[] = { static PyGetSetDef Per_getsets[] = {
{"_p_changed", (getter)Per_get_changed, (setter)Per_set_changed}, {"_p_changed", (getter)Per_get_changed, (setter)Per_set_changed},
{"_p_jar", (getter)Per_get_jar, (setter)Per_set_jar}, {"_p_jar", (getter)Per_get_jar, (setter)Per_set_jar},
{"_p_mtime", (getter)Per_get_mtime}, {"_p_mtime", (getter)Per_get_mtime},
{"_p_oid", (getter)Per_get_oid, (setter)Per_set_oid}, {"_p_oid", (getter)Per_get_oid, (setter)Per_set_oid},
{"_p_serial", (getter)Per_get_serial, (setter)Per_set_serial}, {"_p_serial", (getter)Per_get_serial, (setter)Per_set_serial},
{"_p_state", (getter)Per_get_state}, {"_p_state", (getter)Per_get_state},
{"_p_estimated_size", {"_p_estimated_size",
(getter)Per_get_estimated_size, (setter)Per_set_estimated_size (getter)Per_get_estimated_size, (setter)Per_set_estimated_size
}, },
{NULL} {NULL}
}; };
static struct PyMethodDef Per_methods[] = { static struct PyMethodDef Per_methods[] = {
...@@ -1117,38 +1177,38 @@ static struct PyMethodDef Per_methods[] = { ...@@ -1117,38 +1177,38 @@ static struct PyMethodDef Per_methods[] = {
#define DEFERRED_ADDRESS(ADDR) 0 #define DEFERRED_ADDRESS(ADDR) 0
static PyTypeObject Pertype = { static PyTypeObject Pertype = {
PyObject_HEAD_INIT(DEFERRED_ADDRESS(&PyPersist_MetaType)) PyObject_HEAD_INIT(DEFERRED_ADDRESS(&PyPersist_MetaType))
0, /* ob_size */ 0, /* ob_size */
"persistent.Persistent", /* tp_name */ "persistent.Persistent", /* tp_name */
sizeof(cPersistentObject), /* tp_basicsize */ sizeof(cPersistentObject), /* tp_basicsize */
0, /* tp_itemsize */ 0, /* tp_itemsize */
(destructor)Per_dealloc, /* tp_dealloc */ (destructor)Per_dealloc, /* tp_dealloc */
0, /* tp_print */ 0, /* tp_print */
0, /* tp_getattr */ 0, /* tp_getattr */
0, /* tp_setattr */ 0, /* tp_setattr */
0, /* tp_compare */ 0, /* tp_compare */
0, /* tp_repr */ 0, /* tp_repr */
0, /* tp_as_number */ 0, /* tp_as_number */
0, /* tp_as_sequence */ 0, /* tp_as_sequence */
0, /* tp_as_mapping */ 0, /* tp_as_mapping */
0, /* tp_hash */ 0, /* tp_hash */
0, /* tp_call */ 0, /* tp_call */
0, /* tp_str */ 0, /* tp_str */
(getattrofunc)Per_getattro, /* tp_getattro */ (getattrofunc)Per_getattro, /* tp_getattro */
(setattrofunc)Per_setattro, /* tp_setattro */ (setattrofunc)Per_setattro, /* tp_setattro */
0, /* tp_as_buffer */ 0, /* tp_as_buffer */
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC,
/* tp_flags */ /* tp_flags */
0, /* tp_doc */ 0, /* tp_doc */
(traverseproc)Per_traverse, /* tp_traverse */ (traverseproc)Per_traverse, /* tp_traverse */
0, /* tp_clear */ 0, /* tp_clear */
0, /* tp_richcompare */ 0, /* tp_richcompare */
0, /* tp_weaklistoffset */ 0, /* tp_weaklistoffset */
0, /* tp_iter */ 0, /* tp_iter */
0, /* tp_iternext */ 0, /* tp_iternext */
Per_methods, /* tp_methods */ Per_methods, /* tp_methods */
0, /* tp_members */ 0, /* tp_members */
Per_getsets, /* tp_getset */ Per_getsets, /* tp_getset */
}; };
/* End of code for Persistent objects */ /* End of code for Persistent objects */
...@@ -1160,104 +1220,108 @@ typedef int (*intfunctionwithpythonarg)(PyObject*); ...@@ -1160,104 +1220,108 @@ typedef int (*intfunctionwithpythonarg)(PyObject*);
static int static int
Per_setstate(cPersistentObject *self) Per_setstate(cPersistentObject *self)
{ {
if (unghostify(self) < 0) if (unghostify(self) < 0)
return -1; return -1;
self->state = cPersistent_STICKY_STATE; self->state = cPersistent_STICKY_STATE;
return 0; return 0;
} }
static PyObject * static PyObject *
simple_new(PyObject *self, PyObject *type_object) simple_new(PyObject *self, PyObject *type_object)
{ {
return PyType_GenericNew((PyTypeObject *)type_object, NULL, NULL); return PyType_GenericNew((PyTypeObject *)type_object, NULL, NULL);
} }
static PyMethodDef cPersistence_methods[] = { static PyMethodDef cPersistence_methods[] =
{
{"simple_new", simple_new, METH_O, {"simple_new", simple_new, METH_O,
"Create an object by simply calling a class's __new__ method without " "Create an object by simply calling a class's __new__ method without "
"arguments."}, "arguments."},
{NULL, NULL} {NULL, NULL}
}; };
static cPersistenceCAPIstruct static cPersistenceCAPIstruct
truecPersistenceCAPI = { truecPersistenceCAPI = {
&Pertype, &Pertype,
(getattrofunc)Per_getattro, /*tp_getattr with object key*/ (getattrofunc)Per_getattro, /*tp_getattr with object key*/
(setattrofunc)Per_setattro, /*tp_setattr with object key*/ (setattrofunc)Per_setattro, /*tp_setattr with object key*/
changed, changed,
accessed, accessed,
ghostify, ghostify,
(intfunctionwithpythonarg)Per_setstate, (intfunctionwithpythonarg)Per_setstate,
NULL /* The percachedel slot is initialized in cPickleCache.c when NULL /* The percachedel slot is initialized in cPickleCache.c when
the module is loaded. It uses a function in a different the module is loaded. It uses a function in a different
shared library. */ shared library. */
}; };
void void
initcPersistence(void) initcPersistence(void)
{ {
PyObject *m, *s; PyObject *m, *s;
PyObject *copy_reg; PyObject *copy_reg;
if (init_strings() < 0) if (init_strings() < 0)
return; return;
m = Py_InitModule3("cPersistence", cPersistence_methods, m = Py_InitModule3("cPersistence", cPersistence_methods,
cPersistence_doc_string); cPersistence_doc_string);
Pertype.ob_type = &PyType_Type; Pertype.ob_type = &PyType_Type;
Pertype.tp_new = PyType_GenericNew; Pertype.tp_new = PyType_GenericNew;
if (PyType_Ready(&Pertype) < 0) if (PyType_Ready(&Pertype) < 0)
return; return;
if (PyModule_AddObject(m, "Persistent", (PyObject *)&Pertype) < 0) if (PyModule_AddObject(m, "Persistent", (PyObject *)&Pertype) < 0)
return; return;
cPersistenceCAPI = &truecPersistenceCAPI; cPersistenceCAPI = &truecPersistenceCAPI;
s = PyCObject_FromVoidPtr(cPersistenceCAPI, NULL); s = PyCObject_FromVoidPtr(cPersistenceCAPI, NULL);
if (!s) if (!s)
return; return;
if (PyModule_AddObject(m, "CAPI", s) < 0) if (PyModule_AddObject(m, "CAPI", s) < 0)
return; return;
if (PyModule_AddIntConstant(m, "GHOST", cPersistent_GHOST_STATE) < 0) if (PyModule_AddIntConstant(m, "GHOST", cPersistent_GHOST_STATE) < 0)
return; return;
if (PyModule_AddIntConstant(m, "UPTODATE", cPersistent_UPTODATE_STATE) < 0) if (PyModule_AddIntConstant(m, "UPTODATE", cPersistent_UPTODATE_STATE) < 0)
return; return;
if (PyModule_AddIntConstant(m, "CHANGED", cPersistent_CHANGED_STATE) < 0) if (PyModule_AddIntConstant(m, "CHANGED", cPersistent_CHANGED_STATE) < 0)
return; return;
if (PyModule_AddIntConstant(m, "STICKY", cPersistent_STICKY_STATE) < 0) if (PyModule_AddIntConstant(m, "STICKY", cPersistent_STICKY_STATE) < 0)
return; return;
py_simple_new = PyObject_GetAttrString(m, "simple_new"); py_simple_new = PyObject_GetAttrString(m, "simple_new");
if (!py_simple_new) if (!py_simple_new)
return; return;
copy_reg = PyImport_ImportModule("copy_reg"); copy_reg = PyImport_ImportModule("copy_reg");
if (!copy_reg) if (!copy_reg)
return; return;
copy_reg_slotnames = PyObject_GetAttrString(copy_reg, "_slotnames"); copy_reg_slotnames = PyObject_GetAttrString(copy_reg, "_slotnames");
if (!copy_reg_slotnames) { if (!copy_reg_slotnames)
Py_DECREF(copy_reg); {
return; Py_DECREF(copy_reg);
return;
} }
__newobj__ = PyObject_GetAttrString(copy_reg, "__newobj__"); __newobj__ = PyObject_GetAttrString(copy_reg, "__newobj__");
if (!__newobj__) { if (!__newobj__)
Py_DECREF(copy_reg); {
return; Py_DECREF(copy_reg);
return;
} }
if (!TimeStamp) { if (!TimeStamp)
m = PyImport_ImportModule("persistent.TimeStamp"); {
if (!m) m = PyImport_ImportModule("persistent.TimeStamp");
return; if (!m)
TimeStamp = PyObject_GetAttrString(m, "TimeStamp"); return;
Py_DECREF(m); TimeStamp = PyObject_GetAttrString(m, "TimeStamp");
/* fall through to immediate return on error */ Py_DECREF(m);
/* fall through to immediate return on error */
} }
} }
src/persistent/cPickleCache.c
View file @ 04581d75
/***************************************************************************** /*****************************************************************************
Copyright (c) 2001, 2002 Zope Corporation and Contributors. Copyright (c) 2001, 2002 Zope Corporation and Contributors.
All Rights Reserved. All Rights Reserved.
...@@ -10,87 +10,87 @@ ...@@ -10,87 +10,87 @@
WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
FOR A PARTICULAR PURPOSE FOR A PARTICULAR PURPOSE
****************************************************************************/ ****************************************************************************/
/* /*
Objects are stored under three different regimes: Objects are stored under three different regimes:
Regime 1: Persistent Classes Regime 1: Persistent Classes
Persistent Classes are part of ZClasses. They are stored in the Persistent Classes are part of ZClasses. They are stored in the
self->data dictionary, and are never garbage collected. self->data dictionary, and are never garbage collected.
The klass_items() method returns a sequence of (oid,object) tuples for The klass_items() method returns a sequence of (oid,object) tuples for
every Persistent Class, which should make it possible to implement every Persistent Class, which should make it possible to implement
garbage collection in Python if necessary. garbage collection in Python if necessary.
Regime 2: Ghost Objects Regime 2: Ghost Objects
There is no benefit to keeping a ghost object which has no external There is no benefit to keeping a ghost object which has no external
references, therefore a weak reference scheme is used to ensure that references, therefore a weak reference scheme is used to ensure that
ghost objects are removed from memory as soon as possible, when the ghost objects are removed from memory as soon as possible, when the
last external reference is lost. last external reference is lost.
Ghost objects are stored in the self->data dictionary. Normally a Ghost objects are stored in the self->data dictionary. Normally a
dictionary keeps a strong reference on its values, however this dictionary keeps a strong reference on its values, however this
reference count is 'stolen'. reference count is 'stolen'.
This weak reference scheme leaves a dangling reference, in the This weak reference scheme leaves a dangling reference, in the
dictionary, when the last external reference is lost. To clean up this dictionary, when the last external reference is lost. To clean up this
dangling reference the persistent object dealloc function calls dangling reference the persistent object dealloc function calls
self->cache->_oid_unreferenced(self->oid). The cache looks up the oid self->cache->_oid_unreferenced(self->oid). The cache looks up the oid
in the dictionary, ensures it points to an object whose reference in the dictionary, ensures it points to an object whose reference
count is zero, then removes it from the dictionary. Before removing count is zero, then removes it from the dictionary. Before removing
the object from the dictionary it must temporarily resurrect the the object from the dictionary it must temporarily resurrect the
object in much the same way that class instances are resurrected object in much the same way that class instances are resurrected
before their __del__ is called. before their __del__ is called.
Since ghost objects are stored under a different regime to non-ghost Since ghost objects are stored under a different regime to non-ghost
objects, an extra ghostify function in cPersistenceAPI replaces objects, an extra ghostify function in cPersistenceAPI replaces
self->state=GHOST_STATE assignments that were common in other self->state=GHOST_STATE assignments that were common in other
persistent classes (such as BTrees). persistent classes (such as BTrees).
Regime 3: Non-Ghost Objects Regime 3: Non-Ghost Objects
Non-ghost objects are stored in two data structures: the dictionary Non-ghost objects are stored in two data structures: the dictionary
mapping oids to objects and a doubly-linked list that encodes the mapping oids to objects and a doubly-linked list that encodes the
order in which the objects were accessed. The dictionary reference is order in which the objects were accessed. The dictionary reference is
borrowed, as it is for ghosts. The list reference is a new reference; borrowed, as it is for ghosts. The list reference is a new reference;
the list stores recently used objects, even if they are otherwise the list stores recently used objects, even if they are otherwise
unreferenced, to avoid loading the object from the database again. unreferenced, to avoid loading the object from the database again.
The doubly-link-list nodes contain next and previous pointers linking The doubly-link-list nodes contain next and previous pointers linking
together the cache and all non-ghost persistent objects. together the cache and all non-ghost persistent objects.
The node embedded in the cache is the home position. On every The node embedded in the cache is the home position. On every
attribute access a non-ghost object will relink itself just behind the attribute access a non-ghost object will relink itself just behind the
home position in the ring. Objects accessed least recently will home position in the ring. Objects accessed least recently will
eventually find themselves positioned after the home position. eventually find themselves positioned after the home position.
Occasionally other nodes are temporarily inserted in the ring as Occasionally other nodes are temporarily inserted in the ring as
position markers. The cache contains a ring_lock flag which must be position markers. The cache contains a ring_lock flag which must be
set and unset before and after doing so. Only if the flag is unset can set and unset before and after doing so. Only if the flag is unset can
the cache assume that all nodes are either his own home node, or nodes the cache assume that all nodes are either his own home node, or nodes
from persistent objects. This assumption is useful during the garbage from persistent objects. This assumption is useful during the garbage
collection process. collection process.
The number of non-ghost objects is counted in self->non_ghost_count. The number of non-ghost objects is counted in self->non_ghost_count.
The garbage collection process consists of traversing the ring, and The garbage collection process consists of traversing the ring, and
deactivating (that is, turning into a ghost) every object until deactivating (that is, turning into a ghost) every object until
self->non_ghost_count is down to the target size, or until it self->non_ghost_count is down to the target size, or until it
reaches the home position again. reaches the home position again.
Note that objects in the sticky or changed states are still kept in Note that objects in the sticky or changed states are still kept in
the ring, however they can not be deactivated. The garbage collection the ring, however they can not be deactivated. The garbage collection
process must skip such objects, rather than deactivating them. process must skip such objects, rather than deactivating them.
*/ */
static char cPickleCache_doc_string[] = static char cPickleCache_doc_string[] =
"Defines the PickleCache used by ZODB Connection objects.\n" "Defines the PickleCache used by ZODB Connection objects.\n"
"\n" "\n"
"$Id$\n"; "$Id$\n";
#define DONT_USE_CPERSISTENCECAPI #define DONT_USE_CPERSISTENCECAPI
#include "cPersistence.h" #include "cPersistence.h"
...@@ -99,6 +99,12 @@ static char cPickleCache_doc_string[] = ...@@ -99,6 +99,12 @@ static char cPickleCache_doc_string[] =
#include <stddef.h> #include <stddef.h>
#undef Py_FindMethod #undef Py_FindMethod
/* Python 2.4 backward compat */
#if PY_MAJOR_VERSION <= 2 && PY_MINOR_VERSION < 5
#define Py_ssize_t int
typedef Py_ssize_t (*lenfunc)(PyObject *);
#endif
/* Python string objects to speed lookups; set by module init. */ /* Python string objects to speed lookups; set by module init. */
static PyObject *py__p_changed; static PyObject *py__p_changed;
static PyObject *py__p_deactivate; static PyObject *py__p_deactivate;
...@@ -111,31 +117,32 @@ static cPersistenceCAPIstruct *capi; ...@@ -111,31 +117,32 @@ static cPersistenceCAPIstruct *capi;
that layout of this struct is the same as the start of that layout of this struct is the same as the start of
ccobject_head in cPersistence.c */ ccobject_head in cPersistence.c */
typedef struct { typedef struct {
CACHE_HEAD CACHE_HEAD
int klass_count; /* count of persistent classes */ int klass_count; /* count of persistent classes */
PyObject *data; /* oid -> object dict */ PyObject *data; /* oid -> object dict */
PyObject *jar; /* Connection object */ PyObject *jar; /* Connection object */
int cache_size; /* target number of items in cache */ int cache_size; /* target number of items in cache */
PY_LONG_LONG cache_size_bytes; /* target total estimated size of items in cache */ PY_LONG_LONG cache_size_bytes; /* target total estimated size of
items in cache */
/* Most of the time the ring contains only:
* many nodes corresponding to persistent objects /* Most of the time the ring contains only:
* one 'home' node from the cache. * many nodes corresponding to persistent objects
In some cases it is handy to temporarily add other types * one 'home' node from the cache.
of node into the ring as placeholders. 'ring_lock' is a boolean In some cases it is handy to temporarily add other types
indicating that someone has already done this. Currently this of node into the ring as placeholders. 'ring_lock' is a boolean
is only used by the garbage collection code. */ indicating that someone has already done this. Currently this
is only used by the garbage collection code. */
int ring_lock;
int ring_lock;
/* 'cache_drain_resistance' controls how quickly the cache size will drop
when it is smaller than the configured size. A value of zero means it will /* 'cache_drain_resistance' controls how quickly the cache size will drop
not drop below the configured size (suitable for most caches). Otherwise, when it is smaller than the configured size. A value of zero means it will
it will remove cache_non_ghost_count/cache_drain_resistance items from not drop below the configured size (suitable for most caches). Otherwise,
the cache every time (suitable for rarely used caches, such as those it will remove cache_non_ghost_count/cache_drain_resistance items from
associated with Zope versions. */ the cache every time (suitable for rarely used caches, such as those
associated with Zope versions. */
int cache_drain_resistance;
int cache_drain_resistance;
} ccobject; } ccobject;
...@@ -143,224 +150,231 @@ static int cc_ass_sub(ccobject *self, PyObject *key, PyObject *v); ...@@ -143,224 +150,231 @@ static int cc_ass_sub(ccobject *self, PyObject *key, PyObject *v);
/* ---------------------------------------------------------------- */ /* ---------------------------------------------------------------- */
#define OBJECT_FROM_RING(SELF, HERE) \ #define OBJECT_FROM_RING(SELF, HERE) \
((cPersistentObject *)(((char *)here) - offsetof(cPersistentObject, ring))) ((cPersistentObject *)(((char *)here) - offsetof(cPersistentObject, ring)))
/* Insert self into the ring, following after. */ /* Insert self into the ring, following after. */
static void static void
insert_after(CPersistentRing *self, CPersistentRing *after) insert_after(CPersistentRing *self, CPersistentRing *after)
{ {
assert(self != NULL); assert(self != NULL);
assert(after != NULL); assert(after != NULL);
self->r_prev = after; self->r_prev = after;
self->r_next = after->r_next; self->r_next = after->r_next;
after->r_next->r_prev = self; after->r_next->r_prev = self;
after->r_next = self; after->r_next = self;
} }
/* Remove self from the ring. */ /* Remove self from the ring. */
static void static void
unlink_from_ring(CPersistentRing *self) unlink_from_ring(CPersistentRing *self)
{ {
assert(self != NULL); assert(self != NULL);
self->r_prev->r_next = self->r_next; self->r_prev->r_next = self->r_next;
self->r_next->r_prev = self->r_prev; self->r_next->r_prev = self->r_prev;
} }
static int static int
scan_gc_items(ccobject *self, int target, PY_LONG_LONG target_bytes) scan_gc_items(ccobject *self, int target, PY_LONG_LONG target_bytes)
{ {
/* This function must only be called with the ring lock held, /* This function must only be called with the ring lock held,
because it places non-object placeholders in the ring. because it places non-object placeholders in the ring.
*/ */
cPersistentObject *object; cPersistentObject *object;
CPersistentRing *here; CPersistentRing *here;
CPersistentRing before_original_home; CPersistentRing before_original_home;
int result = -1; /* guilty until proved innocent */ int result = -1; /* guilty until proved innocent */
/* Scan the ring, from least to most recently used, deactivating /* Scan the ring, from least to most recently used, deactivating
* up-to-date objects, until we either find the ring_home again or * up-to-date objects, until we either find the ring_home again or
* or we've ghosted enough objects to reach the target size. * or we've ghosted enough objects to reach the target size.
* Tricky: __getattr__ and __del__ methods can do anything, and in * Tricky: __getattr__ and __del__ methods can do anything, and in
* particular if we ghostify an object with a __del__ method, that method * particular if we ghostify an object with a __del__ method, that method
* can load the object again, putting it back into the MRU part of the * can load the object again, putting it back into the MRU part of the
* ring. Waiting to find ring_home again can thus cause an infinite * ring. Waiting to find ring_home again can thus cause an infinite
* loop (Collector #1208). So before_original_home records the MRU * loop (Collector #1208). So before_original_home records the MRU
* position we start with, and we stop the scan when we reach that. * position we start with, and we stop the scan when we reach that.
*/ */
insert_after(&before_original_home, self->ring_home.r_prev); insert_after(&before_original_home, self->ring_home.r_prev);
here = self->ring_home.r_next; /* least recently used object */ here = self->ring_home.r_next; /* least recently used object */
while (here != &before_original_home && while (here != &before_original_home &&
(self->non_ghost_count > target (self->non_ghost_count > target
|| (target_bytes && self->total_estimated_size > target_bytes) || (target_bytes && self->total_estimated_size > target_bytes)
) )
) { )
assert(self->ring_lock); {
assert(here != &self->ring_home); assert(self->ring_lock);
assert(here != &self->ring_home);
/* At this point we know that the ring only contains nodes
from persistent objects, plus our own home node. We know /* At this point we know that the ring only contains nodes
this because the ring lock is held. We can safely assume from persistent objects, plus our own home node. We know
the current ring node is a persistent object now we know it this because the ring lock is held. We can safely assume
is not the home */ the current ring node is a persistent object now we know it
object = OBJECT_FROM_RING(self, here); is not the home */
object = OBJECT_FROM_RING(self, here);
if (object->state == cPersistent_UPTODATE_STATE) {
CPersistentRing placeholder; if (object->state == cPersistent_UPTODATE_STATE)
PyObject *method; {
PyObject *temp; CPersistentRing placeholder;
int error_occurred = 0; PyObject *method;
/* deactivate it. This is the main memory saver. */ PyObject *temp;
int error_occurred = 0;
/* Add a placeholder, a dummy node in the ring. We need /* deactivate it. This is the main memory saver. */
to do this to mark our position in the ring. It is
possible that the PyObject_GetAttr() call below will /* Add a placeholder, a dummy node in the ring. We need
invoke a __getattr__() hook in Python. Also possible to do this to mark our position in the ring. It is
that deactivation will lead to a __del__ method call. possible that the PyObject_GetAttr() call below will
So another thread might run, and mutate the ring as a side invoke a __getattr__() hook in Python. Also possible
effect of object accesses. There's no predicting then where that deactivation will lead to a __del__ method call.
in the ring here->next will point after that. The So another thread might run, and mutate the ring as a side
placeholder won't move as a side effect of calling Python effect of object accesses. There's no predicting then where
code. in the ring here->next will point after that. The
*/ placeholder won't move as a side effect of calling Python
insert_after(&placeholder, here); code.
method = PyObject_GetAttr((PyObject *)object, py__p_deactivate); */
if (method == NULL) insert_after(&placeholder, here);
error_occurred = 1; method = PyObject_GetAttr((PyObject *)object, py__p_deactivate);
else { if (method == NULL)
temp = PyObject_CallObject(method, NULL); error_occurred = 1;
Py_DECREF(method); else
if (temp == NULL) {
error_occurred = 1; temp = PyObject_CallObject(method, NULL);
} Py_DECREF(method);
if (temp == NULL)
here = placeholder.r_next; error_occurred = 1;
unlink_from_ring(&placeholder); }
if (error_occurred)
goto Done; here = placeholder.r_next;
unlink_from_ring(&placeholder);
if (error_occurred)
goto Done;
} }
else else
here = here->r_next; here = here->r_next;
} }
result = 0; result = 0;
Done: Done:
unlink_from_ring(&before_original_home); unlink_from_ring(&before_original_home);
return result; return result;
} }
static PyObject * static PyObject *
lockgc(ccobject *self, int target_size, PY_LONG_LONG target_size_bytes) lockgc(ccobject *self, int target_size, PY_LONG_LONG target_size_bytes)
{ {
/* This is thread-safe because of the GIL, and there's nothing /* This is thread-safe because of the GIL, and there's nothing
* in between checking the ring_lock and acquiring it that calls back * in between checking the ring_lock and acquiring it that calls back
* into Python. * into Python.
*/ */
if (self->ring_lock) { if (self->ring_lock)
Py_INCREF(Py_None); {
return Py_None; Py_INCREF(Py_None);
return Py_None;
} }
self->ring_lock = 1; self->ring_lock = 1;
if (scan_gc_items(self, target_size, target_size_bytes) < 0) { if (scan_gc_items(self, target_size, target_size_bytes) < 0)
self->ring_lock = 0; {
return NULL; self->ring_lock = 0;
return NULL;
} }
self->ring_lock = 0; self->ring_lock = 0;
Py_INCREF(Py_None); Py_INCREF(Py_None);
return Py_None; return Py_None;
} }
static PyObject * static PyObject *
cc_incrgc(ccobject *self, PyObject *args) cc_incrgc(ccobject *self, PyObject *args)
{ {
int obsolete_arg = -999; int obsolete_arg = -999;
int starting_size = self->non_ghost_count; int starting_size = self->non_ghost_count;
int target_size = self->cache_size; int target_size = self->cache_size;
PY_LONG_LONG target_size_bytes = self->cache_size_bytes; PY_LONG_LONG target_size_bytes = self->cache_size_bytes;
if (self->cache_drain_resistance >= 1) { if (self->cache_drain_resistance >= 1)
/* This cache will gradually drain down to a small size. Check {
a (small) number of objects proportional to the current size */ /* This cache will gradually drain down to a small size. Check
a (small) number of objects proportional to the current size */
int target_size_2 = (starting_size - 1
- starting_size / self->cache_drain_resistance); int target_size_2 = (starting_size - 1
if (target_size_2 < target_size) - starting_size / self->cache_drain_resistance);
target_size = target_size_2; if (target_size_2 < target_size)
target_size = target_size_2;
} }
if (!PyArg_ParseTuple(args, "|i:incrgc", &obsolete_arg)) if (!PyArg_ParseTuple(args, "|i:incrgc", &obsolete_arg))
return NULL; return NULL;
if (obsolete_arg != -999 if (obsolete_arg != -999
&& &&
(PyErr_Warn(PyExc_DeprecationWarning, (PyErr_Warn(PyExc_DeprecationWarning,
"No argument expected") "No argument expected")
< 0)) < 0))
return NULL; return NULL;
return lockgc(self, target_size, target_size_bytes); return lockgc(self, target_size, target_size_bytes);
} }
static PyObject * static PyObject *
cc_full_sweep(ccobject *self, PyObject *args) cc_full_sweep(ccobject *self, PyObject *args)
{ {
int dt = -999; int dt = -999;
/* TODO: This should be deprecated; */ /* TODO: This should be deprecated; */
if (!PyArg_ParseTuple(args, "|i:full_sweep", &dt)) if (!PyArg_ParseTuple(args, "|i:full_sweep", &dt))
return NULL; return NULL;
if (dt == -999) if (dt == -999)
return lockgc(self, 0, 0); return lockgc(self, 0, 0);
else else
return cc_incrgc(self, args); return cc_incrgc(self, args);
} }
static PyObject * static PyObject *
cc_minimize(ccobject *self, PyObject *args) cc_minimize(ccobject *self, PyObject *args)
{ {
int ignored = -999; int ignored = -999;
if (!PyArg_ParseTuple(args, "|i:minimize", &ignored)) if (!PyArg_ParseTuple(args, "|i:minimize", &ignored))
return NULL; return NULL;
if (ignored != -999 if (ignored != -999
&& &&
(PyErr_Warn(PyExc_DeprecationWarning, (PyErr_Warn(PyExc_DeprecationWarning,
"No argument expected") "No argument expected")
< 0)) < 0))
return NULL; return NULL;
return lockgc(self, 0, 0); return lockgc(self, 0, 0);
} }
static int static int
_invalidate(ccobject *self, PyObject *key) _invalidate(ccobject *self, PyObject *key)
{ {
static PyObject *_p_invalidate = NULL; static PyObject *_p_invalidate = NULL;
PyObject *meth, *v; PyObject *meth, *v;
v = PyDict_GetItem(self->data, key); v = PyDict_GetItem(self->data, key);
if (v == NULL) if (v == NULL)
return 0; return 0;
if (_p_invalidate == NULL) if (_p_invalidate == NULL)
{ {
_p_invalidate = PyString_InternFromString("_p_invalidate"); _p_invalidate = PyString_InternFromString("_p_invalidate");
if (_p_invalidate == NULL) if (_p_invalidate == NULL)
{ {
/* It doesn't make any sense to ignore this error, but /* It doesn't make any sense to ignore this error, but
the caller ignores all errors. the caller ignores all errors.
TODO: and why does it do that? This should be fixed TODO: and why does it do that? This should be fixed
*/ */
return -1; return -1;
} }
} }
if (v->ob_refcnt <= 1 && PyType_Check(v)) { if (v->ob_refcnt <= 1 && PyType_Check(v))
{
/* This looks wrong, but it isn't. We use strong references to types /* This looks wrong, but it isn't. We use strong references to types
because they don't have the ring members. because they don't have the ring members.
...@@ -371,13 +385,13 @@ _invalidate(ccobject *self, PyObject *key) ...@@ -371,13 +385,13 @@ _invalidate(ccobject *self, PyObject *key)
return PyDict_DelItem(self->data, key); return PyDict_DelItem(self->data, key);
} }
meth = PyObject_GetAttr(v, _p_invalidate); meth = PyObject_GetAttr(v, _p_invalidate);
if (meth == NULL) if (meth == NULL)
return -1; return -1;
v = PyObject_CallObject(meth, NULL); v = PyObject_CallObject(meth, NULL);
Py_DECREF(meth); Py_DECREF(meth);
return v == NULL ? -1 : 0; return v == NULL ? -1 : 0;
} }
static PyObject * static PyObject *
...@@ -390,36 +404,39 @@ cc_invalidate(ccobject *self, PyObject *inv) ...@@ -390,36 +404,39 @@ cc_invalidate(ccobject *self, PyObject *inv)
{ {
while (PyDict_Next(inv, &i, &key, &v)) while (PyDict_Next(inv, &i, &key, &v))
{ {
if (_invalidate(self, key) < 0) if (_invalidate(self, key) < 0)
return NULL; return NULL;
} }
PyDict_Clear(inv); PyDict_Clear(inv);
} }
else { else
{
if (PyString_Check(inv)) if (PyString_Check(inv))
{ {
if (_invalidate(self, inv) < 0) if (_invalidate(self, inv) < 0)
return NULL; return NULL;
} }
else { else
int l, r; {
int l, r;
l = PyObject_Length(inv);
if (l < 0) l = PyObject_Length(inv);
return NULL; if (l < 0)
for (i=l; --i >= 0; ) { return NULL;
key = PySequence_GetItem(inv, i); for (i=l; --i >= 0; )
if (!key) {
return NULL; key = PySequence_GetItem(inv, i);
r = _invalidate(self, key); if (!key)
Py_DECREF(key); return NULL;
r = _invalidate(self, key);
Py_DECREF(key);
if (r < 0) if (r < 0)
return NULL; return NULL;
} }
/* Dubious: modifying the input may be an unexpected side effect. */ /* Dubious: modifying the input may be an unexpected side effect. */
PySequence_DelSlice(inv, 0, l); PySequence_DelSlice(inv, 0, l);
} }
} }
Py_INCREF(Py_None); Py_INCREF(Py_None);
return Py_None; return Py_None;
...@@ -428,211 +445,221 @@ cc_invalidate(ccobject *self, PyObject *inv) ...@@ -428,211 +445,221 @@ cc_invalidate(ccobject *self, PyObject *inv)
static PyObject * static PyObject *
cc_get(ccobject *self, PyObject *args) cc_get(ccobject *self, PyObject *args)
{ {
PyObject *r, *key, *d = NULL; PyObject *r, *key, *d = NULL;
if (!PyArg_ParseTuple(args, "O|O:get", &key, &d)) if (!PyArg_ParseTuple(args, "O|O:get", &key, &d))
return NULL; return NULL;
r = PyDict_GetItem(self->data, key); r = PyDict_GetItem(self->data, key);
if (!r) { if (!r)
if (d) {
r = d; if (d)
else r = d;
r = Py_None; else
r = Py_None;
} }
Py_INCREF(r); Py_INCREF(r);
return r; return r;
} }
static PyObject * static PyObject *
cc_items(ccobject *self) cc_items(ccobject *self)
{ {
return PyObject_CallMethod(self->data, "items", ""); return PyObject_CallMethod(self->data, "items", "");
} }
static PyObject * static PyObject *
cc_klass_items(ccobject *self) cc_klass_items(ccobject *self)
{ {
PyObject *l,*k,*v; PyObject *l,*k,*v;
Py_ssize_t p = 0; Py_ssize_t p = 0;
l = PyList_New(0); l = PyList_New(0);
if (l == NULL) if (l == NULL)
return NULL; return NULL;
while (PyDict_Next(self->data, &p, &k, &v)) { while (PyDict_Next(self->data, &p, &k, &v))
if(PyType_Check(v)) { {
v = Py_BuildValue("OO", k, v); if(PyType_Check(v))
if (v == NULL) { {
Py_DECREF(l); v = Py_BuildValue("OO", k, v);
return NULL; if (v == NULL)
} {
if (PyList_Append(l, v) < 0) { Py_DECREF(l);
Py_DECREF(v); return NULL;
Py_DECREF(l); }
return NULL; if (PyList_Append(l, v) < 0)
} {
Py_DECREF(v); Py_DECREF(v);
Py_DECREF(l);
return NULL;
}
Py_DECREF(v);
} }
} }
return l; return l;
} }
static PyObject * static PyObject *
cc_debug_info(ccobject *self) cc_debug_info(ccobject *self)
{ {
PyObject *l,*k,*v; PyObject *l,*k,*v;
Py_ssize_t p = 0; Py_ssize_t p = 0;
l = PyList_New(0); l = PyList_New(0);
if (l == NULL) if (l == NULL)
return NULL; return NULL;
while (PyDict_Next(self->data, &p, &k, &v))
{
if (v->ob_refcnt <= 0)
v = Py_BuildValue("Oi", k, v->ob_refcnt);
else if (! PyType_Check(v) &&
(v->ob_type->tp_basicsize >= sizeof(cPersistentObject))
)
v = Py_BuildValue("Oisi",
k, v->ob_refcnt, v->ob_type->tp_name,
((cPersistentObject*)v)->state);
else
v = Py_BuildValue("Ois", k, v->ob_refcnt, v->ob_type->tp_name);
if (v == NULL)
goto err;
if (PyList_Append(l, v) < 0) while (PyDict_Next(self->data, &p, &k, &v))
goto err; {
} if (v->ob_refcnt <= 0)
v = Py_BuildValue("Oi", k, v->ob_refcnt);
else if (! PyType_Check(v) &&
(v->ob_type->tp_basicsize >= sizeof(cPersistentObject))
)
v = Py_BuildValue("Oisi",
k, v->ob_refcnt, v->ob_type->tp_name,
((cPersistentObject*)v)->state);
else
v = Py_BuildValue("Ois", k, v->ob_refcnt, v->ob_type->tp_name);
if (v == NULL)
goto err;
if (PyList_Append(l, v) < 0)
goto err;
}
return l; return l;
err: err:
Py_DECREF(l); Py_DECREF(l);
return NULL; return NULL;
} }
static PyObject * static PyObject *
cc_lru_items(ccobject *self) cc_lru_items(ccobject *self)
{ {
PyObject *l; PyObject *l;
CPersistentRing *here; CPersistentRing *here;
if (self->ring_lock) { if (self->ring_lock)
/* When the ring lock is held, we have no way of know which {
ring nodes belong to persistent objects, and which a /* When the ring lock is held, we have no way of know which
placeholders. */ ring nodes belong to persistent objects, and which a
PyErr_SetString(PyExc_ValueError, placeholders. */
".lru_items() is unavailable during garbage collection"); PyErr_SetString(PyExc_ValueError,
return NULL; ".lru_items() is unavailable during garbage collection");
return NULL;
} }
l = PyList_New(0); l = PyList_New(0);
if (l == NULL) if (l == NULL)
return NULL; return NULL;
here = self->ring_home.r_next; here = self->ring_home.r_next;
while (here != &self->ring_home) { while (here != &self->ring_home)
PyObject *v; {
cPersistentObject *object = OBJECT_FROM_RING(self, here); PyObject *v;
cPersistentObject *object = OBJECT_FROM_RING(self, here);
if (object == NULL) { if (object == NULL)
Py_DECREF(l); {
return NULL; Py_DECREF(l);
return NULL;
} }
v = Py_BuildValue("OO", object->oid, object); v = Py_BuildValue("OO", object->oid, object);
if (v == NULL) { if (v == NULL)
Py_DECREF(l); {
return NULL; Py_DECREF(l);
} return NULL;
if (PyList_Append(l, v) < 0) { }
Py_DECREF(v); if (PyList_Append(l, v) < 0)
Py_DECREF(l); {
return NULL; Py_DECREF(v);
} Py_DECREF(l);
Py_DECREF(v); return NULL;
here = here->r_next; }
Py_DECREF(v);
here = here->r_next;
} }
return l; return l;
} }
static void static void
cc_oid_unreferenced(ccobject *self, PyObject *oid) cc_oid_unreferenced(ccobject *self, PyObject *oid)
{ {
/* This is called by the persistent object deallocation function /* This is called by the persistent object deallocation function
when the reference count on a persistent object reaches when the reference count on a persistent object reaches
zero. We need to fix up our dictionary; its reference is now zero. We need to fix up our dictionary; its reference is now
dangling because we stole its reference count. Be careful to dangling because we stole its reference count. Be careful to
not release the global interpreter lock until this is not release the global interpreter lock until this is
complete. */ complete. */
PyObject *v; PyObject *v;
/* If the cache has been cleared by GC, data will be NULL. */ /* If the cache has been cleared by GC, data will be NULL. */
if (!self->data) if (!self->data)
return; return;
v = PyDict_GetItem(self->data, oid); v = PyDict_GetItem(self->data, oid);
assert(v); assert(v);
assert(v->ob_refcnt == 0); assert(v->ob_refcnt == 0);
/* Need to be very hairy here because a dictionary is about /* Need to be very hairy here because a dictionary is about
to decref an already deleted object. to decref an already deleted object.
*/ */
#ifdef Py_TRACE_REFS #ifdef Py_TRACE_REFS
/* This is called from the deallocation function after the /* This is called from the deallocation function after the
interpreter has untracked the reference. Track it again. interpreter has untracked the reference. Track it again.
*/ */
_Py_NewReference(v); _Py_NewReference(v);
/* Don't increment total refcount as a result of the /* Don't increment total refcount as a result of the
shenanigans played in this function. The _Py_NewReference() shenanigans played in this function. The _Py_NewReference()
call above creates artificial references to v. call above creates artificial references to v.
*/ */
_Py_RefTotal--; _Py_RefTotal--;
assert(v->ob_type); assert(v->ob_type);
#else #else
Py_INCREF(v); Py_INCREF(v);
#endif #endif
assert(v->ob_refcnt == 1); assert(v->ob_refcnt == 1);
/* Incremement the refcount again, because delitem is going to /* Incremement the refcount again, because delitem is going to
DECREF it. If it's refcount reached zero again, we'd call back to DECREF it. If it's refcount reached zero again, we'd call back to
the dealloc function that called us. the dealloc function that called us.
*/ */
Py_INCREF(v); Py_INCREF(v);
/* TODO: Should we call _Py_ForgetReference() on error exit? */ /* TODO: Should we call _Py_ForgetReference() on error exit? */
if (PyDict_DelItem(self->data, oid) < 0) if (PyDict_DelItem(self->data, oid) < 0)
return; return;
Py_DECREF((ccobject *)((cPersistentObject *)v)->cache); Py_DECREF((ccobject *)((cPersistentObject *)v)->cache);
((cPersistentObject *)v)->cache = NULL; ((cPersistentObject *)v)->cache = NULL;
assert(v->ob_refcnt == 1); assert(v->ob_refcnt == 1);
/* Undo the temporary resurrection. /* Undo the temporary resurrection.
Don't DECREF the object, because this function is called from Don't DECREF the object, because this function is called from
the object's dealloc function. If the refcnt reaches zero, it the object's dealloc function. If the refcnt reaches zero, it
will all be invoked recursively. will all be invoked recursively.
*/ */
_Py_ForgetReference(v); _Py_ForgetReference(v);
} }
static PyObject * static PyObject *
cc_ringlen(ccobject *self) cc_ringlen(ccobject *self)
{ {
CPersistentRing *here; CPersistentRing *here;
int c = 0; int c = 0;
for (here = self->ring_home.r_next; here != &self->ring_home; for (here = self->ring_home.r_next; here != &self->ring_home;
here = here->r_next) here = here->r_next)
c++; c++;
return PyInt_FromLong(c); return PyInt_FromLong(c);
} }
static PyObject * static PyObject *
...@@ -645,428 +672,464 @@ cc_update_object_size_estimation(ccobject *self, PyObject *args) ...@@ -645,428 +672,464 @@ cc_update_object_size_estimation(ccobject *self, PyObject *args)
return NULL; return NULL;
/* Note: reference borrowed */ /* Note: reference borrowed */
v = (cPersistentObject *)PyDict_GetItem(self->data, oid); v = (cPersistentObject *)PyDict_GetItem(self->data, oid);
if (v) { if (v)
/* we know this object -- update our "total_size_estimation" {
we must only update when the object is in the ring /* we know this object -- update our "total_size_estimation"
*/ we must only update when the object is in the ring
if (v->ring.r_next) {
self->total_estimated_size += _estimated_size_in_bytes(
_estimated_size_in_24_bits(new_size) - v->estimated_size
);
/* we do this in "Connection" as we need it even when the
object is not in the cache (or not the ring)
*/ */
/* v->estimated_size = new_size; */ if (v->ring.r_next)
{
self->total_estimated_size += _estimated_size_in_bytes(
_estimated_size_in_24_bits(new_size) - v->estimated_size
);
/* we do this in "Connection" as we need it even when the
object is not in the cache (or not the ring)
*/
/* v->estimated_size = new_size; */
}
} }
}
Py_RETURN_NONE; Py_RETURN_NONE;
} }
static struct PyMethodDef cc_methods[] = { static struct PyMethodDef cc_methods[] = {
{"items", (PyCFunction)cc_items, METH_NOARGS, {"items", (PyCFunction)cc_items, METH_NOARGS,
"Return list of oid, object pairs for all items in cache."}, "Return list of oid, object pairs for all items in cache."},
{"lru_items", (PyCFunction)cc_lru_items, METH_NOARGS, {"lru_items", (PyCFunction)cc_lru_items, METH_NOARGS,
"List (oid, object) pairs from the lru list, as 2-tuples."}, "List (oid, object) pairs from the lru list, as 2-tuples."},
{"klass_items", (PyCFunction)cc_klass_items, METH_NOARGS, {"klass_items", (PyCFunction)cc_klass_items, METH_NOARGS,
"List (oid, object) pairs of cached persistent classes."}, "List (oid, object) pairs of cached persistent classes."},
{"full_sweep", (PyCFunction)cc_full_sweep, METH_VARARGS, {"full_sweep", (PyCFunction)cc_full_sweep, METH_VARARGS,
"full_sweep() -- Perform a full sweep of the cache."}, "full_sweep() -- Perform a full sweep of the cache."},
{"minimize", (PyCFunction)cc_minimize, METH_VARARGS, {"minimize", (PyCFunction)cc_minimize, METH_VARARGS,
"minimize([ignored]) -- Remove as many objects as possible\n\n" "minimize([ignored]) -- Remove as many objects as possible\n\n"
"Ghostify all objects that are not modified. Takes an optional\n" "Ghostify all objects that are not modified. Takes an optional\n"
"argument, but ignores it."}, "argument, but ignores it."},
{"incrgc", (PyCFunction)cc_incrgc, METH_VARARGS, {"incrgc", (PyCFunction)cc_incrgc, METH_VARARGS,
"incrgc() -- Perform incremental garbage collection\n\n" "incrgc() -- Perform incremental garbage collection\n\n"
"This method had been depricated!" "This method had been depricated!"
"Some other implementations support an optional parameter 'n' which\n" "Some other implementations support an optional parameter 'n' which\n"
"indicates a repetition count; this value is ignored."}, "indicates a repetition count; this value is ignored."},
{"invalidate", (PyCFunction)cc_invalidate, METH_O, {"invalidate", (PyCFunction)cc_invalidate, METH_O,
"invalidate(oids) -- invalidate one, many, or all ids"}, "invalidate(oids) -- invalidate one, many, or all ids"},
{"get", (PyCFunction)cc_get, METH_VARARGS, {"get", (PyCFunction)cc_get, METH_VARARGS,
"get(key [, default]) -- get an item, or a default"}, "get(key [, default]) -- get an item, or a default"},
{"ringlen", (PyCFunction)cc_ringlen, METH_NOARGS, {"ringlen", (PyCFunction)cc_ringlen, METH_NOARGS,
"ringlen() -- Returns number of non-ghost items in cache."}, "ringlen() -- Returns number of non-ghost items in cache."},
{"debug_info", (PyCFunction)cc_debug_info, METH_NOARGS, {"debug_info", (PyCFunction)cc_debug_info, METH_NOARGS,
"debug_info() -- Returns debugging data about objects in the cache."}, "debug_info() -- Returns debugging data about objects in the cache."},
{"update_object_size_estimation", {"update_object_size_estimation",
(PyCFunction)cc_update_object_size_estimation, (PyCFunction)cc_update_object_size_estimation,
METH_VARARGS, METH_VARARGS,
"update_object_size_estimation(oid, new_size) -- update the caches size estimation for *oid* (if this is known to the cache)."}, "update_object_size_estimation(oid, new_size) -- "
{NULL, NULL} /* sentinel */ "update the caches size estimation for *oid* "
"(if this is known to the cache)."},
{NULL, NULL} /* sentinel */
}; };
static int static int
cc_init(ccobject *self, PyObject *args, PyObject *kwds) cc_init(ccobject *self, PyObject *args, PyObject *kwds)
{ {
int cache_size = 100; int cache_size = 100;
PY_LONG_LONG cache_size_bytes = 0; PY_LONG_LONG cache_size_bytes = 0;
PyObject *jar; PyObject *jar;
if (!PyArg_ParseTuple(args, "O|iL", &jar, &cache_size, &cache_size_bytes)) if (!PyArg_ParseTuple(args, "O|iL", &jar, &cache_size, &cache_size_bytes))
return -1; return -1;
self->jar = NULL; self->jar = NULL;
self->data = PyDict_New(); self->data = PyDict_New();
if (self->data == NULL) { if (self->data == NULL)
Py_DECREF(self); {
return -1; Py_DECREF(self);
return -1;
} }
/* Untrack the dict mapping oids to objects. /* Untrack the dict mapping oids to objects.
The dict contains uncounted references to ghost objects, so it The dict contains uncounted references to ghost objects, so it
isn't safe for GC to visit it. If GC finds an object with more isn't safe for GC to visit it. If GC finds an object with more
referents that refcounts, it will die with an assertion failure. referents that refcounts, it will die with an assertion failure.
When the cache participates in GC, it will need to traverse the When the cache participates in GC, it will need to traverse the
objects in the doubly-linked list, which will account for all the objects in the doubly-linked list, which will account for all the
non-ghost objects. non-ghost objects.
*/ */
PyObject_GC_UnTrack((void *)self->data); PyObject_GC_UnTrack((void *)self->data);
self->jar = jar; self->jar = jar;
Py_INCREF(jar); Py_INCREF(jar);
self->cache_size = cache_size; self->cache_size = cache_size;
self->cache_size_bytes = cache_size_bytes; self->cache_size_bytes = cache_size_bytes;
self->non_ghost_count = 0; self->non_ghost_count = 0;
self->total_estimated_size = 0; self->total_estimated_size = 0;
self->klass_count = 0; self->klass_count = 0;
self->cache_drain_resistance = 0; self->cache_drain_resistance = 0;
self->ring_lock = 0; self->ring_lock = 0;
self->ring_home.r_next = &self->ring_home; self->ring_home.r_next = &self->ring_home;
self->ring_home.r_prev = &self->ring_home; self->ring_home.r_prev = &self->ring_home;
return 0; return 0;
} }
static void static void
cc_dealloc(ccobject *self) cc_dealloc(ccobject *self)
{ {
Py_XDECREF(self->data); Py_XDECREF(self->data);
Py_XDECREF(self->jar); Py_XDECREF(self->jar);
PyObject_GC_Del(self); PyObject_GC_Del(self);
} }
static int static int
cc_clear(ccobject *self) cc_clear(ccobject *self)
{ {
Py_ssize_t pos = 0; Py_ssize_t pos = 0;
PyObject *k, *v; PyObject *k, *v;
/* Clearing the cache is delicate. /* Clearing the cache is delicate.
A non-ghost object will show up in the ring and in the dict. If A non-ghost object will show up in the ring and in the dict. If
we deallocating the dict before clearing the ring, the GC will we deallocating the dict before clearing the ring, the GC will
decref each object in the dict. Since the dict references are decref each object in the dict. Since the dict references are
uncounted, this will lead to objects having negative refcounts. uncounted, this will lead to objects having negative refcounts.
Freeing the non-ghost objects should eliminate many objects from Freeing the non-ghost objects should eliminate many objects from
the cache, but there may still be ghost objects left. It's the cache, but there may still be ghost objects left. It's
not safe to decref the dict until it's empty, so we need to manually not safe to decref the dict until it's empty, so we need to manually
clear those out of the dict, too. We accomplish that by replacing clear those out of the dict, too. We accomplish that by replacing
all the ghost objects with None. all the ghost objects with None.
*/ */
/* We don't need to lock the ring, because the cache is unreachable. /* We don't need to lock the ring, because the cache is unreachable.
It should be impossible for anyone to be modifying the cache. It should be impossible for anyone to be modifying the cache.
*/ */
assert(! self->ring_lock); assert(! self->ring_lock);
while (self->ring_home.r_next != &self->ring_home) { while (self->ring_home.r_next != &self->ring_home)
CPersistentRing *here = self->ring_home.r_next; {
cPersistentObject *o = OBJECT_FROM_RING(self, here); CPersistentRing *here = self->ring_home.r_next;
cPersistentObject *o = OBJECT_FROM_RING(self, here);
if (o->cache) {
Py_INCREF(o); /* account for uncounted reference */ if (o->cache)
if (PyDict_DelItem(self->data, o->oid) < 0) {
return -1; Py_INCREF(o); /* account for uncounted reference */
} if (PyDict_DelItem(self->data, o->oid) < 0)
o->cache = NULL; return -1;
Py_DECREF(self); }
self->ring_home.r_next = here->r_next; o->cache = NULL;
o->ring.r_prev = NULL; Py_DECREF(self);
o->ring.r_next = NULL; self->ring_home.r_next = here->r_next;
Py_DECREF(o); o->ring.r_prev = NULL;
here = here->r_next; o->ring.r_next = NULL;
Py_DECREF(o);
here = here->r_next;
} }
Py_XDECREF(self->jar); Py_XDECREF(self->jar);
while (PyDict_Next(self->data, &pos, &k, &v)) { while (PyDict_Next(self->data, &pos, &k, &v))
Py_INCREF(v); {
if (PyDict_SetItem(self->data, k, Py_None) < 0) Py_INCREF(v);
return -1; if (PyDict_SetItem(self->data, k, Py_None) < 0)
return -1;
} }
Py_XDECREF(self->data); Py_XDECREF(self->data);
self->data = NULL; self->data = NULL;
self->jar = NULL; self->jar = NULL;
return 0; return 0;
} }
static int static int
cc_traverse(ccobject *self, visitproc visit, void *arg) cc_traverse(ccobject *self, visitproc visit, void *arg)
{ {
int err; int err;
CPersistentRing *here; CPersistentRing *here;
/* If we're in the midst of cleaning up old objects, the ring contains /* If we're in the midst of cleaning up old objects, the ring contains
* assorted junk we must not pass on to the visit() callback. This * assorted junk we must not pass on to the visit() callback. This
* should be rare (our cleanup code would need to have called back * should be rare (our cleanup code would need to have called back
* into Python, which in turn triggered Python's gc). When it happens, * into Python, which in turn triggered Python's gc). When it happens,
* simply don't chase any pointers. The cache will appear to be a * simply don't chase any pointers. The cache will appear to be a
* source of external references then, and at worst we miss cleaning * source of external references then, and at worst we miss cleaning
* up a dead cycle until the next time Python's gc runs. * up a dead cycle until the next time Python's gc runs.
*/ */
if (self->ring_lock) if (self->ring_lock)
return 0; return 0;
#define VISIT(SLOT) \
if (SLOT) { \
err = visit((PyObject *)(SLOT), arg); \
if (err) \
return err; \
}
VISIT(self->jar); #define VISIT(SLOT) \
if (SLOT) { \
err = visit((PyObject *)(SLOT), arg); \
if (err) \
return err; \
}
here = self->ring_home.r_next; VISIT(self->jar);
/* It is possible that an object is traversed after it is cleared. here = self->ring_home.r_next;
In that case, there is no ring.
*/
if (!here)
return 0;
while (here != &self->ring_home) { /* It is possible that an object is traversed after it is cleared.
cPersistentObject *o = OBJECT_FROM_RING(self, here); In that case, there is no ring.
VISIT(o); */
here = here->r_next; if (!here)
return 0;
while (here != &self->ring_home)
{
cPersistentObject *o = OBJECT_FROM_RING(self, here);
VISIT(o);
here = here->r_next;
} }
#undef VISIT #undef VISIT
return 0; return 0;
} }
static int static Py_ssize_t
cc_length(ccobject *self) cc_length(ccobject *self)
{ {
return PyObject_Length(self->data); return PyObject_Length(self->data);
} }
static PyObject * static PyObject *
cc_subscript(ccobject *self, PyObject *key) cc_subscript(ccobject *self, PyObject *key)
{ {
PyObject *r; PyObject *r;
r = PyDict_GetItem(self->data, key); r = PyDict_GetItem(self->data, key);
if (r == NULL) { if (r == NULL)
PyErr_SetObject(PyExc_KeyError, key); {
return NULL; PyErr_SetObject(PyExc_KeyError, key);
return NULL;
} }
Py_INCREF(r); Py_INCREF(r);
return r; return r;
} }
static int static int
cc_add_item(ccobject *self, PyObject *key, PyObject *v) cc_add_item(ccobject *self, PyObject *key, PyObject *v)
{ {
int result; int result;
PyObject *oid, *object_again, *jar; PyObject *oid, *object_again, *jar;
cPersistentObject *p; cPersistentObject *p;
/* Sanity check the value given to make sure it is allowed in the cache */ /* Sanity check the value given to make sure it is allowed in the cache */
if (PyType_Check(v)) { if (PyType_Check(v))
/* Its a persistent class, such as a ZClass. Thats ok. */ {
/* Its a persistent class, such as a ZClass. Thats ok. */
} }
else if (v->ob_type->tp_basicsize < sizeof(cPersistentObject)) { else if (v->ob_type->tp_basicsize < sizeof(cPersistentObject))
/* If it's not an instance of a persistent class, (ie Python {
classes that derive from persistent.Persistent, BTrees, /* If it's not an instance of a persistent class, (ie Python
etc), report an error. classes that derive from persistent.Persistent, BTrees,
etc), report an error.
TODO: checking sizeof() seems a poor test.
*/ TODO: checking sizeof() seems a poor test.
PyErr_SetString(PyExc_TypeError, */
"Cache values must be persistent objects."); PyErr_SetString(PyExc_TypeError,
return -1; "Cache values must be persistent objects.");
return -1;
} }
/* Can't access v->oid directly because the object might be a /* Can't access v->oid directly because the object might be a
* persistent class. * persistent class.
*/ */
oid = PyObject_GetAttr(v, py__p_oid); oid = PyObject_GetAttr(v, py__p_oid);
if (oid == NULL) if (oid == NULL)
return -1; return -1;
if (! PyString_Check(oid)) { if (! PyString_Check(oid))
PyErr_Format(PyExc_TypeError, {
"Cached object oid must be a string, not a %s", PyErr_Format(PyExc_TypeError,
oid->ob_type->tp_name); "Cached object oid must be a string, not a %s",
return -1; oid->ob_type->tp_name);
return -1;
} }
/* we know they are both strings. /* we know they are both strings.
* now check if they are the same string. * now check if they are the same string.
*/ */
result = PyObject_Compare(key, oid); result = PyObject_Compare(key, oid);
if (PyErr_Occurred()) { if (PyErr_Occurred())
Py_DECREF(oid); {
return -1; Py_DECREF(oid);
return -1;
} }
Py_DECREF(oid); Py_DECREF(oid);
if (result) { if (result)
PyErr_SetString(PyExc_ValueError, "Cache key does not match oid"); {
return -1; PyErr_SetString(PyExc_ValueError, "Cache key does not match oid");
return -1;
} }
/* useful sanity check, but not strictly an invariant of this class */ /* useful sanity check, but not strictly an invariant of this class */
jar = PyObject_GetAttr(v, py__p_jar); jar = PyObject_GetAttr(v, py__p_jar);
if (jar == NULL) if (jar == NULL)
return -1; return -1;
if (jar==Py_None) { if (jar==Py_None)
Py_DECREF(jar); {
PyErr_SetString(PyExc_ValueError, Py_DECREF(jar);
"Cached object jar missing"); PyErr_SetString(PyExc_ValueError,
return -1; "Cached object jar missing");
return -1;
} }
Py_DECREF(jar); Py_DECREF(jar);
object_again = PyDict_GetItem(self->data, key); object_again = PyDict_GetItem(self->data, key);
if (object_again) { if (object_again)
if (object_again != v) { {
PyErr_SetString(PyExc_ValueError, if (object_again != v)
"A different object already has the same oid"); {
return -1; PyErr_SetString(PyExc_ValueError,
} else { "A different object already has the same oid");
/* re-register under the same oid - no work needed */ return -1;
return 0; }
} else
{
/* re-register under the same oid - no work needed */
return 0;
}
} }
if (PyType_Check(v)) { if (PyType_Check(v))
if (PyDict_SetItem(self->data, key, v) < 0) {
return -1; if (PyDict_SetItem(self->data, key, v) < 0)
self->klass_count++; return -1;
return 0; self->klass_count++;
} else { return 0;
PerCache *cache = ((cPersistentObject *)v)->cache; }
if (cache) { else
if (cache != (PerCache *)self) {
/* This object is already in a different cache. */ PerCache *cache = ((cPersistentObject *)v)->cache;
PyErr_SetString(PyExc_ValueError, if (cache)
"Cache values may only be in one cache."); {
return -1; if (cache != (PerCache *)self)
} /* This object is already in a different cache. */
/* else: PyErr_SetString(PyExc_ValueError,
"Cache values may only be in one cache.");
This object is already one of ours, which is ok. It return -1;
would be very strange if someone was trying to register }
the same object under a different key. /* else:
*/
This object is already one of ours, which is ok. It
would be very strange if someone was trying to register
the same object under a different key.
*/
} }
if (PyDict_SetItem(self->data, key, v) < 0) if (PyDict_SetItem(self->data, key, v) < 0)
return -1; return -1;
/* the dict should have a borrowed reference */ /* the dict should have a borrowed reference */
Py_DECREF(v); Py_DECREF(v);
p = (cPersistentObject *)v; p = (cPersistentObject *)v;
Py_INCREF(self); Py_INCREF(self);
p->cache = (PerCache *)self; p->cache = (PerCache *)self;
if (p->state >= 0) { if (p->state >= 0)
/* insert this non-ghost object into the ring just {
behind the home position. */ /* insert this non-ghost object into the ring just
self->non_ghost_count++; behind the home position. */
ring_add(&self->ring_home, &p->ring); self->non_ghost_count++;
/* this list should have a new reference to the object */ ring_add(&self->ring_home, &p->ring);
Py_INCREF(v); /* this list should have a new reference to the object */
Py_INCREF(v);
} }
return 0; return 0;
} }
static int static int
cc_del_item(ccobject *self, PyObject *key) cc_del_item(ccobject *self, PyObject *key)
{ {
PyObject *v; PyObject *v;
cPersistentObject *p; cPersistentObject *p;
/* unlink this item from the ring */ /* unlink this item from the ring */
v = PyDict_GetItem(self->data, key); v = PyDict_GetItem(self->data, key);
if (v == NULL) { if (v == NULL)
PyErr_SetObject(PyExc_KeyError, key); {
return -1; PyErr_SetObject(PyExc_KeyError, key);
return -1;
} }
if (PyType_Check(v)) { if (PyType_Check(v))
self->klass_count--; {
} else { self->klass_count--;
p = (cPersistentObject *)v;
if (p->state >= 0) {
self->non_ghost_count--;
ring_del(&p->ring);
/* The DelItem below will account for the reference
held by the list. */
} else {
/* This is a ghost object, so we haven't kept a reference
count on it. For it have stayed alive this long
someone else must be keeping a reference to
it. Therefore we need to temporarily give it back a
reference count before calling DelItem below */
Py_INCREF(v);
}
Py_DECREF((PyObject *)p->cache);
p->cache = NULL;
} }
else
{
p = (cPersistentObject *)v;
if (p->state >= 0)
{
self->non_ghost_count--;
ring_del(&p->ring);
/* The DelItem below will account for the reference
held by the list. */
}
else
{
/* This is a ghost object, so we haven't kept a reference
count on it. For it have stayed alive this long
someone else must be keeping a reference to
it. Therefore we need to temporarily give it back a
reference count before calling DelItem below */
Py_INCREF(v);
}
if (PyDict_DelItem(self->data, key) < 0) { Py_DECREF((PyObject *)p->cache);
PyErr_SetString(PyExc_RuntimeError, p->cache = NULL;
"unexpectedly couldn't remove key in cc_ass_sub");
return -1;
} }
return 0; if (PyDict_DelItem(self->data, key) < 0)
{
PyErr_SetString(PyExc_RuntimeError,
"unexpectedly couldn't remove key in cc_ass_sub");
return -1;
}
return 0;
} }
static int static int
cc_ass_sub(ccobject *self, PyObject *key, PyObject *v) cc_ass_sub(ccobject *self, PyObject *key, PyObject *v)
{ {
if (!PyString_Check(key)) { if (!PyString_Check(key))
PyErr_Format(PyExc_TypeError, {
"cPickleCache key must be a string, not a %s", PyErr_Format(PyExc_TypeError,
key->ob_type->tp_name); "cPickleCache key must be a string, not a %s",
return -1; key->ob_type->tp_name);
return -1;
} }
if (v) if (v)
return cc_add_item(self, key, v); return cc_add_item(self, key, v);
else else
return cc_del_item(self, key); return cc_del_item(self, key);
} }
static PyMappingMethods cc_as_mapping = { static PyMappingMethods cc_as_mapping =
(inquiry)cc_length, /*mp_length*/ {
(binaryfunc)cc_subscript, /*mp_subscript*/ (lenfunc)cc_length, /*mp_length*/
(objobjargproc)cc_ass_sub, /*mp_ass_subscript*/ (binaryfunc)cc_subscript, /*mp_subscript*/
}; (objobjargproc)cc_ass_sub, /*mp_ass_subscript*/
};
static PyObject * static PyObject *
cc_cache_data(ccobject *self, void *context) cc_cache_data(ccobject *self, void *context)
{ {
return PyDict_Copy(self->data); return PyDict_Copy(self->data);
} }
static PyGetSetDef cc_getsets[] = { static PyGetSetDef cc_getsets[] =
{
{"cache_data", (getter)cc_cache_data}, {"cache_data", (getter)cc_cache_data},
{NULL} {NULL}
}; };
static PyMemberDef cc_members[] = { static PyMemberDef cc_members[] = {
{"cache_size", T_INT, offsetof(ccobject, cache_size)}, {"cache_size", T_INT, offsetof(ccobject, cache_size)},
{"cache_size_bytes", T_LONG, offsetof(ccobject, cache_size_bytes)}, {"cache_size_bytes", T_LONG, offsetof(ccobject, cache_size_bytes)},
{"total_estimated_size", T_LONG, offsetof(ccobject, total_estimated_size), {"total_estimated_size", T_LONG, offsetof(ccobject, total_estimated_size),
RO}, RO},
{"cache_drain_resistance", T_INT, {"cache_drain_resistance", T_INT,
offsetof(ccobject, cache_drain_resistance)}, offsetof(ccobject, cache_drain_resistance)},
{"cache_non_ghost_count", T_INT, offsetof(ccobject, non_ghost_count), RO}, {"cache_non_ghost_count", T_INT, offsetof(ccobject, non_ghost_count), RO},
{"cache_klass_count", T_INT, offsetof(ccobject, klass_count), RO}, {"cache_klass_count", T_INT, offsetof(ccobject, klass_count), RO},
{NULL} {NULL}
}; };
/* This module is compiled as a shared library. Some compilers don't /* This module is compiled as a shared library. Some compilers don't
...@@ -1079,82 +1142,83 @@ static PyMemberDef cc_members[] = { ...@@ -1079,82 +1142,83 @@ static PyMemberDef cc_members[] = {
#define DEFERRED_ADDRESS(ADDR) 0 #define DEFERRED_ADDRESS(ADDR) 0
static PyTypeObject Cctype = { static PyTypeObject Cctype = {
PyObject_HEAD_INIT(DEFERRED_ADDRESS(&PyType_Type)) PyObject_HEAD_INIT(DEFERRED_ADDRESS(&PyType_Type))
0, /* ob_size */ 0, /* ob_size */
"persistent.PickleCache", /* tp_name */ "persistent.PickleCache", /* tp_name */
sizeof(ccobject), /* tp_basicsize */ sizeof(ccobject), /* tp_basicsize */
0, /* tp_itemsize */ 0, /* tp_itemsize */
(destructor)cc_dealloc, /* tp_dealloc */ (destructor)cc_dealloc, /* tp_dealloc */
0, /* tp_print */ 0, /* tp_print */
0, /* tp_getattr */ 0, /* tp_getattr */
0, /* tp_setattr */ 0, /* tp_setattr */
0, /* tp_compare */ 0, /* tp_compare */
0, /* tp_repr */ 0, /* tp_repr */
0, /* tp_as_number */ 0, /* tp_as_number */
0, /* tp_as_sequence */ 0, /* tp_as_sequence */
&cc_as_mapping, /* tp_as_mapping */ &cc_as_mapping, /* tp_as_mapping */
0, /* tp_hash */ 0, /* tp_hash */
0, /* tp_call */ 0, /* tp_call */
0, /* tp_str */ 0, /* tp_str */
0, /* tp_getattro */ 0, /* tp_getattro */
0, /* tp_setattro */ 0, /* tp_setattro */
0, /* tp_as_buffer */ 0, /* tp_as_buffer */
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC,
/* tp_flags */ /* tp_flags */
0, /* tp_doc */ 0, /* tp_doc */
(traverseproc)cc_traverse, /* tp_traverse */ (traverseproc)cc_traverse, /* tp_traverse */
(inquiry)cc_clear, /* tp_clear */ (inquiry)cc_clear, /* tp_clear */
0, /* tp_richcompare */ 0, /* tp_richcompare */
0, /* tp_weaklistoffset */ 0, /* tp_weaklistoffset */
0, /* tp_iter */ 0, /* tp_iter */
0, /* tp_iternext */ 0, /* tp_iternext */
cc_methods, /* tp_methods */ cc_methods, /* tp_methods */
cc_members, /* tp_members */ cc_members, /* tp_members */
cc_getsets, /* tp_getset */ cc_getsets, /* tp_getset */
0, /* tp_base */ 0, /* tp_base */
0, /* tp_dict */ 0, /* tp_dict */
0, /* tp_descr_get */ 0, /* tp_descr_get */
0, /* tp_descr_set */ 0, /* tp_descr_set */
0, /* tp_dictoffset */ 0, /* tp_dictoffset */
(initproc)cc_init, /* tp_init */ (initproc)cc_init, /* tp_init */
}; };
void void
initcPickleCache(void) initcPickleCache(void)
{ {
PyObject *m; PyObject *m;
Cctype.ob_type = &PyType_Type; Cctype.ob_type = &PyType_Type;
Cctype.tp_new = &PyType_GenericNew; Cctype.tp_new = &PyType_GenericNew;
if (PyType_Ready(&Cctype) < 0) { if (PyType_Ready(&Cctype) < 0)
return; {
return;
} }
m = Py_InitModule3("cPickleCache", NULL, cPickleCache_doc_string); m = Py_InitModule3("cPickleCache", NULL, cPickleCache_doc_string);
capi = (cPersistenceCAPIstruct *)PyCObject_Import( capi = (cPersistenceCAPIstruct *)PyCObject_Import(
"persistent.cPersistence", "CAPI"); "persistent.cPersistence", "CAPI");
if (!capi) if (!capi)
return; return;
capi->percachedel = (percachedelfunc)cc_oid_unreferenced; capi->percachedel = (percachedelfunc)cc_oid_unreferenced;
py__p_changed = PyString_InternFromString("_p_changed"); py__p_changed = PyString_InternFromString("_p_changed");
if (!py__p_changed) if (!py__p_changed)
return; return;
py__p_deactivate = PyString_InternFromString("_p_deactivate"); py__p_deactivate = PyString_InternFromString("_p_deactivate");
if (!py__p_deactivate) if (!py__p_deactivate)
return; return;
py__p_jar = PyString_InternFromString("_p_jar"); py__p_jar = PyString_InternFromString("_p_jar");
if (!py__p_jar) if (!py__p_jar)
return; return;
py__p_oid = PyString_InternFromString("_p_oid"); py__p_oid = PyString_InternFromString("_p_oid");
if (!py__p_oid) if (!py__p_oid)
return; return;
if (PyModule_AddStringConstant(m, "cache_variant", "stiff/c") < 0) if (PyModule_AddStringConstant(m, "cache_variant", "stiff/c") < 0)
return; return;
/* This leaks a reference to Cctype, but it doesn't matter. */ /* This leaks a reference to Cctype, but it doesn't matter. */
if (PyModule_AddObject(m, "PickleCache", (PyObject *)&Cctype) < 0) if (PyModule_AddObject(m, "PickleCache", (PyObject *)&Cctype) < 0)
return; return;
} }
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7