Merged changes from Catalog-BTrees-Integration branch.

trunk/src/BTrees/BTreeItemsTemplate.c

@@ -237,76 +237,89 @@ BTreeItems_seek(BTreeItems *self, int i)
                                 
   /* Whew, we got here so we have a valid offset! */
 
-  while ((delta = i - pseudoindex) != 0) 
-    {
-      if (delta < 0) 
-        {
-          /* First, would we drop below zero? */
-          if (pseudoindex >= 0 && pseudoindex + delta < 0) goto no_match;
+  delta = i - pseudoindex;
+  if (delta)
+    while (delta)
+      {
+        if (delta < 0) 
+          {
+            /* First, would we drop below zero? */
+            if (pseudoindex >= 0 && pseudoindex + delta < 0) goto no_match;
       
-          /* Next, do we have to backup a bucket? */
-          if (currentoffset + delta < 0) 
-            {
-              if (currentbucket == self->firstbucket) goto no_match;
+            /* Next, do we have to backup a bucket? */
+            if (currentoffset + delta < 0) 
+              {
+                if (currentbucket == self->firstbucket) goto no_match;
               
-              b=PreviousBucket(currentbucket, self->firstbucket, i);
-              if (b==NULL) goto no_match;
+                b=PreviousBucket(currentbucket, self->firstbucket, i);
+                if (b==NULL) goto no_match;
               
-              PER_ALLOW_DEACTIVATION(currentbucket);
-              ASSIGNB(currentbucket, b);
-              UNLESS (PER_USE(currentbucket)) goto err;
-
-              delta += currentoffset;
-              pseudoindex -= currentoffset + 1;
-
-              if ((currentoffset = currentbucket->len - 1) < 0)
-                /* We backed into an empty bucket. Fix the psuedo index */
-                if (++pseudoindex == 0) goto no_match;
-            }
-          else 
-            {	/* Local adjustment */
-              pseudoindex += delta;
-              currentoffset += delta;
-            }
-
-        if (currentbucket == self->firstbucket &&
-            currentoffset < self->first) goto no_match;
-
-        } 
-      else if (delta > 0)
-        {
+                PER_ALLOW_DEACTIVATION(currentbucket);
+                ASSIGNB(currentbucket, b);
+                UNLESS (PER_USE(currentbucket)) goto err;
+
+                delta += currentoffset;
+                pseudoindex -= currentoffset + 1;
+
+                if ((currentoffset = currentbucket->len - 1) < 0)
+                  /* We backed into an empty bucket. Fix the psuedo index */
+                  if (++pseudoindex == 0) goto no_match;
+              }
+            else 
+              {	/* Local adjustment */
+                pseudoindex += delta;
+                currentoffset += delta;
+              }
+
+            if (currentbucket == self->firstbucket &&
+                currentoffset < self->first) goto no_match;
+
+          } 
+        else if (delta > 0)
+          {
           
-          /* Simple backwards range check */
-          if (pseudoindex < 0 && pseudoindex + delta >= 0) 
-            goto no_match;
+            /* Simple backwards range check */
+            if (pseudoindex < 0 && pseudoindex + delta >= 0) 
+              goto no_match;
           
-          /* Next, do we go forward a bucket? */
-          if (currentoffset + delta >= currentbucket->len) 
-            {
-              while (1)
-                {
-                  if ((b=currentbucket->next) == NULL) goto no_match;
-                  delta -= currentbucket->len - currentoffset;
-                  pseudoindex += (currentbucket->len - currentoffset);
-                  Py_INCREF(b);
-                  PER_ALLOW_DEACTIVATION(currentbucket);
-                  ASSIGNB(currentbucket, b);
-                  UNLESS (PER_USE(currentbucket)) goto err;
-                  currentoffset = 0;
-                  if (currentbucket->len) break;
-                } 
-            }
-          else
-            {	/* Local adjustment */
-              pseudoindex += delta;
-              currentoffset += delta;
-            }
-          if (currentbucket == self->lastbucket &&
-              currentoffset > self->last) goto no_match;
+            /* Next, do we go forward a bucket? */
+            if (currentoffset + delta >= currentbucket->len) 
+              {
+                while (1)
+                  {
+                    if (currentbucket == self->lastbucket) goto no_match;
+
+                    if ((b=currentbucket->next) == NULL) goto no_match;
+                    delta -= currentbucket->len - currentoffset;
+                    pseudoindex += (currentbucket->len - currentoffset);
+                    Py_INCREF(b);
+                    PER_ALLOW_DEACTIVATION(currentbucket);
+                    ASSIGNB(currentbucket, b);
+                    UNLESS (PER_USE(currentbucket)) goto err;
+                    currentoffset = 0;
+                    if (currentbucket->len) break;
+                  } 
+              }
+            else
+              {	/* Local adjustment */
+                pseudoindex += delta;
+                currentoffset += delta;
+              }
+            if (currentbucket == self->lastbucket &&
+                currentoffset > self->last) goto no_match;
           
-        }
+          }
+
+        delta = i - pseudoindex;
+      }
+  else
+    {                           /* Sanity check current bucket/offset */
+      if (currentbucket == self->firstbucket &&currentoffset < self->first)
+        goto no_match;
+      if (currentbucket == self->lastbucket && currentoffset > self->last)
+        goto no_match;
     }
-  
+
   PER_ALLOW_DEACTIVATION(currentbucket);
 
   if (currentbucket==self->currentbucket) Py_DECREF(currentbucket);
@@ -472,14 +485,26 @@ newBTreeItems(char kind,
 	
   UNLESS (self = PyObject_NEW(BTreeItems, &BTreeItemsType)) return NULL;
   self->kind=kind;
+
   self->first=lowoffset;
   self->last=highoffset;
-  Py_XINCREF(lowbucket);
-  self->firstbucket = lowbucket;
-  Py_XINCREF(highbucket);
-  self->lastbucket = highbucket;
-  Py_XINCREF(lowbucket);
-  self->currentbucket = lowbucket;
+
+  if (! lowbucket || (lowbucket==highbucket && lowoffset > highoffset))
+    {
+      self->firstbucket   = 0;
+      self->lastbucket    = 0;
+      self->currentbucket = 0;
+    }
+  else
+    {
+      Py_INCREF(lowbucket);
+      self->firstbucket = lowbucket;
+      Py_XINCREF(highbucket);
+      self->lastbucket = highbucket;
+      Py_XINCREF(lowbucket);
+      self->currentbucket = lowbucket;
+    }
+
   self->currentoffset = lowoffset;
   self->pseudoindex = 0;
 
@@ -525,6 +550,7 @@ nextBTreeItems(SetIteration *i)
           PyErr_Clear();
         }
     }
+  return 0;
 }
 
 static int 
@@ -558,4 +584,5 @@ nextTreeSetItems(SetIteration *i)
           PyErr_Clear();
         }
     }
+  return 0;
 }

trunk/src/BTrees/BTreeModuleTemplate.c

@@ -85,7 +85,7 @@
 
 static char BTree_module_documentation[] = 
 ""
-"\n$Id: BTreeModuleTemplate.c,v 1.4 2001/02/19 18:15:10 jim Exp $"
+"\n$Id: BTreeModuleTemplate.c,v 1.5 2001/03/15 13:16:22 jim Exp $"
 ;
 
 #ifdef PERSISTENT
@@ -113,7 +113,7 @@ static char BTree_module_documentation[] =
 #define PER_CHANGED(O) 0
 #endif
 
-PyObject *sort_str, *reverse_str;
+static PyObject *sort_str, *reverse_str, *items_str, *__setstate___str;
 
 static void PyVar_Assign(PyObject **v, PyObject *e) { Py_XDECREF(*v); *v=e;}
 #define ASSIGN(V,E) PyVar_Assign(&(V),(E))
@@ -129,6 +129,9 @@ static void PyVar_Assign(PyObject **v, PyObject *e) { Py_XDECREF(*v); *v=e;}
 
 #define SameType_Check(O1, O2) ((O1)->ob_type==(O2)->ob_type)
 
+#define ASSERT(C, S, R) if (! (C)) { \
+  PyErr_SetString(PyExc_AssertionError, (S)); return (R); }
+
 typedef struct BTreeItemStruct {
   KEY_TYPE key;
   PyObject *value;
@@ -163,6 +166,9 @@ typedef struct {
   BTreeItem *data;
 } BTree;
 
+staticforward PyExtensionClass BTreeType;
+
+
 #define BTREE(O) ((BTree*)(O))
 
 typedef struct SetIteration_s 
@@ -267,6 +273,8 @@ PyMalloc(size_t sz)
 {
   void *r;
 
+  ASSERT(sz > 0, "non-positive size malloc", NULL);
+
   if (r=malloc(sz)) return r;
 
   PyErr_NoMemory();
@@ -278,10 +286,14 @@ PyRealloc(void *p, size_t sz)
 {
   void *r;
 
-  if (r=realloc(p,sz)) return r;
+  ASSERT(sz > 0, "non-positive size realloc", NULL);
 
-  PyErr_NoMemory();
-  return NULL;
+  if (p) r=realloc(p,sz);
+  else r=malloc(sz);
+
+  UNLESS (r) PyErr_NoMemory();
+
+  return r;
 }
 
 #include "BTreeItemsTemplate.c"
@@ -290,6 +302,7 @@ PyRealloc(void *p, size_t sz)
 #include "BTreeTemplate.c"
 #include "TreeSetTemplate.c"
 #include "SetOpTemplate.c"
+#include "MergeTemplate.c"
 
 static struct PyMethodDef module_methods[] = {
   {"difference", (PyCFunction) difference_m,	METH_VARARGS,
@@ -324,6 +337,8 @@ INITMODULE ()
 
   UNLESS (sort_str=PyString_FromString("sort")) return;
   UNLESS (reverse_str=PyString_FromString("reverse")) return;
+  UNLESS (items_str=PyString_FromString("items")) return;
+  UNLESS (__setstate___str=PyString_FromString("__setstate__")) return;
 
   UNLESS (PyExtensionClassCAPI=PyCObject_Import("ExtensionClass","CAPI"))
       return;
@@ -372,7 +387,7 @@ INITMODULE ()
   d = PyModule_GetDict(m);
 
   PyDict_SetItemString(d, "__version__",
-		       PyString_FromString("$Revision: 1.4 $"));
+		       PyString_FromString("$Revision: 1.5 $"));
 
   PyExtensionClass_Export(d,PREFIX "Bucket", BucketType);
   PyExtensionClass_Export(d,PREFIX "BTree", BTreeType);

trunk/src/BTrees/IOBTree.c

@@ -2,7 +2,7 @@
 #define PERSISTENT
 
 #define PREFIX "IO"
-#define DEFAULT_MAX_BUCKET_SIZE 30
+#define DEFAULT_MAX_BUCKET_SIZE 60
 #define DEFAULT_MAX_BTREE_SIZE 500
 #define INITMODULE initIOBTree
                                 

trunk/src/BTrees/Interfaces.py

+##############################################################################
+# 
+# Zope Public License (ZPL) Version 1.0
+# -------------------------------------
+# 
+# Copyright (c) Digital Creations.  All rights reserved.
+# 
+# This license has been certified as Open Source(tm).
+# 
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+# 
+# 1. Redistributions in source code must retain the above copyright
+#    notice, this list of conditions, and the following disclaimer.
+# 
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions, and the following disclaimer in
+#    the documentation and/or other materials provided with the
+#    distribution.
+# 
+# 3. Digital Creations requests that attribution be given to Zope
+#    in any manner possible. Zope includes a "Powered by Zope"
+#    button that is installed by default. While it is not a license
+#    violation to remove this button, it is requested that the
+#    attribution remain. A significant investment has been put
+#    into Zope, and this effort will continue if the Zope community
+#    continues to grow. This is one way to assure that growth.
+# 
+# 4. All advertising materials and documentation mentioning
+#    features derived from or use of this software must display
+#    the following acknowledgement:
+# 
+#      "This product includes software developed by Digital Creations
+#      for use in the Z Object Publishing Environment
+#      (http://www.zope.org/)."
+# 
+#    In the event that the product being advertised includes an
+#    intact Zope distribution (with copyright and license included)
+#    then this clause is waived.
+# 
+# 5. Names associated with Zope or Digital Creations must not be used to
+#    endorse or promote products derived from this software without
+#    prior written permission from Digital Creations.
+# 
+# 6. Modified redistributions of any form whatsoever must retain
+#    the following acknowledgment:
+# 
+#      "This product includes software developed by Digital Creations
+#      for use in the Z Object Publishing Environment
+#      (http://www.zope.org/)."
+# 
+#    Intact (re-)distributions of any official Zope release do not
+#    require an external acknowledgement.
+# 
+# 7. Modifications are encouraged but must be packaged separately as
+#    patches to official Zope releases.  Distributions that do not
+#    clearly separate the patches from the original work must be clearly
+#    labeled as unofficial distributions.  Modifications which do not
+#    carry the name Zope may be packaged in any form, as long as they
+#    conform to all of the clauses above.
+# 
+# 
+# Disclaimer
+# 
+#   THIS SOFTWARE IS PROVIDED BY DIGITAL CREATIONS ``AS IS'' AND ANY
+#   EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+#   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+#   PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL DIGITAL CREATIONS OR ITS
+#   CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+#   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+#   USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+#   ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+#   OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+#   OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+#   SUCH DAMAGE.
+# 
+# 
+# This software consists of contributions made by Digital Creations and
+# many individuals on behalf of Digital Creations.  Specific
+# attributions are listed in the accompanying credits file.
+# 
+##############################################################################
 
 import OOBTree, Interface
 
@@ -74,6 +158,9 @@ class ISetMutable(IKeyed):
 
     def remove(key):
         """Remove the key from the set."""
+        
+    def update(seq):
+        """Add the items from the given sequence to the set"""
 
 class IKeySequence(IKeyed, ISized):
 
@@ -89,6 +176,7 @@ class ISet(IKeySequence, ISetMutable):
 
 class ITreeSet(IKeyed, ISetMutable):
     pass
+    
 
 class IDictionaryIsh(IKeyed, ISized):
 
@@ -116,6 +204,14 @@ class IDictionaryIsh(IKeyed, ISized):
 
         Raise a key error if the key if not in the collection."""
 
+    def update(collection):
+        """Add the items from the given collection object to the collection
+
+        The input collection must be a sequence of key-value tuples,
+        or an object with an 'items' method that returns a sequence of
+        key-value tuples.
+        """
+
     def values(min=None, max=None):
         """Return a IReadSequence containing the values in the collection
 

trunk/src/BTrees/Length.py

+##############################################################################
+# 
+# Zope Public License (ZPL) Version 1.0
+# -------------------------------------
+# 
+# Copyright (c) Digital Creations.  All rights reserved.
+# 
+# This license has been certified as Open Source(tm).
+# 
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+# 
+# 1. Redistributions in source code must retain the above copyright
+#    notice, this list of conditions, and the following disclaimer.
+# 
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions, and the following disclaimer in
+#    the documentation and/or other materials provided with the
+#    distribution.
+# 
+# 3. Digital Creations requests that attribution be given to Zope
+#    in any manner possible. Zope includes a "Powered by Zope"
+#    button that is installed by default. While it is not a license
+#    violation to remove this button, it is requested that the
+#    attribution remain. A significant investment has been put
+#    into Zope, and this effort will continue if the Zope community
+#    continues to grow. This is one way to assure that growth.
+# 
+# 4. All advertising materials and documentation mentioning
+#    features derived from or use of this software must display
+#    the following acknowledgement:
+# 
+#      "This product includes software developed by Digital Creations
+#      for use in the Z Object Publishing Environment
+#      (http://www.zope.org/)."
+# 
+#    In the event that the product being advertised includes an
+#    intact Zope distribution (with copyright and license included)
+#    then this clause is waived.
+# 
+# 5. Names associated with Zope or Digital Creations must not be used to
+#    endorse or promote products derived from this software without
+#    prior written permission from Digital Creations.
+# 
+# 6. Modified redistributions of any form whatsoever must retain
+#    the following acknowledgment:
+# 
+#      "This product includes software developed by Digital Creations
+#      for use in the Z Object Publishing Environment
+#      (http://www.zope.org/)."
+# 
+#    Intact (re-)distributions of any official Zope release do not
+#    require an external acknowledgement.
+# 
+# 7. Modifications are encouraged but must be packaged separately as
+#    patches to official Zope releases.  Distributions that do not
+#    clearly separate the patches from the original work must be clearly
+#    labeled as unofficial distributions.  Modifications which do not
+#    carry the name Zope may be packaged in any form, as long as they
+#    conform to all of the clauses above.
+# 
+# 
+# Disclaimer
+# 
+#   THIS SOFTWARE IS PROVIDED BY DIGITAL CREATIONS ``AS IS'' AND ANY
+#   EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+#   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+#   PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL DIGITAL CREATIONS OR ITS
+#   CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+#   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+#   USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+#   ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+#   OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+#   OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+#   SUCH DAMAGE.
+# 
+# 
+# This software consists of contributions made by Digital Creations and
+# many individuals on behalf of Digital Creations.  Specific
+# attributions are listed in the accompanying credits file.
+# 
+##############################################################################
+
+import Persistence
+
+class Length(Persistence.Persistent):
+    """BTree lengths aqre too expensive to compute
+
+    Objects that use BTrees need to keep track of lengths themselves.
+    This class provides an object for doing this.
+
+    As a bonus, the object support application-level conflict resolution.
+    """
+
+    def __init__(self, v=0): self.value=v
+
+    def __getstate__(self): return self.value
+
+    def __setstate__(self, v): self.value=v
+
+    def set(self, v): self.value=v
+
+    def _p_resolveConflict(self, old, s1, s2): return s1 + s2 - old
+
+    def _p_independent(self):
+        # My state doesn't depend on or materially effect the state of
+        # other objects.
+        return 1
+
+    def change(self, delta): self.value = self.value + delta
+
+    def __call__(self, *args): return self.value

trunk/src/BTrees/SetOpTemplate.c

@@ -110,6 +110,23 @@ nextIntSet(SetIteration *i)
 }
 #endif
 
+#ifdef KEY_CHECK
+static int 
+nextKeyAsSet(SetIteration *i)
+{
+  if (i->position >= 0)
+    {
+      if (i->position < 1)
+        {
+          i->position ++;
+        }
+      else
+        i->position = -1;
+    }
+  return 0;
+}
+#endif
+
 static int
 initSetIteration(SetIteration *i, PyObject *s, int w, int *merge)
 {
@@ -169,6 +186,19 @@ initSetIteration(SetIteration *i, PyObject *s, int w, int *merge)
       i->next=nextIntSet;
       i->hasValue=0;
     }
+#endif
+#ifdef KEY_CHECK
+  else if (KEY_CHECK(s))
+    {
+      int copied=1;
+
+      i->set = s;
+      Py_INCREF(s);
+      i->next=nextKeyAsSet;
+      i->hasValue=0;
+      COPY_KEY_FROM_ARG(i->key, s, &copied);
+      UNLESS (copied) return -1;
+    }
 #endif
   else
     {
@@ -300,7 +330,7 @@ set_operation(PyObject *s1, PyObject *s2,
 	{
 	  if(c2)
 	    {
-	      if(r->len >= r->size && Bucket_grow(r,1) < 0) goto err;
+	      if(r->len >= r->size && Bucket_grow(r, ! merge) < 0) goto err;
               COPY_KEY(r->keys[r->len], i2.key);
               INCREF_KEY(r->keys[r->len]);
               if (merge)

trunk/src/BTrees/SetTemplate.c

@@ -90,58 +90,59 @@ Set_insert(Bucket *self, PyObject *args)
   int i;
 
   UNLESS (PyArg_ParseTuple(args, "O", &key)) return NULL;
-  if ( (i=_bucket_set(self, key, Py_None, 1, 1)) < 0) return NULL;
+  if ( (i=_bucket_set(self, key, Py_None, 1, 1, 0)) < 0) return NULL;
   return PyInt_FromLong(i);
 }
 
 static PyObject *
-Set_remove(Bucket *self, PyObject *args)
-{
-  PyObject *key;
-
-  UNLESS (PyArg_ParseTuple(args, "O", &key)) return NULL;
-  if (_bucket_set(self, key, NULL, 0, 1) < 0) return NULL;
-
-  Py_INCREF(Py_None);
-  return Py_None;
-}
-
-static PyObject *
-set_getstate(Bucket *self, PyObject *args)
+Set_update(Bucket *self, PyObject *args)
 {
-  PyObject *r=0, *o=0, *items=0;
-  int i, l;
+  PyObject *seq=0, *o, *t, *v, *tb;
+  int i, n=0, ind;
 
-  PER_USE_OR_RETURN(self, NULL);
-
-  l=self->len;
+  UNLESS(PyArg_ParseTuple(args, "|O:update", &seq)) return NULL;
 
-  UNLESS (items=PyTuple_New(self->len)) goto err;
-  for (i=0; i<l; i++)
+  if (seq)
     {
-      COPY_KEY_TO_OBJECT(o, self->keys[i]);
-      UNLESS (o) goto err;
-      PyTuple_SET_ITEM(items, i, o);
+      for (i=0; ; i++)
+        {
+          UNLESS (o=PySequence_GetItem(seq, i))
+            {
+              PyErr_Fetch(&t, &v, &tb);
+              if (t != PyExc_IndexError)
+                {
+                  PyErr_Restore(t, v, tb);
+                  return NULL;
+                }
+              Py_XDECREF(t);
+              Py_XDECREF(v);
+              Py_XDECREF(tb);
+              break;
+            }
+          ind=_bucket_set(self, o, Py_None, 1, 1, 0);
+          Py_DECREF(o);
+          if (ind < 0) return NULL;
+          n += ind;
+        }
     }
 
-  if (self->next) 
-    r=Py_BuildValue("OO", items, self->next);
-  else
-    r=Py_BuildValue("(O)", items);
+  return PyInt_FromLong(n);
+}
 
-  PER_ALLOW_DEACTIVATION(self);
+static PyObject *
+Set_remove(Bucket *self, PyObject *args)
+{
+  PyObject *key;
 
-  return r;
+  UNLESS (PyArg_ParseTuple(args, "O", &key)) return NULL;
+  if (_bucket_set(self, key, NULL, 0, 1, 0) < 0) return NULL;
 
-err:
-  PER_ALLOW_DEACTIVATION(self);
-  Py_XDECREF(items);
-  Py_XDECREF(r);
-  return NULL;
+  Py_INCREF(Py_None);
+  return Py_None;
 }
 
-static PyObject *
-set_setstate(Bucket *self, PyObject *args)
+static int
+_set_setstate(Bucket *self, PyObject *args)
 {
   PyObject *k, *items;
   Bucket *next=0;
@@ -149,14 +150,10 @@ set_setstate(Bucket *self, PyObject *args)
   KEY_TYPE *keys;
   VALUE_TYPE *values;
 
-  PER_PREVENT_DEACTIVATION(self); 
+  UNLESS (PyArg_ParseTuple(args, "O|O", &items, &next))
+    return -1;
 
-  UNLESS (PyArg_ParseTuple(args, "O", &args)) goto err;
-
-  UNLESS (PyArg_ParseTuple(args, "O|O!", &items, self->ob_type, &next))
-    goto err;
-
-  if ((l=PyTuple_Size(items)) < 0) goto err;
+  if ((l=PyTuple_Size(items)) < 0) return -1;
 
   for (i=self->len; --i >= 0; )
     {
@@ -172,7 +169,7 @@ set_setstate(Bucket *self, PyObject *args)
   
   if (l > self->size)
     {
-      UNLESS (keys=PyRealloc(self->keys, sizeof(KEY_TYPE)*l)) goto err;
+      UNLESS (keys=PyRealloc(self->keys, sizeof(KEY_TYPE)*l)) return -1;
       self->keys=keys;
       self->size=l;
     }
@@ -181,25 +178,39 @@ set_setstate(Bucket *self, PyObject *args)
     {
       k=PyTuple_GET_ITEM(items, i);
       COPY_KEY_FROM_ARG(self->keys[i], k, &copied);
-      UNLESS (copied) return NULL;
-      INCREF_KEY(k);
+      UNLESS (copied) return -1;
+      INCREF_KEY(self->keys[i]);
     }
 
   self->len=l;
 
+  if (next)
+    {
+      self->next=next;
+      Py_INCREF(next);
+    }
+
+  return 0;
+}
+
+static PyObject *
+set_setstate(Bucket *self, PyObject *args)
+{
+  int r;
+
+  UNLESS (PyArg_ParseTuple(args, "O", &args)) return NULL;
+
+  PER_PREVENT_DEACTIVATION(self); 
+  r=_set_setstate(self, args);
   PER_ALLOW_DEACTIVATION(self);
+
+  if (r < 0) return NULL;
   Py_INCREF(Py_None);
   return Py_None;
-
- perr:
-  self->len=i;
- err:
-  PER_ALLOW_DEACTIVATION(self);
-  return NULL;
 }
 
 static struct PyMethodDef Set_methods[] = {
-  {"__getstate__", (PyCFunction) set_getstate,	METH_VARARGS,
+  {"__getstate__", (PyCFunction) bucket_getstate,	METH_VARARGS,
    "__getstate__() -- Return the picklable state of the object"},
   {"__setstate__", (PyCFunction) set_setstate,	METH_VARARGS,
    "__setstate__() -- Set the state of the object"},
@@ -216,11 +227,17 @@ static struct PyMethodDef Set_methods[] = {
    "minKey([key]) -- Fine the minimum key\n\n"
    "If an argument is given, find the minimum >= the argument"},
 #ifdef PERSISTENT
+  {"_p_resolveConflict", (PyCFunction) bucket__p_resolveConflict, METH_VARARGS,
+   "_p_resolveConflict() -- Reinitialize from a newly created copy"},
   {"_p_deactivate", (PyCFunction) bucket__p_deactivate, METH_VARARGS,
    "_p_deactivate() -- Reinitialize from a newly created copy"},
 #endif
   {"insert",	(PyCFunction)Set_insert,	METH_VARARGS,
    "insert(id,[ignored]) -- Add a key to the set"},
+  {"update",	(PyCFunction)Set_update,	METH_VARARGS,
+   "update(seq) -- Add the items from the given sequence to the set"},
+  {"__init__",	(PyCFunction)Set_update,	METH_VARARGS,
+   "__init__(seq) -- Initialize with  the items from the given sequence"},
   {"remove",	(PyCFunction)Set_remove,	METH_VARARGS,
    "remove(id) -- Remove an id from the set"},
 

trunk/src/BTrees/TreeSetTemplate.c

@@ -94,6 +94,42 @@ TreeSet_insert(BTree *self, PyObject *args)
   return PyInt_FromLong(i);
 }
 
+static PyObject *
+TreeSet_update(BTree *self, PyObject *args)
+{
+  PyObject *seq=0, *o, *t, *v, *tb;
+  int i, n=0, ind;
+
+  UNLESS(PyArg_ParseTuple(args, "|O:update", &seq)) return NULL;
+
+  if (seq)
+    {
+      for (i=0; ; i++)
+        {
+          UNLESS (o=PySequence_GetItem(seq, i))
+            {
+              PyErr_Fetch(&t, &v, &tb);
+              if (t != PyExc_IndexError)
+                {
+                  PyErr_Restore(t, v, tb);
+                  return NULL;
+                }
+              Py_XDECREF(t);
+              Py_XDECREF(v);
+              Py_XDECREF(tb);
+              break;
+            }
+          ind=_BTree_set(self, o, Py_None, 1, 1);
+          Py_DECREF(o);
+          if (ind < 0) return NULL;
+          n += ind;
+        }
+    }
+
+  return PyInt_FromLong(n);
+}
+
+
 static PyObject *
 TreeSet_remove(BTree *self, PyObject *args)
 {
@@ -105,10 +141,26 @@ TreeSet_remove(BTree *self, PyObject *args)
   return Py_None;
 }
 
+static PyObject *
+TreeSet_setstate(BTree *self, PyObject *args)
+{
+  int r;
+
+  if (!PyArg_ParseTuple(args,"O",&args)) return NULL;
+ 
+  PER_PREVENT_DEACTIVATION(self); 
+  r=_BTree_setstate(self, args, 1);
+  PER_ALLOW_DEACTIVATION(self);
+
+  if (r < 0) return NULL;
+  Py_INCREF(Py_None);
+  return Py_None;
+}
+
 static struct PyMethodDef TreeSet_methods[] = {
   {"__getstate__", (PyCFunction) BTree_getstate,	METH_VARARGS,
    "__getstate__() -- Return the picklable state of the object"},
-  {"__setstate__", (PyCFunction) BTree_setstate,	METH_VARARGS,
+  {"__setstate__", (PyCFunction) TreeSet_setstate,	METH_VARARGS,
    "__setstate__() -- Set the state of the object"},
   {"has_key",	(PyCFunction) BTree_has_key,	METH_VARARGS,
      "has_key(key) -- Test whether the bucket contains the given key"},
@@ -124,9 +176,15 @@ static struct PyMethodDef TreeSet_methods[] = {
    "clear() -- Remove all of the items from the BTree"},  
   {"insert",	(PyCFunction)TreeSet_insert,	METH_VARARGS,
    "insert(id,[ignored]) -- Add an id to the set"},
+  {"update",	(PyCFunction)TreeSet_update,	METH_VARARGS,
+   "update(seq) -- Add the items from the given sequence to the set"},
+  {"__init__",	(PyCFunction)TreeSet_update,	METH_VARARGS,
+   "__init__(seq) -- Initialize with  the items from the given sequence"},
   {"remove",	(PyCFunction)TreeSet_remove,	METH_VARARGS,
    "remove(id) -- Remove a key from the set"},
 #ifdef PERSISTENT
+  {"_p_resolveConflict", (PyCFunction) BTree__p_resolveConflict, METH_VARARGS,
+   "_p_resolveConflict() -- Reinitialize from a newly created copy"},
   {"_p_deactivate", (PyCFunction) BTree__p_deactivate,	METH_VARARGS,
    "_p_deactivate() -- Reinitialize from a newly created copy"},
 #endif

trunk/src/BTrees/convert.py

+##############################################################################
+# 
+# Zope Public License (ZPL) Version 1.0
+# -------------------------------------
+# 
+# Copyright (c) Digital Creations.  All rights reserved.
+# 
+# This license has been certified as Open Source(tm).
+# 
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+# 
+# 1. Redistributions in source code must retain the above copyright
+#    notice, this list of conditions, and the following disclaimer.
+# 
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions, and the following disclaimer in
+#    the documentation and/or other materials provided with the
+#    distribution.
+# 
+# 3. Digital Creations requests that attribution be given to Zope
+#    in any manner possible. Zope includes a "Powered by Zope"
+#    button that is installed by default. While it is not a license
+#    violation to remove this button, it is requested that the
+#    attribution remain. A significant investment has been put
+#    into Zope, and this effort will continue if the Zope community
+#    continues to grow. This is one way to assure that growth.
+# 
+# 4. All advertising materials and documentation mentioning
+#    features derived from or use of this software must display
+#    the following acknowledgement:
+# 
+#      "This product includes software developed by Digital Creations
+#      for use in the Z Object Publishing Environment
+#      (http://www.zope.org/)."
+# 
+#    In the event that the product being advertised includes an
+#    intact Zope distribution (with copyright and license included)
+#    then this clause is waived.
+# 
+# 5. Names associated with Zope or Digital Creations must not be used to
+#    endorse or promote products derived from this software without
+#    prior written permission from Digital Creations.
+# 
+# 6. Modified redistributions of any form whatsoever must retain
+#    the following acknowledgment:
+# 
+#      "This product includes software developed by Digital Creations
+#      for use in the Z Object Publishing Environment
+#      (http://www.zope.org/)."
+# 
+#    Intact (re-)distributions of any official Zope release do not
+#    require an external acknowledgement.
+# 
+# 7. Modifications are encouraged but must be packaged separately as
+#    patches to official Zope releases.  Distributions that do not
+#    clearly separate the patches from the original work must be clearly
+#    labeled as unofficial distributions.  Modifications which do not
+#    carry the name Zope may be packaged in any form, as long as they
+#    conform to all of the clauses above.
+# 
+# 
+# Disclaimer
+# 
+#   THIS SOFTWARE IS PROVIDED BY DIGITAL CREATIONS ``AS IS'' AND ANY
+#   EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+#   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+#   PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL DIGITAL CREATIONS OR ITS
+#   CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+#   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+#   USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+#   ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+#   OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+#   OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+#   SUCH DAMAGE.
+# 
+# 
+# This software consists of contributions made by Digital Creations and
+# many individuals on behalf of Digital Creations.  Specific
+# attributions are listed in the accompanying credits file.
+# 
+##############################################################################
+
+def convert(old, new, threshold=200, f=None, None=None):
+    "Utility for converting old btree new new"
+    n=0
+    for k, v in old.items():
+        if f is not None: v=f(v)
+        new[k]=v
+        n=n+1
+        if n > threshold:
+            get_transaction().commit(1)
+            old._p_jar.cacheMinimize(3)
+            n=0
+
+    get_transaction().commit(1)
+    old._p_jar.cacheMinimize(3)

trunk/src/BTrees/intkeymacros.h

 #define KEY_TYPE int
-#define TEST_KEY(KEY, TARGET) ( (KEY) - (TARGET) )
+#define KEY_CHECK PyInt_Check
+#define TEST_KEY(K, T) (((K) < (T)) ? -1 : (((K) > (T)) ? 1: 0)) 
 #define DECREF_KEY(KEY)
 #define INCREF_KEY(k)
 #define COPY_KEY(KEY, E) (KEY=(E))

trunk/src/BTrees/intvaluemacros.h

 #define VALUE_TYPE int
-#define TEST_VALUE(VALUE, TARGET) ( (VALUE) - (TARGET) )
+#define TEST_VALUE(K, T) (((K) < (T)) ? -1 : (((K) > (T)) ? 1: 0)) 
+#define VALUE_SAME(VALUE, TARGET) ( (VALUE) == (TARGET) )
 #define DECLARE_VALUE(NAME) VALUE_TYPE NAME
 #define VALUE_PARSE "i"
 #define DECREF_VALUE(k)

trunk/src/BTrees/tests/__init__.py

+# If tests is a package, debugging is a bit easier.
+

trunk/src/BTrees/tests/testBTrees.py

@@ -122,56 +122,65 @@ class Base:
         os.system('rm fs_tmp__*')
                 
     def testLoadAndStore(self):
-        t = self.t
-        try:
-            root = self._getRoot()
-            root['t'] = t
-            get_transaction().commit()
-        except:
-            self._closeDB(root)
-            self._delDB()
-            raise
-
-        self._closeDB(root)
-
-        try:
-            root = self._getRoot()
-            #XXX BTree stuff doesn't implement comparison
-            if hasattr(t, 'items'):
-                assert list(root['t'].items()) == list(t.items())
-            else:
-                assert list(root['t'].keys()) == list(t.keys())
-        finally:
+        for i in 0, 10, 1000:
+            t = self.t.__class__()
+            self._populate(t, i)
+            try:
+                root = self._getRoot()
+                root[i] = t
+                get_transaction().commit()
+            except:
+                self._closeDB(root)
+                self._delDB()
+                raise
+
             self._closeDB(root)
-            self._delDB()
+
+            try:
+                root = self._getRoot()
+                #XXX BTree stuff doesn't implement comparison
+                if hasattr(t, 'items'):
+                    assert list(root[i].items()) == list(t.items())
+                else:
+                    assert list(root[i].keys()) == list(t.keys())
+            finally:
+                self._closeDB(root)
+                self._delDB()
             
     def testGhostUnghost(self):
-        t = self.t
-        try:
-            root = self._getRoot()
-            root['t'] = t
-            get_transaction().commit()
-        except:
-            self._closeDB(root)
-            self._delDB()
-            raise
-
-        self._closeDB(root)
-
-        try:
-            root = self._getRoot()
-            root['t']._p_changed = None
-            get_transaction().commit()
-            if hasattr(t,'items'):
-                assert list(root['t'].items()) == list(t.items())
-            else:
-                assert list(root['t'].keys()) == list(t.keys())
-        finally:
+        for i in 0, 10, 1000:
+            t = self.t.__class__()
+            self._populate(t, i)
+            try:
+                root = self._getRoot()
+                root[i] = t
+                get_transaction().commit()
+            except:
+                self._closeDB(root)
+                self._delDB()
+                raise
+
             self._closeDB(root)
-            self._delDB()
+
+            try:
+                root = self._getRoot()
+                root[i]._p_changed = None
+                get_transaction().commit()
+                if hasattr(t,'items'):
+                    assert list(root[i].items()) == list(t.items())
+                else:
+                    assert list(root[i].keys()) == list(t.keys())
+            finally:
+                self._closeDB(root)
+                self._delDB()
 
 class MappingBase(Base):
     """ Tests common to mappings (buckets, btrees) """
+
+    def _populate(self, t, l):
+        # Make some data
+        for i in range(l): t[i]=i
+    
     def testGetItemFails(self):
         self.assertRaises(KeyError, self._getitemfail)
 
@@ -271,8 +280,45 @@ class MappingBase(Base):
         diff = lsubtract(list(self.t.keys()), [])
         assert diff == [], diff
 
+    def testUpdate(self):
+        "mapping update"
+        d={}
+        l=[]
+        for i in range(10000):
+            k=whrandom.randint(-2000, 2000)
+            d[k]=i
+            l.append((k, i))
+            
+        items=d.items()
+        items.sort()
+
+        self.t.update(d)
+        assert list(self.t.items()) == items
+
+        self.t.clear()
+        assert list(self.t.items()) == []
+
+        self.t.update(l)
+        assert list(self.t.items()) == items
+
+    def testEmptyRangeSearches(self):
+        t=self.t
+        t.update([(1,1),(5,5),(9,9)])
+        assert list(t.keys(-6,-4))==[], list(t.keys(-6,-4))
+        assert list(t.keys(2,4))==[], list(t.keys(2,4))
+        assert list(t.keys(6,8))==[], list(t.keys(6,8))
+        assert list(t.keys(10,12))==[], list(t.keys(10,12))
+        
+
 class NormalSetTests(Base):
     """ Test common to all set types """
+
+
+    def _populate(self, t, l): 
+        # Make some data
+        t.update(range(l))
+
+
     def testInsertReturnsValue(self):
         t = self.t
         assert t.insert(5) == 1
@@ -343,6 +389,29 @@ class NormalSetTests(Base):
         assert t.minKey(3) == 3
         assert t.minKey(9) == 10
 
+    def testUpdate(self):
+        "mapping update"
+        d={}
+        l=[]
+        for i in range(10000):
+            k=whrandom.randint(-2000, 2000)
+            d[k]=i
+            l.append(k)
+            
+        items=d.keys()
+        items.sort()
+
+        self.t.update(l)
+        assert list(self.t.keys()) == items
+
+    def testEmptyRangeSearches(self):
+        t=self.t
+        t.update([1,5,9])
+        assert list(t.keys(-6,-4))==[], list(t.keys(-6,-4))
+        assert list(t.keys(2,4))==[], list(t.keys(2,4))
+        assert list(t.keys(6,8))==[], list(t.keys(6,8))
+        assert list(t.keys(10,12))==[], list(t.keys(10,12))
+
 class ExtendedSetTests(NormalSetTests):
     def testLen(self):
         t = self.t

trunk/src/ZODB/BaseStorage.py

@@ -84,9 +84,10 @@
 ##############################################################################
 """Handy standard storage machinery
 """
-__version__='$Revision: 1.10 $'[11:-2]
+__version__='$Revision: 1.11 $'[11:-2]
 
-import time, bpthread, UndoLogCompatible
+import ThreadLock, bpthread
+import time, UndoLogCompatible
 from POSException import UndoError
 from TimeStamp import TimeStamp
 z64='\0'*8
@@ -101,7 +102,7 @@ class BaseStorage(UndoLogCompatible.UndoLogCompatible):
         self.__name__=name
 
         # Allocate locks:
-        l=bpthread.allocate_lock()
+        l=ThreadLock.allocate_lock()
         self._lock_acquire=l.acquire
         self._lock_release=l.release
         l=bpthread.allocate_lock()

trunk/src/ZODB/ConflictResolution.py

+##############################################################################
+# 
+# Zope Public License (ZPL) Version 1.0
+# -------------------------------------
+# 
+# Copyright (c) Digital Creations.  All rights reserved.
+# 
+# This license has been certified as Open Source(tm).
+# 
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+# 
+# 1. Redistributions in source code must retain the above copyright
+#    notice, this list of conditions, and the following disclaimer.
+# 
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions, and the following disclaimer in
+#    the documentation and/or other materials provided with the
+#    distribution.
+# 
+# 3. Digital Creations requests that attribution be given to Zope
+#    in any manner possible. Zope includes a "Powered by Zope"
+#    button that is installed by default. While it is not a license
+#    violation to remove this button, it is requested that the
+#    attribution remain. A significant investment has been put
+#    into Zope, and this effort will continue if the Zope community
+#    continues to grow. This is one way to assure that growth.
+# 
+# 4. All advertising materials and documentation mentioning
+#    features derived from or use of this software must display
+#    the following acknowledgement:
+# 
+#      "This product includes software developed by Digital Creations
+#      for use in the Z Object Publishing Environment
+#      (http://www.zope.org/)."
+# 
+#    In the event that the product being advertised includes an
+#    intact Zope distribution (with copyright and license included)
+#    then this clause is waived.
+# 
+# 5. Names associated with Zope or Digital Creations must not be used to
+#    endorse or promote products derived from this software without
+#    prior written permission from Digital Creations.
+# 
+# 6. Modified redistributions of any form whatsoever must retain
+#    the following acknowledgment:
+# 
+#      "This product includes software developed by Digital Creations
+#      for use in the Z Object Publishing Environment
+#      (http://www.zope.org/)."
+# 
+#    Intact (re-)distributions of any official Zope release do not
+#    require an external acknowledgement.
+# 
+# 7. Modifications are encouraged but must be packaged separately as
+#    patches to official Zope releases.  Distributions that do not
+#    clearly separate the patches from the original work must be clearly
+#    labeled as unofficial distributions.  Modifications which do not
+#    carry the name Zope may be packaged in any form, as long as they
+#    conform to all of the clauses above.
+# 
+# 
+# Disclaimer
+# 
+#   THIS SOFTWARE IS PROVIDED BY DIGITAL CREATIONS ``AS IS'' AND ANY
+#   EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+#   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+#   PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL DIGITAL CREATIONS OR ITS
+#   CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+#   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+#   USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+#   ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+#   OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+#   OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+#   SUCH DAMAGE.
+# 
+# 
+# This software consists of contributions made by Digital Creations and
+# many individuals on behalf of Digital Creations.  Specific
+# attributions are listed in the accompanying credits file.
+# 
+##############################################################################
+from cStringIO import StringIO
+from cPickle import Unpickler, Pickler
+import sys
+
+#import traceback
+
+bad_classes={}
+bad_class=bad_classes.has_key
+
+ResolvedSerial='rs'
+
+def _classFactory(location, name,
+                  _silly=('__doc__',), _globals={}):
+    return getattr(__import__(location, _globals, _globals, _silly),
+                   name)
+
+def state(self, oid, serial, prfactory):
+    p=self.loadSerial(oid, serial)
+    file=StringIO(p)
+    unpickler=Unpickler(file)
+    unpickler.persistent_load=prfactory
+    class_tuple=unpickler.load()
+    state=unpickler.load()
+    return state
+
+
+class PersistentReference:
+
+    def __repr__(self):
+        return "PR(%s %s)" % (id(self), self.data)
+
+    def __getstate__(self):
+        raise "Can't pickle PersistentReference"
+
+class PersistentReferenceFactory:
+
+    data=None
+    
+    def __call__(self, oid,
+                 getattr=getattr, None=None):
+
+        data=self.data
+        if not data: data=self.data={}
+
+        r=data.get(oid, None)
+        if r is None:
+            r=PersistentReference()
+            r.data=oid
+            data[oid]=r
+
+        return r
+
+def persistent_id(object,
+                  PersistentReference=PersistentReference,
+                  getattr=getattr
+                  ):
+    if getattr(object, '__class__', 0) is not PersistentReference:
+        return None
+    return object.data
+
+class ConflictResolvingStorage:
+    "Mix-in class that provides conflict resolution handling for storages"
+
+    def tryToResolveConflict(self, oid, committedSerial, oldSerial, newpickle):
+        #class_tuple, old, committed, newstate = ('',''), 0, 0, 0
+        try:
+            file=StringIO(newpickle)
+            unpickler=Unpickler(file)
+            prfactory=PersistentReferenceFactory()
+            unpickler.persistent_load=prfactory
+            class_tuple=unpickler.load()[0]
+            if bad_class(class_tuple):
+                #sys.stderr.write(' b%s ' % class_tuple[1]); sys.stderr.flush()
+                return 0
+            
+            newstate=unpickler.load()
+            klass=_classFactory(class_tuple[0], class_tuple[1])
+            klass._p_resolveConflict                    
+            inst=klass.__basicnew__()
+            
+            try:
+                resolve=inst._p_resolveConflict
+            except AttributeError:
+                bad_classes[class_tuple]=1
+                #traceback.print_exc()
+                #sys.stderr.write(' b%s ' % class_tuple[1]); sys.stderr.flush()
+                return 0
+            
+            old=state(self, oid, oldSerial, prfactory)
+            committed=state(self, oid, committedSerial, prfactory)
+
+            resolved=resolve(old, committed, newstate)
+                
+            file=StringIO()
+            pickler=Pickler(file,1)
+            pickler.persistent_id=persistent_id
+            pickler.dump(class_tuple)
+            pickler.dump(resolved)
+            #sys.stderr.write(' r%s ' % class_tuple[1]); sys.stderr.flush()
+            return file.getvalue(1)
+        except Exception, v:
+            #print '='*70
+            #print v, v.args
+            #print '='*70
+            #print old
+            #print '='*70
+            #print committed
+            #print '='*70
+            #print newstate
+            #print '='*70
+            
+            #traceback.print_exc()
+
+            #sys.stderr.write(' c%s ' % class_tuple[1]); sys.stderr.flush()
+
+            return 0
+

trunk/src/ZODB/Connection.py

@@ -84,8 +84,8 @@
 ##############################################################################
 """Database connection support
 
-$Id: Connection.py,v 1.45 2001/02/09 14:11:49 jim Exp $"""
-__version__='$Revision: 1.45 $'[11:-2]
+$Id: Connection.py,v 1.46 2001/03/15 13:16:26 jim Exp $"""
+__version__='$Revision: 1.46 $'[11:-2]
 
 from cPickleCache import PickleCache
 from POSException import ConflictError, ExportError
@@ -94,8 +94,9 @@ from cPickle import Unpickler, Pickler
 from ExtensionClass import Base
 from time import time
 import Transaction, string, ExportImport, sys, traceback, TmpStore
-from zLOG import LOG, ERROR
+from zLOG import LOG, ERROR, BLATHER
 from coptimizations import new_persistent_id
+from ConflictResolution import ResolvedSerial
 
 ExtensionKlass=Base.__class__
 
@@ -230,7 +231,10 @@ class Connection(ExportImport.ExportImport):
 
         This just deactivates the thing.
         """
-        self._cache.invalidate(object._p_oid)
+        if object is self:
+            self._cache.invalidate(self._invalidated)
+        else:
+            self._cache.invalidate(object._p_oid)
 
     def cacheFullSweep(self, dt=0): self._cache.full_sweep(dt)
     def cacheMinimize(self, dt=0): self._cache.minimize(dt)
@@ -257,6 +261,8 @@ class Connection(ExportImport.ExportImport):
         db._closeConnection(self)
                         
     def commit(self, object, transaction, _type=type, _st=type('')):
+        if object is self:
+            return # we registered ourself  
         oid=object._p_oid
         invalid=self._invalid
         if oid is None or object._p_jar is not self:
@@ -267,7 +273,12 @@ class Connection(ExportImport.ExportImport):
             self._creating.append(oid)
 
         elif object._p_changed:
-            if invalid(oid) or invalid(None): raise ConflictError, `oid`
+            if (
+                (invalid(oid) and not hasattr(object, '_p_resolveConflict'))
+                or
+                invalid(None)
+                ):
+                raise ConflictError, `oid`
             self._invalidating.append(oid)
 
         else:
@@ -328,7 +339,13 @@ class Connection(ExportImport.ExportImport):
                 self._creating.append(oid)
             else:
                 #XXX We should never get here
-                if invalid(oid) or invalid(None): raise ConflictError, `oid`
+                if (
+                    (invalid(oid) and
+                     not hasattr(object, '_p_resolveConflict'))
+                    or
+                    invalid(None)
+                    ):
+                    raise ConflictError, `oid`
                 self._invalidating.append(oid)
                 
             klass = object.__class__
@@ -362,8 +379,12 @@ class Connection(ExportImport.ExportImport):
                 # Note that if s is false, then the storage defered the return
                 if _type(s) is _st:
                     # normal case
-                    object._p_serial=s
-                    object._p_changed=0
+                    if s is ResolvedSerial:
+                        # resolved conflict
+                        object._p_changed=None
+                    else:
+                        object._p_serial=s
+                        object._p_changed=0
                 else:
                     # defered returns
                     for oi, s in s:
@@ -389,6 +410,10 @@ class Connection(ExportImport.ExportImport):
         tmp=self._tmp
         if tmp is _None: return
         src=self._storage
+
+        LOG('ZODB', BLATHER,
+            'Commiting subtransaction of size %s' % src.getSize())
+        
         self._storage=tmp
         self._tmp=_None
 
@@ -487,7 +512,13 @@ class Connection(ExportImport.ExportImport):
             # notifications between the time we check and the time we
             # read.
             invalid=self._invalid
-            if invalid(oid) or invalid(None): raise ConflictError, `oid`
+            if invalid(oid) or invalid(None):
+                if not hasattr(object.__class__, '_p_independent'):
+                    get_transaction().register(self)
+                    raise ConflictError(`oid`, `object.__class__`)
+                invalid=1
+            else:
+                invalid=0
 
             file=StringIO(p)
             unpickler=Unpickler(file)
@@ -503,6 +534,14 @@ class Connection(ExportImport.ExportImport):
 
             object._p_serial=serial
 
+            if invalid:
+                if object._p_independent():
+                    try: del self._invalidated[oid]
+                    except KeyError: pass
+                else:
+                    get_transaction().register(self)
+                    raise ConflictError(`oid`, `object.__class__`)
+
         except:
             t, v =sys.exc_info()[:2]
             LOG('ZODB',ERROR, "Couldn't load state for %s" % `oid`,

trunk/src/ZODB/FileStorage.py

@@ -184,7 +184,7 @@
 #   may have a back pointer to a version record or to a non-version
 #   record.
 #
-__version__='$Revision: 1.50 $'[11:-2]
+__version__='$Revision: 1.51 $'[11:-2]
 
 import struct, time, os, bpthread, string, base64, sys
 from struct import pack, unpack
@@ -197,6 +197,7 @@ from zLOG import LOG, WARNING, ERROR, PANIC, register_subsystem
 register_subsystem('ZODB FS')
 import BaseStorage
 from cPickle import Pickler, Unpickler
+import ConflictResolution
 
 try: from posix import fsync
 except: fsync=None
@@ -240,7 +241,8 @@ class FileStorageQuotaError(FileStorageError,
 
 packed_version='FS21'
 
-class FileStorage(BaseStorage.BaseStorage):
+class FileStorage(BaseStorage.BaseStorage,
+                  ConflictResolution.ConflictResolvingStorage):
     _packt=z64
 
     def __init__(self, file_name, create=0, read_only=0, stop=None,
@@ -663,18 +665,23 @@ class FileStorage(BaseStorage.BaseStorage):
                         raise POSException.VersionLockError, (
                             `oid`, locked_version)
 
-                if serial != oserial: raise POSException.ConflictError, (
-                    serial, oserial)
-
+                if serial != oserial:
+                    data=self.tryToResolveConflict(oid, oserial, serial, data)
+                    if not data:
+                        raise POSException.ConflictError, (
+                            serial, oserial)
+            else:
+                oserial=serial
+                    
             tfile=self._tfile
             write=tfile.write
             pos=self._pos
             here=pos+(tfile.tell()+self._thl)
             self._tappend((oid, here))
-            serial=self._serial
+            newserial=self._serial
             write(pack(">8s8s8s8sH8s",
-                       oid,serial,p64(old),p64(pos),
-                       len(version),p64(len(data))
+                       oid, newserial, p64(old), p64(pos),
+                       len(version), p64(len(data))
                        )
                   )
             if version:
@@ -695,7 +702,8 @@ class FileStorage(BaseStorage.BaseStorage):
                 raise FileStorageQuotaError, (
                     'The storage quota has been exceeded.')
 
-            return serial
+            return (serial == oserial and newserial
+                    or ConflictResolution.ResolvedSerial)
         
         finally:
             self._lock_release()