New populate() method for threaded tests

Storage.importFrom is changed for resumable imports.

neo/client/Storage.py

@@ -232,10 +232,10 @@ class Storage(BaseStorage.BaseStorage,
         return self.app.importFrom(source, None, None,
                 self.tryToResolveConflict)
 
-    def importFrom(self, source, start=None, stop=None):
+    def importFrom(self, source, start=None, stop=None, preindex=None):
         """ Allow import only a part of the source storage """
         return self.app.importFrom(source, start, stop,
-                self.tryToResolveConflict)
+                self.tryToResolveConflict, preindex)
 
     def restore(self, oid, serial, data, version, prev_txn, transaction):
         raise NotImplementedError

neo/client/app.py

@@ -1000,18 +1000,20 @@ class Application(object):
             return result
 
     @profiler_decorator
-    def importFrom(self, source, start, stop, tryToResolveConflict):
-        serials = {}
+    def importFrom(self, source, start, stop, tryToResolveConflict,
+            preindex=None):
+        if preindex is None:
+            preindex = {}
         transaction_iter = source.iterator(start, stop)
         for transaction in transaction_iter:
             tid = transaction.tid
             self.tpc_begin(transaction, tid, transaction.status)
             for r in transaction:
                 oid = r.oid
-                pre = serials.get(oid, None)
+                pre = preindex.get(oid)
                 # TODO: bypass conflict resolution, locks...
                 self.store(oid, pre, r.data, r.version, transaction)
-                serials[oid] = tid
+                preindex[oid] = tid
             conflicted = self.tpc_vote(transaction, tryToResolveConflict)
             assert not conflicted, conflicted
             real_tid = self.tpc_finish(transaction, tryToResolveConflict)

neo/tests/stat_zodb.py

 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
-import math, os, random, sys
+import math, os, random, sys, time
 from cStringIO import StringIO
-from ZODB.utils import p64
+from persistent.TimeStamp import TimeStamp
+from ZODB.utils import p64, newTid
 from ZODB.BaseStorage import TransactionRecord
 from ZODB.FileStorage import FileStorage
 
@@ -96,8 +97,10 @@ class DummyZODB(object):
             size = 0
             def iterator(storage, *args):
                 args = ' ', '', '', {}
+                tid = None
                 for i in xrange(1, transaction_count+1):
-                    t =  dummy_transaction(p64(i), *args)
+                    tid = newTid(tid)
+                    t =  dummy_transaction(tid, *args)
                     storage.size += t.size
                     yield t
             def getSize(self):

neo/tests/threaded/__init__.py

@@ -655,6 +655,15 @@ class NEOCluster(object):
             self.client.setPoll(True)
         return Storage.Storage(None, self.name, _app=self.client, **kw)
 
+    def populate(self, dummy_zodb=None):
+        if dummy_zodb is None:
+            from ..stat_zodb import PROD1
+            dummy_zodb = PROD1()
+        importFrom = self.getZODBStorage().importFrom
+        preindex = {}
+        as_storage = dummy_zodb.as_storage
+        return lambda count: importFrom(as_storage(count), preindex=preindex)
+
     def getTransaction(self):
         txn = transaction.TransactionManager()
         return txn, self.db.open(transaction_manager=txn)