Implement an operation handler for storage node. This does not support replication yet.

git-svn-id: https://svn.erp5.org/repos/neo/branches/prototype3@67 71dcc9de-d417-0410-9af5-da40c76e7ee4

README

@@ -16,3 +16,5 @@ TODO
 
     - Stopping packet processing by returning a boolean value from
       a handler, otherwise too tricky to exchange a handler with another
+
+    - Replication.

neo/storage/app.py

@@ -3,6 +3,7 @@ import MySQLdb
 import os
 from time import time
 from struct import pack, unpack
+from collections import deque
 
 from neo.config import ConfigurationManager
 from neo.protocol import Packet, ProtocolError, \
@@ -240,8 +241,21 @@ class Application(object):
         # for locking objects against load operations.
         self.load_lock_dict = {}
 
+        # This is a queue of events used to delay operations due to locks.
+        self.event_queue = deque()
+
         while 1:
             em.poll(1)
 
+    def queueEvent(self, callable, *args, **kwargs):
+        self.event_queue.append((callable, args, kwargs))
+
+    def executeQueuedEvents(self):
+        l = len(self.event_queue)
+        p = self.event_queue.popleft
+        for i in xrange(l):
+            callable, args, kwargs = p()
+            callable(*args, **kwargs)
+
     def getPartition(self, oid_or_tid):
         return unpack('!Q', oid_or_tid)[0] % self.num_partitions

neo/storage/database.py

@@ -74,13 +74,14 @@ class DatabaseManager(object):
         raise NotImplementedError('this method must be overridden')
 
     def getObject(self, oid, tid = None, before_tid = None):
-        """Return a tuple of a serial, a compression
+        """Return a tuple of a serial, next serial, a compression
         specification, a checksum, and object data, if a given object
         ID is present. Otherwise, return None. If tid is None and
         before_tid is None, the latest revision is taken. If tid is
         specified, the given revision is taken. If tid is not specified,
         but before_tid is specified, the latest revision before the
-        given revision is taken."""
+        given revision is taken. The next serial is a serial right after
+        before_tid, if specified. Otherwise, it is None."""
         raise NotImplementedError('this method must be overridden')
 
     def changePartitionTable(self, ptid, cell_list):
@@ -125,8 +126,14 @@ class DatabaseManager(object):
         raise NotImplementedError('this method must be overridden')
 
     def getObjectHistory(self, oid, length = 1):
-        """Return a list of serials for a given object ID. The length
-        specifies the maximum size of such a list. The first serial
+        """Return a list of serials and sizes for a given object ID.
+        The length specifies the maximum size of such a list. The first serial
         must be the last serial, and the list must be sorted in descending
         order. If there is no such object ID in a database, return None."""
         raise NotImplementedError('this method must be overridden')
+
+    def getTIDList(self, offset, length, num_partitions, partition_list):
+        """Return a list of TIDs in descending order from an offset,
+        at most the specified length. The list of partitions are passed
+        to filter out non-applicable TIDs."""
+        raise NotImplementedError('this method must be overridden')

neo/storage/handler.py

@@ -181,9 +181,20 @@ class StorageEventHandler(EventHandler):
     def handleUnlockInformation(self, conn, packet, tid):
         raise NotImplementedError('this method must be overridden')
 
+    def handleAskObject(self, conn, packet, oid, serial, tid):
+        self.handleUnexpectedPacket(conn, packet)
+
+    def handleAskTIDs(self, conn, packet, first, last):
+        self.handleUnexpectedPacket(conn, packet)
+
+    def handleAskObjectHistory(self, conn, packet, oid, length):
+        self.handleUnexpectedPacket(conn, packet)
+
+    def handleAskStoreTransaction(self, conn, packet, tid, user, desc,
+                                  ext, oid_list):
+        self.handleUnexpectedPacket(conn, packet)
+
     def handleAskStoreObject(self, conn, packet, msg_id, oid, serial,
                              compression, data, checksum, tid):
         self.handleUnexpectedPacket(conn, packet)
 
-    def handleAskObject(self, conn, packet, oid, serial, tid):
-        self.handleUnexpectedPacket(conn, packet)

neo/storage/mysqldb.py

@@ -267,12 +267,29 @@ class MySQLDatabaseManager(DatabaseManager):
             r = q("""SELECT serial, compression, checksum, data FROM obj
                         WHERE oid = '%s' AND serial = '%s'""" \
                     % (oid, tid))
+            try:
+                serial, compression, checksum, data = r[0]
+                next_serial = None
+            except IndexError:
+                return None
         elif before_tid is not None:
             before_tid = e(before_tid)
             r = q("""SELECT serial, compression, checksum, data FROM obj
                         WHERE oid = '%s' AND serial < '%s'
                         ORDER BY serial DESC LIMIT 1""" \
                     % (oid, before_tid))
+            try:
+                serial, compression, checksum, data = r[0]
+                r = q("""SELECT serial FROM obj
+                            WHERE oid = '%s' AND serial > '%s'
+                            ORDER BY serial LIMIT 1""" \
+                        % (oid, before_tid))
+                try:
+                    next_serial = r[0][0]
+                except IndexError:
+                    next_serial = None
+            except IndexError:
+                return None
         else:
             # XXX I want to express "HAVING serial = MAX(serial)", but
             # MySQL does not use an index for a HAVING clause!
@@ -280,10 +297,13 @@ class MySQLDatabaseManager(DatabaseManager):
                         WHERE oid = '%s' ORDER BY serial DESC LIMIT 1""" \
                     % oid)
             try:
-            return r[0]
+                serial, compression, checksum, data = r[0]
+                next_serial = None
             except IndexError:
                 return None
 
+        return serial, next_serial, compression, checksum, data
+
     def doSetPartitionTable(self, ptid, cell_list, reset):
         q = self.query
         e = self.escape
@@ -403,9 +423,17 @@ class MySQLDatabaseManager(DatabaseManager):
         q = self.query
         e = self.escape
         oid = e(oid)
-        r = q("""SELECT serial FROM obj WHERE oid = '%s'
+        r = q("""SELECT serial, LENGTH(data) FROM obj WHERE oid = '%s'
                     ORDER BY serial DESC LIMIT %d""" \
                 % (oid, length))
         if r:
-            return [t[0] for t in r]
+            return r
         return None
+
+    def getTIDList(self, offset, length, num_partitions, partition_list):
+        q = self.query
+        e = self.escape
+        r = q("""SELECT tid FROM trans WHERE MOD(tid,%d) in (%s)
+                    ORDER BY tid DESC LIMIT %d""" \
+                % (offset, num_partitions, ','.join(partition_list), length))
+        return [t[0] for t in r]