lib.profiling: Remove.

Use a non-intrusive code profiling tool with NEO instead, like pprofile.

neo/client/app.py

@@ -15,7 +15,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 from cPickle import dumps, loads
-from zlib import compress as real_compress, decompress
+from zlib import compress, decompress
 from neo.lib.locking import Empty
 from random import shuffle
 import heapq
@@ -31,7 +31,7 @@ from neo.lib import logging
 from neo.lib.protocol import NodeTypes, Packets, \
     INVALID_PARTITION, ZERO_HASH, ZERO_TID
 from neo.lib.event import EventManager
-from neo.lib.util import makeChecksum as real_makeChecksum, dump
+from neo.lib.util import makeChecksum, dump
 from neo.lib.locking import Lock
 from neo.lib.connection import MTClientConnection, ConnectionClosed
 from neo.lib.node import NodeManager
@@ -45,25 +45,9 @@ from .iterator import Iterator
 from .cache import ClientCache
 from .pool import ConnectionPool
 from neo.lib.util import p64, u64, parseMasterList
-from neo.lib.profiling import profiler_decorator, PROFILING_ENABLED
 from neo.lib.debug import register as registerLiveDebugger
 from .container import ThreadContainer, TransactionContainer
 
-if PROFILING_ENABLED:
-    # Those functions require a "real" python function wrapper before they can
-    # be decorated.
-    @profiler_decorator
-    def compress(data):
-        return real_compress(data)
-
-    @profiler_decorator
-    def makeChecksum(data):
-        return real_makeChecksum(data)
-else:
-    # If profiling is disabled, directly use original functions.
-    compress = real_compress
-    makeChecksum = real_makeChecksum
-
 CHECKED_SERIAL = master.CHECKED_SERIAL
 
 
@@ -150,7 +134,6 @@ class Application(object):
         if self.pt is not None:
             self.pt.log()
 
-    @profiler_decorator
     def _handlePacket(self, conn, packet, kw={}, handler=None):
         """
           conn
@@ -180,7 +163,6 @@ class Application(object):
         finally:
             conn.unlock()
 
-    @profiler_decorator
     def _waitAnyMessage(self, queue, block=True):
         """
           Handle all pending packets.
@@ -218,7 +200,6 @@ class Application(object):
             # Don't leave access to thread context, even if a raise happens.
             self.setHandlerData(None)
 
-    @profiler_decorator
     def _ask(self, conn, packet, handler=None, **kw):
         self.setHandlerData(None)
         queue = self._getThreadQueue()
@@ -242,18 +223,15 @@ class Application(object):
                 _handlePacket(qconn, qpacket, kw)
         return self.getHandlerData()
 
-    @profiler_decorator
     def _askStorage(self, conn, packet, **kw):
         """ Send a request to a storage node and process its answer """
         return self._ask(conn, packet, handler=self.storage_handler, **kw)
 
-    @profiler_decorator
     def _askPrimary(self, packet, **kw):
         """ Send a request to the primary master and process its answer """
         return self._ask(self._getMasterConnection(), packet,
             handler=self.primary_handler, **kw)
 
-    @profiler_decorator
     def _getMasterConnection(self):
         """ Connect to the primary master node on demand """
         # acquire the lock to allow only one thread to connect to the primary
@@ -274,7 +252,6 @@ class Application(object):
         self._getMasterConnection()
         return self.pt
 
-    @profiler_decorator
     def _connectToPrimaryNode(self):
         """
             Lookup for the current primary master node
@@ -351,7 +328,6 @@ class Application(object):
     def getDB(self):
         return self._db
 
-    @profiler_decorator
     def new_oid(self):
         """Get a new OID."""
         self._oid_lock_acquire()
@@ -373,7 +349,6 @@ class Application(object):
         # return the last OID used, this is inaccurate
         return int(u64(self.last_oid))
 
-    @profiler_decorator
     def load(self, oid, tid=None, before_tid=None):
         """
         Internal method which manage load, loadSerial and loadBefore.
@@ -445,7 +420,6 @@ class Application(object):
         finally:
             self._load_lock_release()
 
-    @profiler_decorator
     def _loadFromStorage(self, oid, at_tid, before_tid):
         packet = Packets.AskObject(oid, at_tid, before_tid)
         for node, conn in self.cp.iterateForObject(oid, readable=True):
@@ -468,7 +442,6 @@ class Application(object):
         # connection error
         raise NEOStorageError('connection failure')
 
-    @profiler_decorator
     def _loadFromCache(self, oid, at_tid=None, before_tid=None):
         """
         Load from local cache, return None if not found.
@@ -479,7 +452,6 @@ class Application(object):
             return result
         return self._cache.load(oid, before_tid)
 
-    @profiler_decorator
     def tpc_begin(self, transaction, tid=None, status=' '):
         """Begin a new transaction."""
         txn_container = self._txn_container
@@ -496,7 +468,6 @@ class Application(object):
         txn_context['txn'] = transaction
         txn_context['ttid'] = answer_ttid
 
-    @profiler_decorator
     def store(self, oid, serial, data, version, transaction):
         """Store object."""
         txn_context = self._txn_container.get(transaction)
@@ -570,7 +541,6 @@ class Application(object):
         conn.ask(Packets.AskHasLock(txn_context['ttid'], oid),
                  timeout=5, queue=txn_context['queue'])
 
-    @profiler_decorator
     def _handleConflicts(self, txn_context, tryToResolveConflict):
         result = []
         append = result.append
@@ -664,7 +634,6 @@ class Application(object):
                 serial), data=data)
         return result
 
-    @profiler_decorator
     def waitResponses(self, queue):
         """Wait for all requests to be answered (or their connection to be
         detected as closed)"""
@@ -673,7 +642,6 @@ class Application(object):
         while pending(queue):
             _waitAnyMessage(queue)
 
-    @profiler_decorator
     def waitStoreResponses(self, txn_context, tryToResolveConflict):
         result = []
         append = result.append
@@ -704,7 +672,6 @@ class Application(object):
                 append((oid, ResolvedSerial))
         return result
 
-    @profiler_decorator
     def tpc_vote(self, transaction, tryToResolveConflict):
         """Store current transaction."""
         txn_context = self._txn_container.get(transaction)
@@ -744,7 +711,6 @@ class Application(object):
         txn_context['txn_voted'] = True
         return result
 
-    @profiler_decorator
     def tpc_abort(self, transaction):
         """Abort current transaction."""
         txn_container = self._txn_container
@@ -773,7 +739,6 @@ class Application(object):
         self.dispatcher.forget_queue(queue, flush_queue=False)
         txn_container.delete(transaction)
 
-    @profiler_decorator
     def tpc_finish(self, transaction, tryToResolveConflict, f=None):
         """Finish current transaction."""
         txn_container = self._txn_container
@@ -990,7 +955,6 @@ class Application(object):
                 self._insertMetadata(txn_info, txn_ext)
             return result
 
-    @profiler_decorator
     def importFrom(self, source, start, stop, tryToResolveConflict,
             preindex=None):
         if preindex is None:

neo/client/pool.py

@@ -21,7 +21,6 @@ from neo.lib import logging
 from neo.lib.locking import RLock
 from neo.lib.protocol import NodeTypes, Packets
 from neo.lib.connection import MTClientConnection, ConnectionClosed
-from neo.lib.profiling import profiler_decorator
 from neo.lib.exception import NodeNotReady
 from .exception import NEOStorageError
 
@@ -52,7 +51,6 @@ class ConnectionPool(object):
         self.connection_lock_release = l.release
         self.node_failure_dict = {}
 
-    @profiler_decorator
     def _initNodeConnection(self, node):
         """Init a connection to a given storage node."""
         app = self.app
@@ -75,7 +73,6 @@ class ConnectionPool(object):
             logging.info('Connected %r', node)
         return conn
 
-    @profiler_decorator
     def _dropConnections(self):
         """Drop connections."""
         for conn in self.connection_dict.values():
@@ -93,11 +90,9 @@ class ConnectionPool(object):
             finally:
                 conn.unlock()
 
-    @profiler_decorator
     def notifyFailure(self, node):
         self.node_failure_dict[node.getUUID()] = time.time() + MAX_FAILURE_AGE
 
-    @profiler_decorator
     def getCellSortKey(self, cell):
         uuid = cell.getUUID()
         if uuid in self.connection_dict:
@@ -107,7 +102,6 @@ class ConnectionPool(object):
             return CELL_GOOD
         return CELL_FAILED
 
-    @profiler_decorator
     def getConnForCell(self, cell):
         return self.getConnForNode(cell.getNode())
 
@@ -142,7 +136,6 @@ class ConnectionPool(object):
                 # wait a bit to avoid a busy loop
                 time.sleep(1)
 
-    @profiler_decorator
     def getConnForNode(self, node):
         """Return a locked connection object to a given node
         If no connection exists, create a new one"""
@@ -170,7 +163,6 @@ class ConnectionPool(object):
                 finally:
                     self.connection_lock_release()
 
-    @profiler_decorator
     def removeConnection(self, node):
         """Explicitly remove connection when a node is broken."""
         self.connection_dict.pop(node.getUUID(), None)

neo/lib/connection.py

@@ -22,7 +22,6 @@ from .connector import ConnectorException, ConnectorTryAgainException, \
         ConnectorInProgressException, ConnectorConnectionRefusedException, \
         ConnectorConnectionClosedException
 from .locking import RLock
-from .profiling import profiler_decorator
 from .protocol import uuid_str, Errors, \
         PacketMalformedError, Packets, ParserState
 from .util import ReadBuffer
@@ -105,7 +104,6 @@ class HandlerSwitcher(object):
         """ Return the last (may be unapplied) handler registered """
         return self._pending[-1][1]
 
-    @profiler_decorator
     def emit(self, request, timeout, on_timeout, kw={}):
         # register the request in the current handler
         _pending = self._pending
@@ -150,7 +148,6 @@ class HandlerSwitcher(object):
         finally:
             self._is_handling = False
 
-    @profiler_decorator
     def _handle(self, connection, packet):
         assert len(self._pending) == 1 or self._pending[0][0]
         logging.packet(connection, packet, False)
@@ -197,7 +194,6 @@ class HandlerSwitcher(object):
         self._next_timeout, self._next_timeout_msg_id, self._next_on_timeout = \
             next_timeout or (None, None, None)
 
-    @profiler_decorator
     def setHandler(self, handler):
         can_apply = len(self._pending) == 1 and not self._pending[0][0]
         if can_apply:
@@ -437,7 +433,6 @@ class Connection(BaseConnection):
     def getPeerId(self):
         return self.peer_id
 
-    @profiler_decorator
     def _getNextId(self):
         next_id = self.cur_id
         self.cur_id = (next_id + 1) & 0xffffffff
@@ -553,7 +548,6 @@ class Connection(BaseConnection):
             self._handlers.handle(self, self._queue.pop(0))
         self.close()
 
-    @profiler_decorator
     def _recv(self):
         """Receive data from a connector."""
         try:
@@ -581,7 +575,6 @@ class Connection(BaseConnection):
             self._base_timeout = time() # last known remote activity
             self.read_buf.append(data)
 
-    @profiler_decorator
     def _send(self):
         """Send data to a connector."""
         if not self.write_buf:
@@ -610,7 +603,6 @@ class Connection(BaseConnection):
             else:
                 self.write_buf = [msg[n:]]
 
-    @profiler_decorator
     def _addPacket(self, packet):
         """Add a packet into the write buffer."""
         if self.connector is None:
@@ -633,7 +625,6 @@ class Connection(BaseConnection):
         self._addPacket(packet)
         return msg_id
 
-    @profiler_decorator
     @not_closed
     def ask(self, packet, timeout=CRITICAL_TIMEOUT, on_timeout=None, **kw):
         """
@@ -743,7 +734,6 @@ class MTClientConnection(ClientConnection):
         finally:
             self.unlock()
 
-    @profiler_decorator
     def ask(self, packet, timeout=CRITICAL_TIMEOUT, on_timeout=None,
             queue=None, **kw):
         self.lock()

neo/lib/dispatcher.py

@@ -16,7 +16,6 @@
 
 from functools import wraps
 from .locking import Lock, Empty
-from .profiling import profiler_decorator
 EMPTY = {}
 NOBODY = []
 
@@ -53,7 +52,6 @@ class Dispatcher:
         self.poll_thread = poll_thread
 
     @giant_lock
-    @profiler_decorator
     def dispatch(self, conn, msg_id, packet, kw):
         """
         Retrieve register-time provided queue, and put conn and packet in it.
@@ -87,7 +85,6 @@ class Dispatcher:
         self.poll_thread.start()
 
     @giant_lock
-    @profiler_decorator
     def register(self, conn, msg_id, queue):
         """Register an expectation for a reply."""
         if self.poll_thread is not None:
@@ -95,7 +92,6 @@ class Dispatcher:
         self.message_table.setdefault(id(conn), {})[msg_id] = queue
         self._increfQueue(queue)
 
-    @profiler_decorator
     def unregister(self, conn):
         """ Unregister a connection and put fake packet in queues to unlock
         threads excepting responses from that connection """
@@ -116,7 +112,6 @@ class Dispatcher:
             _decrefQueue(queue)
 
     @giant_lock
-    @profiler_decorator
     def forget(self, conn, msg_id):
         """ Forget about a specific message for a specific connection.
         Actually makes it "expected by nobody", so we know we can ignore it,
@@ -132,7 +127,6 @@ class Dispatcher:
         return queue
 
     @giant_lock
-    @profiler_decorator
     def forget_queue(self, queue, flush_queue=True):
         """
         Forget all pending messages for given queue.
@@ -160,13 +154,11 @@ class Dispatcher:
                 except Empty:
                     break
 
-    @profiler_decorator
     def registered(self, conn):
         """Check if a connection is registered into message table."""
         return len(self.message_table.get(id(conn), EMPTY)) != 0
 
     @giant_lock
-    @profiler_decorator
     def pending(self, queue):
         return not queue.empty() or self.queue_dict.get(id(queue), 0) > 0
 

neo/lib/event.py

@@ -18,7 +18,6 @@ from time import time
 from select import epoll, EPOLLIN, EPOLLOUT, EPOLLERR, EPOLLHUP
 from errno import EINTR, EAGAIN
 from . import logging
-from .profiling import profiler_decorator
 
 class EpollEventManager(object):
     """This class manages connections and events based on epoll(5)."""
@@ -187,7 +186,6 @@ class EpollEventManager(object):
             self.reader_set.remove(fd)
             self.epoll.modify(fd, fd in self.writer_set and EPOLLOUT)
 
-    @profiler_decorator
     def addWriter(self, conn):
         connector = conn.getConnector()
         assert connector is not None, conn.whoSetConnector()

neo/lib/profiling.py

-#
-# Copyright (C) 2010  Nexedi SA
-#
-# This program is free software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 2
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-"""
-Profiling is done with tiny-profiler, a very simple profiler.
-
-It is different from python's built-in profilers in that it requires
-developpers to explicitely put probes on specific methods, reducing:
-- profiling overhead
-- undesired result entries
-
-You can get this profiler at:
-  https://svn.erp5.org/repos/public/erp5/trunk/utils/tiny_profiler
-"""
-
-PROFILING_ENABLED = False
-
-if PROFILING_ENABLED:
-    from tiny_profiler import profiler_decorator, profiler_report
-else:
-    def profiler_decorator(func):
-        return func
-
-    def profiler_report():
-        pass

tools/perfs

@@ -11,8 +11,6 @@ from ZODB.FileStorage import FileStorage
 from neo.tests import DB_PREFIX
 from neo.tests.benchmark import BenchmarkRunner
 from neo.tests.functional import NEOCluster
-from neo.lib.profiling import PROFILING_ENABLED, profiler_decorator, \
-    profiler_report
 
 class ImportBenchmark(BenchmarkRunner):
     """ Test import of a datafs """
@@ -62,7 +60,6 @@ class ImportBenchmark(BenchmarkRunner):
     def runImport(self, neo):
 
         def counter(wrapped, d):
-            @profiler_decorator
             def wrapper(*args, **kw):
                 # count number of tick per second
                 t = int(time())
@@ -134,8 +131,6 @@ class ImportBenchmark(BenchmarkRunner):
 
 def main(args=None):
     ImportBenchmark().run()
-    if PROFILING_ENABLED:
-        print profiler_report()
 
 if __name__ == "__main__":
     main()