Skip to content

Z
ZODB
Commit 8db9c2bc authored by Jim Fulton's avatar Jim Fulton
Browse files
Options

Improved/added docstrings to support reference documentation on zodb.org

parent 975f715b
Hide whitespace changes
Inline Side-by-side
Showing with 309 additions and 56 deletions
src/ZODB/Connection.py
View file @ 8db9c2bc
......@@ -78,9 +78,13 @@ def className(obj):
IPersistentDataManager,
ISynchronizer)
class Connection(ExportImport, object):
"""Connection to ZODB for loading and storing objects."""
"""Connection to ZODB for loading and storing objects.
Connections manage object state in collaboration with transaction
managers. They're created by calling the
:meth:`~ZODB.DB.open` method on :py:class:`database
<ZODB.DB>` objects.
"""
_code_timestamp = 0
......
src/ZODB/DB.py
View file @ 8db9c2bc
......@@ -39,7 +39,7 @@ import transaction
from persistent.TimeStamp import TimeStamp
import six
from . import POSException
from . import POSException, valuedoc
logger = logging.getLogger('ZODB.DB')
......@@ -323,11 +323,9 @@ def getTID(at, before):
before = TimeStamp(before).raw()
return before
@implementer(IDatabase)
class DB(object):
"""The Object Database
-------------------
The DB class coordinates the activities of multiple database
Connection instances. Most of the work is done by the
......@@ -340,36 +338,19 @@ class DB(object):
connections are opened, a warning is logged, and if more than twice
that many, a critical problem is logged.
The class variable 'klass' is used by open() to create database
connections. It is set to Connection, but a subclass could override
it to provide a different connection implementation.
The database provides a few methods intended for application code
-- open, close, undo, and pack -- and a large collection of
methods for inspecting the database and its connections' caches.
:Cvariables:
- `klass`: Class used by L{open} to create database connections
:Groups:
- `User Methods`: __init__, open, close, undo, pack, classFactory
- `Inspection Methods`: getName, getSize, objectCount,
getActivityMonitor, setActivityMonitor
- `Connection Pool Methods`: getPoolSize, getHistoricalPoolSize,
setPoolSize, setHistoricalPoolSize, getHistoricalTimeout,
setHistoricalTimeout
- `Transaction Methods`: invalidate
- `Other Methods`: lastTransaction, connectionDebugInfo
- `Cache Inspection Methods`: cacheDetail, cacheExtremeDetail,
cacheFullSweep, cacheLastGCTime, cacheMinimize, cacheSize,
cacheDetailSize, getCacheSize, getHistoricalCacheSize, setCacheSize,
setHistoricalCacheSize
"""
klass = Connection # Class to use for connections
_activity_monitor = next = previous = None
def __init__(self, storage,
#: Database storage, implementing :interface:`~ZODB.interfaces.IStorage`
storage = valuedoc.ValueDoc('storage object')
def __init__(self,
storage,
pool_size=7,
pool_timeout=1<<31,
cache_size=400,
......@@ -385,23 +366,53 @@ class DB(object):
**storage_args):
"""Create an object database.
:Parameters:
- `storage`: the storage used by the database, e.g. FileStorage
- `pool_size`: expected maximum number of open connections
- `cache_size`: target size of Connection object cache
- `cache_size_bytes`: target size measured in total estimated size
of objects in the Connection object cache.
"0" means unlimited.
- `historical_pool_size`: expected maximum number of total
:param storage: the storage used by the database, such as a
:class:`~ZODB.FileStorage.FileStorage.FileStorage`.
This can be a string path name to use a constructed
:class:`~ZODB.FileStorage.FileStorage.FileStorage`
storage or ``None`` to use a constructed
:class:`~ZODB.MappingStorage.MappingStorage`.
:param int pool_size: expected maximum number of open connections.
Warnings are logged when this is exceeded and critical
messages are logged if twice the pool size is exceeded.
:param seconds pool_timeout: Maximum age of inactive connections
When a connection has remained unused in a connection
pool for more than pool_timeout seconds, it will be
discarded and it's resources released.
:param objects cache_size`: target maximum number of non-ghost
objects in each connection object cache.
:param int cache_size_bytes`: target total memory usage of non-ghost
objects in each connection object cache.
:param int historical_pool_size`: expected maximum number of total
historical connections
- `historical_cache_size`: target size of Connection object cache for
historical (`at` or `before`) connections
- `historical_cache_size_bytes` -- similar to `cache_size_bytes` for
the historical connection.
- `historical_timeout`: minimum number of seconds that
an unused historical connection will be kept, or None.
- `xrefs` - Boolian flag indicating whether implicit cross-database
references are allowed
:param objects historical_cache_size`: target maximum number
of non-ghost objects in each historical connection object
cache.
:param int historical_cache_size_bytes`: target total memory
usage of non-ghost objects in each historical connection
object cache.
:param seconds historical_timeout: Maximum age of inactive
historical connections. When a connection has remained
unused in a historical connection pool for more than pool_timeout
seconds, it will be discarded and it's resources
released.
:param str database_name: The name of this database in a
multi-database configuration. The name is used when
constructing cross-database references ans when accessing
database connections fron other databases.
:param dict databases: dictionary of database name to
databases in a multi-database configuration. The new
database will add itself to this dictionary. The
dictionary is used when getting connections in other databases.
:param boolean xrefs: Flag indicating whether cross-database
references are allowed from this database to other
databases in a multi-database configuration.
:param int large_record_size: When object records are saved
that are larger than this, a warning is issued,
suggesting that blobs should be used instead.
:param storage_args: Extra keywork arguments passed to a
storage constructor if a path name or None is passed as
the storage argument.
"""
# Allocate lock.
......@@ -494,9 +505,7 @@ class DB(object):
self.historical_pool.map(f)
def cacheDetail(self):
"""Return information on objects in the various caches
Organized by class.
"""Return object counts by class accross all connections.
"""
detail = {}
......@@ -514,6 +523,12 @@ class DB(object):
return sorted(detail.items())
def cacheExtremeDetail(self):
"""Return information about all of the objects in the object caches.
Information includes a connection number, class, object id,
reference count and state. The reference count returned
excludes references help by ZODB itself.
"""
detail = []
conn_no = [0] # A mutable reference to a counter
# sys.getrefcount is a CPython implementation detail
......@@ -563,7 +578,7 @@ class DB(object):
self._connectionMap(f)
return detail
def cacheFullSweep(self):
def cacheFullSweep(self): # XXX this is the same as cacheMinimize
self._connectionMap(lambda c: c._cache.full_sweep())
def cacheLastGCTime(self):
......@@ -577,9 +592,13 @@ class DB(object):
return m[0]
def cacheMinimize(self):
"""Minimize cache sizes for all connections
"""
self._connectionMap(lambda c: c._cache.minimize())
def cacheSize(self):
"""Return the total count of non-ghost objects in all object caches
"""
m = [0]
def f(con, m=m):
m[0] += con._cache.cache_non_ghost_count
......@@ -588,6 +607,8 @@ class DB(object):
return m[0]
def cacheDetailSize(self):
"""Return non-ghost counts sizes for all connections.
"""
m = []
def f(con, m=m):
m.append({'connection': repr(con),
......@@ -625,38 +646,60 @@ class DB(object):
del self._mvcc_storage
def getCacheSize(self):
"""Get the configured cache size (objects).
"""
return self._cache_size
def getCacheSizeBytes(self):
"""Get the configured cache size in bytes.
"""
return self._cache_size_bytes
def lastTransaction(self):
"""Get the storage last transaction id.
"""
return self.storage.lastTransaction()
def getName(self):
"""Get the storage name
"""
return self.storage.getName()
def getPoolSize(self):
"""Get the configured pool size
"""
return self.pool.size
def getSize(self):
"""Get the approximate database size, in bytes
"""
return self.storage.getSize()
def getHistoricalCacheSize(self):
"""Get the configured historical cache size (objects).
"""
return self._historical_cache_size
def getHistoricalCacheSizeBytes(self):
"""Get the configured historical cache size in bytes.
"""
return self._historical_cache_size_bytes
def getHistoricalPoolSize(self):
"""Get the configured historical pool size
"""
return self.historical_pool.size
def getHistoricalTimeout(self):
"""Get the configured historical pool timeout
"""
return self.historical_pool.timeout
transform_record_data = untransform_record_data = lambda self, data: data
def objectCount(self):
"""Get the approximate object count
"""
return len(self.storage)
def open(self, transaction_manager=None, at=None, before=None):
......@@ -729,6 +772,15 @@ class DB(object):
return result
def connectionDebugInfo(self):
"""Get debugging information about connections
This is especially useful to debug connections that seem to be
leaking or open too long. Information includes connection
info, the connection before setting, and, if a connection is
open, the time it was opened. The info is the result of
calling :meth:`~ZODB.Connection.Connection.getDebugInfo` on
the connection, and the connection's cache size.
"""
result = []
t = time.time()
......@@ -790,6 +842,8 @@ class DB(object):
return find_global(modulename, globalname)
def setCacheSize(self, size):
"""Reconfigure the cache size (non-ghost object count)
"""
with self._lock:
self._cache_size = size
def setsize(c):
......@@ -797,6 +851,8 @@ class DB(object):
self.pool.map(setsize)
def setCacheSizeBytes(self, size):
"""Reconfigure the cache total size in bytes
"""
with self._lock:
self._cache_size_bytes = size
def setsize(c):
......@@ -804,6 +860,8 @@ class DB(object):
self.pool.map(setsize)
def setHistoricalCacheSize(self, size):
"""Reconfigure the historical cache size (non-ghost object count)
"""
with self._lock:
self._historical_cache_size = size
def setsize(c):
......@@ -811,6 +869,8 @@ class DB(object):
self.historical_pool.map(setsize)
def setHistoricalCacheSizeBytes(self, size):
"""Reconfigure the historical cache total size in bytes
"""
with self._lock:
self._historical_cache_size_bytes = size
def setsize(c):
......@@ -818,21 +878,33 @@ class DB(object):
self.historical_pool.map(setsize)
def setPoolSize(self, size):
"""Reconfigure the connection pool size
"""
with self._lock:
self.pool.size = size
def setHistoricalPoolSize(self, size):
"""Reconfigure the connection historical pool size
"""
with self._lock:
self.historical_pool.size = size
def setHistoricalTimeout(self, timeout):
"""Reconfigure the connection historical pool timeout
"""
with self._lock:
self.historical_pool.timeout = timeout
def history(self, *args, **kw):
return self.storage.history(*args, **kw)
def history(self, oid, size=1):
"""Get revision history information for an object.
See :meth:`ZODB.interfaces.IStorage.history`.
"""
return self.storage.history(oid, size)
def supportsUndo(self):
"""Return whether the database supports undo.
"""
try:
f = self.storage.supportsUndo
except AttributeError:
......@@ -840,11 +912,20 @@ class DB(object):
return f()
def undoLog(self, *args, **kw):
"""Return a sequence of descriptions for transactions.
See :meth:`ZODB.interfaces.IStorageUndoable.undoLog`.
"""
if not self.supportsUndo():
return ()
return self.storage.undoLog(*args, **kw)
def undoInfo(self, *args, **kw):
"""Return a sequence of descriptions for transactions.
See :meth:`ZODB.interfaces.IStorageUndoable.undoInfo`.
"""
if not self.supportsUndo():
return ()
return self.storage.undoInfo(*args, **kw)
......@@ -894,6 +975,14 @@ class DB(object):
self.undoMultiple([id], txn)
def transaction(self, note=None):
"""Execute a block of code as a transaction.
If a note is given, it will be added to the transaction's
description.
The ``transaction`` method returns a context manager that can
be used with the ``with`` statement.
"""
return ContextManager(self, note)
def new_oid(self):
......@@ -966,4 +1055,11 @@ class TransactionalUndo(object):
return "%s:%s" % (self._storage.sortKey(), id(self))
def connection(*args, **kw):
"""Create a database :class:`connection <ZODB.Connection.Connection>`.
A database is created using the given arguments and opened to
create the returned connection. The database will be closed when
the connection is closed. This is a convenience function to avoid
managing a separate database object.
"""
return DB(*args, **kw).open_then_close_db_when_connection_closes()
src/ZODB/DemoStorage.py
View file @ 8db9c2bc
......@@ -40,10 +40,52 @@ from .utils import load_current, maxtid
ZODB.interfaces.IStorageIteration,
)
class DemoStorage(ConflictResolvingStorage):
"""A storage that stores changes against a read-only base database
This storage was originally meant to support distribution of
application demonstrations with populated read-only databases (on
CDROM) and writable in-memory databases.
Demo storages are extemely convenient for testing where setup of a
base database can be shared by many tests.
Demo storages are also handy for staging appplications where a
read-only snapshot of a production database (often accomplished
using a `beforestorage
<https://pypi.python.org/pypi/zc.beforestorage>`_) is combined
with a changes database implemented with a
:class:`~ZODB.FileStorage.FileStorage.FileStorage`.
"""
def __init__(self, name=None, base=None, changes=None,
close_base_on_close=None, close_changes_on_close=None):
"""Create a demo storage
:param str name: The storage name used by the
:meth:`~ZODB.interfaces.IStorage.getName` and
:meth:`~ZODB.interfaces.IStorage.sortKey` methods.
:param object base: base storage
:param object changes: changes storage
:param bool close_base_on_close: A Flag indicating whether the base
database should be closed when the demo storage is closed.
:param bool close_changes_on_close: A Flag indicating whether the
changes database should be closed when the demo storage is closed.
If a base database isn't provided, a
:class:`~ZODB.MappingStorage.MappingStorage` will be
constructed and used.
If ``close_base_on_close`` isn't specified, it will be ``True`` if
a base database was provided and ``False`` otherwise.
If a changes database isn't provided, a
:class:`~ZODB.MappingStorage.MappingStorage` will be
constructed and used and blob support will be provided using a
temporary blob directory.
If ``close_changes_on_close`` isn't specified, it will be ``True`` if
a changes database was provided and ``False`` otherwise.
"""
if close_base_on_close is None:
if base is None:
......@@ -51,6 +93,8 @@ class DemoStorage(ConflictResolvingStorage):
close_base_on_close = False
else:
close_base_on_close = True
elif base is None:
base = ZODB.MappingStorage.MappingStorage()
self.base = base
self.close_base_on_close = close_base_on_close
......@@ -285,10 +329,17 @@ class DemoStorage(ConflictResolvingStorage):
raise
def pop(self):
"""Close the changes database and return the base.
"""
self.changes.close()
return self.base
def push(self, changes=None):
"""Create a new demo storage using the storage as a base.
The given changes are used as the changes for the returned
storage and ``False`` is passed as ``close_base_on_close``.
"""
return self.__class__(base=self, changes=changes,
close_base_on_close=False)
......
src/ZODB/FileStorage/FileStorage.py
View file @ 8db9c2bc
......@@ -140,7 +140,8 @@ class FileStorage(
ConflictResolvingStorage,
BaseStorage,
):
"""Storage that saves data in a file
"""
# Set True while a pack is in progress; undo is blocked for the duration.
_pack_is_in_progress = False
......@@ -148,6 +149,82 @@ class FileStorage(
def __init__(self, file_name, create=False, read_only=False, stop=None,
quota=None, pack_gc=True, pack_keep_old=True, packer=None,
blob_dir=None):
"""Create a file storage
:param str file_name: Path to store data file
:param bool create: Flag indicating whether a file should be
created even if it already exists.
:param bool read_only: Flag indicating whether the file is
read only. Only one process is able to open the file
non-read-only.
:param bytes stop: Time-travel transaction id
When the file is opened, data will be read up to the given
transaction id. Transaction ids correspond to times and
you can compute transaction ids for a given time using
:class:`~ZODB.TimeStamp.TimeStamp`.
:param int quota: File-size quota
:param bool pack_gc: Flag indicating whether garbage
collection should be performed when packing.
:param bool pack_keep_old: flag indicating whether old data
files should be retained after packing as a ``.old`` file.
:param callable packer: An alternative
:interface:`packer <ZODB.FileStorage.interfaces.IFileStoragePacker>`.
:param str blob_dir: A blob-directory path name.
Blobs will be supported if this option is provided.
A file storage stores data in a single file that behaves like
a traditional transaction log. New data records are appended
to the end of the file. Periodically, the file is packed to
free up space. When this is done, current records as of the
pack time or later are copied to a new file, which replaces
the old file.
FileStorages keep in-memory indexes mapping object oids to the
location of their current records in the file. Back pointers to
previous records allow access to non-current records from the
current records.
In addition to the data file, some ancillary files are
created. These can be lost without affecting data
integrity, however losing the index file may cause extremely
slow startup. Each has a name that's a concatenation of the
original file and a suffix. The files are listed below by
suffix:
.index
Snapshot of the in-memory index. This are created on
shutdown, packing, and after rebuilding an index when one
was not found. For large databases, creating a
file-storage object without an index file can take very
long because it's necessary to scan the data file to build
the index.
.lock
A lock file preventing multiple processes from opening a
file storage on non-read-only mode.
.tmp
A file used to store data being committed in the first phase
of 2-phase commit
.index_tmp
A temporary file used when saving the in-memory index to
avoid overwriting an existing index until a new index has
been fully saved.
.pack
A temporary file written while packing containing current
records as of and after the pack time.
.old
The previous database file after a pack.
When the database is packed, current records as of the pack
time and later are written to the ``.pack`` file. At the end
of packing, the ``.old`` file is removed, if it exists, and
the data file is renamed to the ``.old`` file and finally the
``.pack`` file is rewritten to the data file.
"""
if read_only:
self._is_read_only = True
......
src/ZODB/FileStorage/interfaces.py
View file @ 8db9c2bc
......@@ -16,18 +16,27 @@ import zope.interface
class IFileStoragePacker(zope.interface.Interface):
def __call__(storage, referencesf, stop, gc):
"""Pack the file storage into a new file
r"""Pack the file storage into a new file
:param FileStorage storage: The storage object to be packed
:param callable referencesf: A function that extracts object
references from a pickle bytes string. This is usually
``ZODB.serialize.referencesf``.
:param bytes stop: A transaction id representing the time at
which to stop packing.
:param bool gc: A flag indicating whether garbage collection
should be performed.
The new file will have the same name as the old file with
'.pack' appended. (The packer can get the old file name via
``.pack`` appended. (The packer can get the old file name via
storage._file.name.) If blobs are supported, if the storages
blob_dir attribute is not None or empty, then a .removed file
most be created in the blob directory. This file contains of
the form:
must be created in the blob directory. This file contains records of
the form::
(oid+serial).encode('hex')+'\n'
or, of the form:
or, of the form::
oid.encode('hex')+'\n'
......
src/ZODB/MappingStorage.py
View file @ 8db9c2bc
......@@ -32,8 +32,22 @@ import zope.interface
ZODB.interfaces.IStorageIteration,
)
class MappingStorage(object):
"""In-memory storage implementation
Note that this implementation is somewhat naive and inefficient
with regard to locking. It's implementation is primarily meant to
be a simple illustration of storage implementation. It's also
useful for testing and exploration where scalability and efficiency
are unimportant.
"""
def __init__(self, name='MappingStorage'):
"""Create a mapping storage
The name parameter is used by the
:meth:`~ZODB.interfaces.IStorage.getName` and
:meth:`~ZODB.interfaces.IStorage.sortKey` methods.
"""
self.__name__ = name
self._data = {} # {oid->{tid->pickle}}
self._transactions = BTrees.OOBTree.OOBTree() # {tid->TransactionRecord}
......
src/ZODB/interfaces.py
View file @ 8db9c2bc
......@@ -437,7 +437,6 @@ class IStorage(Interface):
"""A storage is responsible for storing and retrieving data of objects.
Consistency and locking
-----------------------
When transactions are committed, a storage assigns monotonically
increasing transaction identifiers (tids) to the transactions and
......@@ -472,6 +471,9 @@ class IStorage(Interface):
Finalize the storage, releasing any external resources. The
storage should not be used after this method is called.
Note that daabses close theor storages when they're closed, so
this method isn't generally called from application code.
"""
def getName():
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7