Commit 706a0912 authored by Michael Howitz's avatar Michael Howitz Committed by GitHub

Merge pull request #357 from zopefoundation/config-with-pure-python

* Config with pure python
* Lint the code.
* Add support for Python 3.9 and 3.10.
parents 1f4c6429 f79c50b6
[run]
source = src/ZODB/
parallel = true
omit =
src/ZODB/tests/*
src/ZODB/scripts/tests/*
[report]
exclude_lines =
pragma: nocover
pragma: no cover
if __name__ == ['"]__main__['"]:
assert False
self.fail
# Generated from:
# https://github.com/zopefoundation/meta/tree/master/config/pure-python
#
# EditorConfig Configuration file, for more details see:
# http://EditorConfig.org
# EditorConfig is a convention description, that could be interpreted
# by multiple editors to enforce common coding conventions for specific
# file types
# top-most EditorConfig file:
# Will ignore other EditorConfig files in Home directory or upper tree level.
root = true
[*] # For All Files
# Unix-style newlines with a newline ending every file
end_of_line = lf
insert_final_newline = true
trim_trailing_whitespace = true
# Set default charset
charset = utf-8
# Indent style default
indent_style = space
# Max Line Length - a hard line wrap, should be disabled
max_line_length = off
[*.{py,cfg,ini}]
# 4 space indentation
indent_size = 4
[*.{yml,zpt,pt,dtml,zcml}]
# 2 space indentation
indent_size = 2
[{Makefile,.gitmodules}]
# Tab indentation (no size specified, but view as 4 spaces)
indent_style = tab
indent_size = unset
tab_width = unset
# Generated from:
# https://github.com/zopefoundation/meta/tree/master/config/pure-python
name: tests name: tests
on: on:
...@@ -5,27 +7,41 @@ on: ...@@ -5,27 +7,41 @@ on:
pull_request: pull_request:
schedule: schedule:
- cron: '0 12 * * 0' # run once a week on Sunday - cron: '0 12 * * 0' # run once a week on Sunday
# Allow to run this workflow manually from the Actions tab
workflow_dispatch:
jobs: jobs:
build: build:
strategy: strategy:
# We want to see all failures:
fail-fast: false fail-fast: false
matrix: matrix:
os:
- ubuntu
- windows
config: config:
# [Python version, tox env] # [Python version, tox env]
- ["3.9", "lint"]
- ["2.7", "py27"] - ["2.7", "py27"]
- ["3.5", "py35"] - ["3.5", "py35"]
- ["3.6", "py36"] - ["3.6", "py36"]
- ["3.7", "py37"] - ["3.7", "py37"]
- ["3.8", "py38"] - ["3.8", "py38"]
- ["3.8", "py38-pure"] - ["3.9", "py39"]
- ["3.10", "py310"]
- ["pypy2", "pypy"] - ["pypy2", "pypy"]
- ["pypy3", "pypy3"] - ["pypy3", "pypy3"]
- ["3.7", "docs"] - ["3.9", "docs"]
- ["3.7", "coverage"] - ["3.9", "coverage"]
- ["3.8", "py38-pure"]
exclude:
- { os: windows, config: ["3.9", "lint"] }
- { os: windows, config: ["3.9", "docs"] }
- { os: windows, config: ["3.9", "coverage"] }
- { os: windows, config: ["pypy2", "pypy"] }
runs-on: ubuntu-latest runs-on: ${{ matrix.os }}-latest
name: ${{ matrix.config[1] }} name: ${{ matrix.os }}-${{ matrix.config[1] }}
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v2
- name: Set up Python - name: Set up Python
......
bin # Generated from:
eggs # https://github.com/zopefoundation/meta/tree/master/config/pure-python
develop-eggs *.dll
parts *.egg-info/
.installed.cfg *.profraw
build
doc/_build
__pycache__
*.pyc *.pyc
*.pyo
*.so *.so
.tox
.coverage .coverage
.coverage.* .coverage.*
nosetests.xml
coverage.xml
*.egg-info
*.egg
dist
testing.log
.eggs/ .eggs/
.dir-locals.el .installed.cfg
htmlcov .mr.developer.cfg
tmp .tox/
*~ .vscode/
.*.swp __pycache__/
bin/
build/
coverage.xml
develop-eggs/
develop/
dist/
docs/_build
eggs/
etc/
lib/ lib/
lib64 lib64
log/
parts/
pyvenv.cfg pyvenv.cfg
var/
# Generated from:
# https://github.com/zopefoundation/meta/tree/master/config/pure-python
[meta]
template = "pure-python"
commit-id = "de499940b679dcda1c60b089f30134146da31e9a"
[python]
with-windows = true
with-pypy = true
with-future-python = false
with-legacy-python = true
with-docs = true
with-sphinx-doctests = false
[tox]
use-flake8 = true
testenv-setenv = [
"ZOPE_INTERFACE_STRICT_IRO=1",
]
additional-envlist = [
"py38-pure",
]
testenv-additional = [
"",
"[testenv:py38-pure]",
"basepython = python3.8",
"setenv =",
" PURE_PYTHON = 1",
]
[coverage]
fail-under = 80
[flake8]
additional-config = [
"# F401 imported but unused",
"per-file-ignores =",
" src/ZODB/FileStorage/__init__.py: F401",
" src/ZODB/__init__.py: F401",
]
[manifest]
additional-rules = [
"include COPYING",
"recursive-include docs *.ico",
"recursive-include docs *.png",
"recursive-include docs *.svg",
"recursive-include src *.fs",
"recursive-include src *.rst",
"recursive-include src *.test",
"recursive-include src *.txt",
"recursive-include src *.xml",
]
[check-manifest]
additional-ignores = [
"docs/_build/doctest/*/*/*/*",
"docs/_build/doctest/*/*/*",
"docs/_build/doctest/*/*",
"docs/_build/html/*/*/*/*",
"docs/_build/html/*/*/*",
"docs/_build/html/*/*",
]
[github-actions]
additional-config = [
"- [\"3.8\", \"py38-pure\"]",
]
additional-exclude = [
"- { os: windows, config: [\"pypy2\", \"pypy\"] }",
]
...@@ -2,7 +2,7 @@ ...@@ -2,7 +2,7 @@
Change History Change History
================ ================
5.6.1 (unreleased) 5.7.0 (unreleased)
================== ==================
- Fix ``TypeError: can't concat str to bytes`` when running fsoids.py script with Python 3. - Fix ``TypeError: can't concat str to bytes`` when running fsoids.py script with Python 3.
...@@ -24,6 +24,8 @@ ...@@ -24,6 +24,8 @@
- Fix deprecation warnings occurring on Python 3.10. - Fix deprecation warnings occurring on Python 3.10.
- Add support for Python 3.9 and 3.10.
5.6.0 (2020-06-11) 5.6.0 (2020-06-11)
================== ==================
......
# Generated from:
# https://github.com/zopefoundation/meta/tree/master/config/pure-python
include *.rst include *.rst
include *.txt include *.txt
include *.py include buildout.cfg
include *.ini include tox.ini
exclude .coveragerc
exclude .travis.yml
exclude appveyor.yml
exclude buildout.cfg
include COPYING
recursive-include doc *
recursive-include src *
global-exclude *.dll recursive-include docs *.py
global-exclude *.pyc recursive-include docs *.rst
global-exclude *.pyo recursive-include docs *.txt
global-exclude *.so recursive-include docs Makefile
global-exclude *~
recursive-include src *.py
include COPYING
recursive-include docs *.ico
recursive-include docs *.png
recursive-include docs *.svg
recursive-include src *.fs
recursive-include src *.rst
recursive-include src *.test
recursive-include src *.txt
recursive-include src *.xml
environment:
matrix:
- python: 27
- python: 27-x64
- python: 35
- python: 35-x64
- python: 36
- python: 36-x64
- python: 37
- python: 37-x64
- python: 38
- python: 38-x64
install:
- "SET PATH=C:\\Python%PYTHON%;c:\\Python%PYTHON%\\scripts;%PATH%"
- echo "C:\Program Files\Microsoft SDKs\Windows\v7.1\Bin\SetEnv.cmd" /x64 > "C:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\bin\amd64\vcvars64.bat"
- python -m pip install -U pip setuptools wheel
- pip install -U -e .[test]
build_script:
- python -m pip install -U wheel
- python -W ignore setup.py -q bdist_wheel
test_script:
- zope-testrunner --test-path=src -vvv --auto-color
##############################################################################
#
# Copyright (c) 2006 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Bootstrap a buildout-based project
Simply run this script in a directory containing a buildout.cfg.
The script accepts buildout command-line options, so you can
use the -c option to specify an alternate configuration file.
"""
import os
import shutil
import sys
import tempfile
from optparse import OptionParser
tmpeggs = tempfile.mkdtemp()
usage = '''\
[DESIRED PYTHON FOR BUILDOUT] bootstrap.py [options]
Bootstraps a buildout-based project.
Simply run this script in a directory containing a buildout.cfg, using the
Python that you want bin/buildout to use.
Note that by using --find-links to point to local resources, you can keep
this script from going over the network.
'''
parser = OptionParser(usage=usage)
parser.add_option("-v", "--version", help="use a specific zc.buildout version")
parser.add_option("-t", "--accept-buildout-test-releases",
dest='accept_buildout_test_releases',
action="store_true", default=False,
help=("Normally, if you do not specify a --version, the "
"bootstrap script and buildout gets the newest "
"*final* versions of zc.buildout and its recipes and "
"extensions for you. If you use this flag, "
"bootstrap and buildout will get the newest releases "
"even if they are alphas or betas."))
parser.add_option("-c", "--config-file",
help=("Specify the path to the buildout configuration "
"file to be used."))
parser.add_option("-f", "--find-links",
help=("Specify a URL to search for buildout releases"))
parser.add_option("--allow-site-packages",
action="store_true", default=False,
help=("Let bootstrap.py use existing site packages"))
parser.add_option("--setuptools-version",
help="use a specific setuptools version")
options, args = parser.parse_args()
######################################################################
# load/install setuptools
try:
if options.allow_site_packages:
import setuptools
import pkg_resources
from urllib.request import urlopen
except ImportError:
from urllib2 import urlopen
ez = {}
exec(urlopen('https://bootstrap.pypa.io/ez_setup.py').read(), ez)
if not options.allow_site_packages:
# ez_setup imports site, which adds site packages
# this will remove them from the path to ensure that incompatible versions
# of setuptools are not in the path
import site
# inside a virtualenv, there is no 'getsitepackages'.
# We can't remove these reliably
if hasattr(site, 'getsitepackages'):
for sitepackage_path in site.getsitepackages():
sys.path[:] = [x for x in sys.path if sitepackage_path not in x]
setup_args = dict(to_dir=tmpeggs, download_delay=0)
if options.setuptools_version is not None:
setup_args['version'] = options.setuptools_version
ez['use_setuptools'](**setup_args)
import setuptools
import pkg_resources
# This does not (always?) update the default working set. We will
# do it.
for path in sys.path:
if path not in pkg_resources.working_set.entries:
pkg_resources.working_set.add_entry(path)
######################################################################
# Install buildout
ws = pkg_resources.working_set
cmd = [sys.executable, '-c',
'from setuptools.command.easy_install import main; main()',
'-mZqNxd', tmpeggs]
find_links = os.environ.get(
'bootstrap-testing-find-links',
options.find_links or
('http://downloads.buildout.org/'
if options.accept_buildout_test_releases else None)
)
if find_links:
cmd.extend(['-f', find_links])
setuptools_path = ws.find(
pkg_resources.Requirement.parse('setuptools')).location
requirement = 'zc.buildout'
version = options.version
if version is None and not options.accept_buildout_test_releases:
# Figure out the most recent final version of zc.buildout.
import setuptools.package_index
_final_parts = '*final-', '*final'
def _final_version(parsed_version):
try:
return not parsed_version.is_prerelease
except AttributeError:
# Older setuptools
for part in parsed_version:
if (part[:1] == '*') and (part not in _final_parts):
return False
return True
index = setuptools.package_index.PackageIndex(
search_path=[setuptools_path])
if find_links:
index.add_find_links((find_links,))
req = pkg_resources.Requirement.parse(requirement)
if index.obtain(req) is not None:
best = []
bestv = None
for dist in index[req.project_name]:
distv = dist.parsed_version
if _final_version(distv):
if bestv is None or distv > bestv:
best = [dist]
bestv = distv
elif distv == bestv:
best.append(dist)
if best:
best.sort()
version = best[-1].version
if version:
requirement = '=='.join((requirement, version))
cmd.append(requirement)
import subprocess
if subprocess.call(cmd, env=dict(os.environ, PYTHONPATH=setuptools_path)) != 0:
raise Exception(
"Failed to execute command:\n%s" % repr(cmd)[1:-1])
######################################################################
# Import and run buildout
ws.add_entry(tmpeggs)
ws.require(requirement)
import zc.buildout.buildout
if not [a for a in args if '=' not in a]:
args.append('bootstrap')
# if -c was provided, we push it back into args for buildout' main function
if options.config_file is not None:
args[0:0] = ['-c', options.config_file]
zc.buildout.buildout.main(args)
shutil.rmtree(tmpeggs)
...@@ -56,7 +56,7 @@ master_doc = 'index' ...@@ -56,7 +56,7 @@ master_doc = 'index'
# General information about the project. # General information about the project.
project = 'ZODB' project = 'ZODB'
copyright = '2009-2020, Zope Foundation' copyright = '2009-2021, Zope Foundation'
# The version info for the project you're documenting, acts as replacement for # The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the # |version| and |release|, also used in various other places throughout the
......
#! /usr/bin/env python
"""Update version numbers and release dates for the next release.
usage: release.py version date
version should be a string like "3.2.0c1"
date should be a string like "23-Sep-2003"
The following files are updated:
- setup.py
- NEWS.txt
- doc/guide/zodb.tex
- src/ZEO/__init__.py
- src/ZEO/version.txt
- src/ZODB/__init__.py
"""
import fileinput
import os
import re
# In file filename, replace the first occurrence of regexp pat with
# string repl.
def replace(filename, pat, repl):
from sys import stderr as e # fileinput hijacks sys.stdout
foundone = False
for line in fileinput.input([filename], inplace=True, backup="~"):
if foundone:
print line,
else:
match = re.search(pat, line)
if match is not None:
foundone = True
new = re.sub(pat, repl, line)
print new,
print >> e, "In %s, replaced:" % filename
print >> e, " ", repr(line)
print >> e, " ", repr(new)
else:
print line,
if not foundone:
print >> e, "*" * 60, "Oops!"
print >> e, " Failed to find %r in %r" % (pat, filename)
# Nothing in our codebase cares about ZEO/version.txt. Jeremy said
# someone asked for it so that a shell script could read up the ZEO
# version easily.
# Before ZODB 3.4, the ZEO version was one smaller than the ZODB version;
# e.g., ZEO 2.2.7 shipped with ZODB 3.2.7. Now ZEO and ZODB share their
# version number.
def write_zeoversion(path, version):
with open(path, "w") as f:
print >> f, version
def main(args):
version, date = args
replace("setup.py",
r'^VERSION = "\S+"$',
'VERSION = "%s"' % version)
replace("src/ZODB/__init__.py",
r'__version__ = "\S+"',
'__version__ = "%s"' % version)
replace("src/ZEO/__init__.py",
r'version = "\S+"',
'version = "%s"' % version)
write_zeoversion("src/ZEO/version.txt", version)
replace("NEWS.txt",
r"^Release date: .*",
"Release date: %s" % date)
replace("doc/guide/zodb.tex",
r"release{\S+}",
"release{%s}" % version)
if __name__ == "__main__":
import sys
main(sys.argv[1:])
# Generated from:
# https://github.com/zopefoundation/meta/tree/master/config/pure-python
[bdist_wheel] [bdist_wheel]
universal = 1 universal = 1
[flake8]
doctests = 1
# F401 imported but unused
per-file-ignores =
src/ZODB/FileStorage/__init__.py: F401
src/ZODB/__init__.py: F401
[check-manifest]
ignore =
.editorconfig
.meta.toml
docs/_build/html/_sources/*
docs/_build/doctest/*/*/*/*
docs/_build/doctest/*/*/*
docs/_build/doctest/*/*
docs/_build/html/*/*/*/*
docs/_build/html/*/*/*
docs/_build/html/*/*
...@@ -13,7 +13,7 @@ ...@@ -13,7 +13,7 @@
############################################################################## ##############################################################################
from setuptools import setup, find_packages from setuptools import setup, find_packages
version = '5.6.1.dev0' version = '5.7.0.dev0'
classifiers = """\ classifiers = """\
Intended Audience :: Developers Intended Audience :: Developers
...@@ -26,6 +26,8 @@ Programming Language :: Python :: 3.5 ...@@ -26,6 +26,8 @@ Programming Language :: Python :: 3.5
Programming Language :: Python :: 3.6 Programming Language :: Python :: 3.6
Programming Language :: Python :: 3.7 Programming Language :: Python :: 3.7
Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.8
Programming Language :: Python :: 3.9
Programming Language :: Python :: 3.10
Programming Language :: Python :: Implementation :: CPython Programming Language :: Python :: Implementation :: CPython
Programming Language :: Python :: Implementation :: PyPy Programming Language :: Python :: Implementation :: PyPy
Topic :: Database Topic :: Database
...@@ -35,10 +37,12 @@ Operating System :: Unix ...@@ -35,10 +37,12 @@ Operating System :: Unix
Framework :: ZODB Framework :: ZODB
""" """
def read(path): def read(path):
with open(path) as f: with open(path) as f:
return f.read() return f.read()
long_description = read("README.rst") + "\n\n" + read("CHANGES.rst") long_description = read("README.rst") + "\n\n" + read("CHANGES.rst")
tests_require = [ tests_require = [
...@@ -67,6 +71,13 @@ setup( ...@@ -67,6 +71,13 @@ setup(
tests_require=tests_require, tests_require=tests_require,
extras_require={ extras_require={
'test': tests_require, 'test': tests_require,
'docs': [
'Sphinx',
'ZODB',
'j1m.sphinxautozconfig',
'sphinx_rtd_theme',
'sphinxcontrib_zopeext',
]
}, },
install_requires=[ install_requires=[
'persistent >= 4.4.0', 'persistent >= 4.4.0',
......
...@@ -72,7 +72,7 @@ class ActivityMonitor(object): ...@@ -72,7 +72,7 @@ class ActivityMonitor(object):
'loads': 0, 'loads': 0,
'stores': 0, 'stores': 0,
'connections': 0, 'connections': 0,
}) })
div = res[0] div = res[0]
div_end = div['end'] div_end = div['end']
......
...@@ -20,7 +20,6 @@ from __future__ import print_function ...@@ -20,7 +20,6 @@ from __future__ import print_function
import time import time
import logging import logging
import sys
from struct import pack as _structpack, unpack as _structunpack from struct import pack as _structpack, unpack as _structunpack
import zope.interface import zope.interface
...@@ -35,6 +34,7 @@ from ._compat import py2_hasattr ...@@ -35,6 +34,7 @@ from ._compat import py2_hasattr
log = logging.getLogger("ZODB.BaseStorage") log = logging.getLogger("ZODB.BaseStorage")
class BaseStorage(UndoLogCompatible): class BaseStorage(UndoLogCompatible):
"""Base class that supports storage implementations. """Base class that supports storage implementations.
...@@ -74,12 +74,12 @@ class BaseStorage(UndoLogCompatible): ...@@ -74,12 +74,12 @@ class BaseStorage(UndoLogCompatible):
perhaps other things. It is always held when load() is called, so perhaps other things. It is always held when load() is called, so
presumably the load() implementation should also acquire the lock. presumably the load() implementation should also acquire the lock.
""" """
_transaction=None # Transaction that is being committed _transaction = None # Transaction that is being committed
_tstatus=' ' # Transaction status, used for copying data _tstatus = ' ' # Transaction status, used for copying data
_is_read_only = False _is_read_only = False
def __init__(self, name, base=None): def __init__(self, name, base=None):
self.__name__= name self.__name__ = name
log.debug("create storage %s", self.__name__) log.debug("create storage %s", self.__name__)
# Allocate locks: # Allocate locks:
...@@ -93,7 +93,7 @@ class BaseStorage(UndoLogCompatible): ...@@ -93,7 +93,7 @@ class BaseStorage(UndoLogCompatible):
self._commit_lock_release = self._commit_lock.release self._commit_lock_release = self._commit_lock.release
t = time.time() t = time.time()
t = self._ts = TimeStamp(*(time.gmtime(t)[:5] + (t%60,))) t = self._ts = TimeStamp(*(time.gmtime(t)[:5] + (t % 60,)))
self._tid = t.raw() self._tid = t.raw()
# ._oid is the highest oid in use (0 is always in use -- it's # ._oid is the highest oid in use (0 is always in use -- it's
...@@ -122,7 +122,7 @@ class BaseStorage(UndoLogCompatible): ...@@ -122,7 +122,7 @@ class BaseStorage(UndoLogCompatible):
return self.__name__ return self.__name__
def getSize(self): def getSize(self):
return len(self)*300 # WAG! return len(self)*300 # WAG!
def history(self, oid, version, length=1, filter=None): def history(self, oid, version, length=1, filter=None):
return () return ()
...@@ -151,7 +151,7 @@ class BaseStorage(UndoLogCompatible): ...@@ -151,7 +151,7 @@ class BaseStorage(UndoLogCompatible):
self._oid = possible_new_max_oid self._oid = possible_new_max_oid
def registerDB(self, db): def registerDB(self, db):
pass # we don't care pass # we don't care
def isReadOnly(self): def isReadOnly(self):
return self._is_read_only return self._is_read_only
...@@ -279,6 +279,7 @@ class BaseStorage(UndoLogCompatible): ...@@ -279,6 +279,7 @@ class BaseStorage(UndoLogCompatible):
""" """
copy(other, self, verbose) copy(other, self, verbose)
def copy(source, dest, verbose=0): def copy(source, dest, verbose=0):
"""Copy transactions from a source to a destination storage """Copy transactions from a source to a destination storage
...@@ -287,7 +288,7 @@ def copy(source, dest, verbose=0): ...@@ -287,7 +288,7 @@ def copy(source, dest, verbose=0):
""" """
_ts = None _ts = None
ok = 1 ok = 1
preindex = {}; preindex = {}
preget = preindex.get preget = preindex.get
# restore() is a new storage API method which has an identical # restore() is a new storage API method which has an identical
# signature to store() except that it does not return anything. # signature to store() except that it does not return anything.
...@@ -310,7 +311,8 @@ def copy(source, dest, verbose=0): ...@@ -310,7 +311,8 @@ def copy(source, dest, verbose=0):
else: else:
t = TimeStamp(tid) t = TimeStamp(tid)
if t <= _ts: if t <= _ts:
if ok: print(('Time stamps out of order %s, %s' % (_ts, t))) if ok:
print(('Time stamps out of order %s, %s' % (_ts, t)))
ok = 0 ok = 0
_ts = t.laterThan(_ts) _ts = t.laterThan(_ts)
tid = _ts.raw() tid = _ts.raw()
...@@ -351,23 +353,24 @@ def checkCurrentSerialInTransaction(self, oid, serial, transaction): ...@@ -351,23 +353,24 @@ def checkCurrentSerialInTransaction(self, oid, serial, transaction):
raise POSException.ReadConflictError( raise POSException.ReadConflictError(
oid=oid, serials=(committed_tid, serial)) oid=oid, serials=(committed_tid, serial))
BaseStorage.checkCurrentSerialInTransaction = checkCurrentSerialInTransaction BaseStorage.checkCurrentSerialInTransaction = checkCurrentSerialInTransaction
@zope.interface.implementer(ZODB.interfaces.IStorageTransactionInformation) @zope.interface.implementer(ZODB.interfaces.IStorageTransactionInformation)
class TransactionRecord(TransactionMetaData): class TransactionRecord(TransactionMetaData):
"""Abstract base class for iterator protocol""" """Abstract base class for iterator protocol"""
def __init__(self, tid, status, user, description, extension): def __init__(self, tid, status, user, description, extension):
self.tid = tid self.tid = tid
self.status = status self.status = status
TransactionMetaData.__init__(self, user, description, extension) TransactionMetaData.__init__(self, user, description, extension)
@zope.interface.implementer(ZODB.interfaces.IStorageRecordInformation) @zope.interface.implementer(ZODB.interfaces.IStorageRecordInformation)
class DataRecord(object): class DataRecord(object):
"""Abstract base class for iterator protocol""" """Abstract base class for iterator protocol"""
version = '' version = ''
def __init__(self, oid, tid, data, prev): def __init__(self, oid, tid, data, prev):
......
...@@ -29,9 +29,11 @@ from pickle import PicklingError ...@@ -29,9 +29,11 @@ from pickle import PicklingError
logger = logging.getLogger('ZODB.ConflictResolution') logger = logging.getLogger('ZODB.ConflictResolution')
class BadClassName(Exception): class BadClassName(Exception):
pass pass
class BadClass(object): class BadClass(object):
def __init__(self, *args): def __init__(self, *args):
...@@ -40,8 +42,11 @@ class BadClass(object): ...@@ -40,8 +42,11 @@ class BadClass(object):
def __reduce__(self): def __reduce__(self):
raise BadClassName(*self.args) raise BadClassName(*self.args)
_class_cache = {} _class_cache = {}
_class_cache_get = _class_cache.get _class_cache_get = _class_cache.get
def find_global(*args): def find_global(*args):
cls = _class_cache_get(args, 0) cls = _class_cache_get(args, 0)
if cls == 0: if cls == 0:
...@@ -60,23 +65,24 @@ def find_global(*args): ...@@ -60,23 +65,24 @@ def find_global(*args):
if cls == 1: if cls == 1:
# Not importable # Not importable
if (isinstance(args, tuple) and len(args) == 2 and if (isinstance(args, tuple) and len(args) == 2 and
isinstance(args[0], six.string_types) and isinstance(args[0], six.string_types) and
isinstance(args[1], six.string_types) isinstance(args[1], six.string_types)):
):
return BadClass(*args) return BadClass(*args)
else: else:
raise BadClassName(*args) raise BadClassName(*args)
return cls return cls
def state(self, oid, serial, prfactory, p=''): def state(self, oid, serial, prfactory, p=''):
p = p or self.loadSerial(oid, serial) p = p or self.loadSerial(oid, serial)
p = self._crs_untransform_record_data(p) p = self._crs_untransform_record_data(p)
file = BytesIO(p) file = BytesIO(p)
unpickler = PersistentUnpickler( unpickler = PersistentUnpickler(
find_global, prfactory.persistent_load, file) find_global, prfactory.persistent_load, file)
unpickler.load() # skip the class tuple unpickler.load() # skip the class tuple
return unpickler.load() return unpickler.load()
class IPersistentReference(zope.interface.Interface): class IPersistentReference(zope.interface.Interface):
'''public contract for references to persistent objects from an object '''public contract for references to persistent objects from an object
with conflicts.''' with conflicts.'''
...@@ -114,10 +120,10 @@ class IPersistentReference(zope.interface.Interface): ...@@ -114,10 +120,10 @@ class IPersistentReference(zope.interface.Interface):
have two references to the same object that are spelled with different have two references to the same object that are spelled with different
data (for instance, one with a class and one without).''' data (for instance, one with a class and one without).'''
@zope.interface.implementer(IPersistentReference) @zope.interface.implementer(IPersistentReference)
class PersistentReference(object): class PersistentReference(object):
weak = False weak = False
oid = database_name = klass = None oid = database_name = klass = None
...@@ -134,7 +140,7 @@ class PersistentReference(object): ...@@ -134,7 +140,7 @@ class PersistentReference(object):
self.data = self.oid, klass.args self.data = self.oid, klass.args
elif isinstance(data, (bytes, str)): elif isinstance(data, (bytes, str)):
self.oid = data self.oid = data
else: # a list else: # a list
reference_type = data[0] reference_type = data[0]
# 'm' = multi_persistent: (database_name, oid, klass) # 'm' = multi_persistent: (database_name, oid, klass)
# 'n' = multi_oid: (database_name, oid) # 'n' = multi_oid: (database_name, oid)
...@@ -165,11 +171,11 @@ class PersistentReference(object): ...@@ -165,11 +171,11 @@ class PersistentReference(object):
def __cmp__(self, other): def __cmp__(self, other):
if self is other or ( if self is other or (
isinstance(other, PersistentReference) and isinstance(other, PersistentReference) and
self.oid == other.oid and self.oid == other.oid and
self.database_name == other.database_name and self.database_name == other.database_name and
not self.weak and not self.weak and
not other.weak): not other.weak):
return 0 return 0
else: else:
raise ValueError( raise ValueError(
...@@ -211,6 +217,7 @@ class PersistentReference(object): ...@@ -211,6 +217,7 @@ class PersistentReference(object):
elif isinstance(data, list) and data[0] == 'm': elif isinstance(data, list) and data[0] == 'm':
return data[1][2] return data[1][2]
class PersistentReferenceFactory(object): class PersistentReferenceFactory(object):
data = None data = None
...@@ -218,7 +225,8 @@ class PersistentReferenceFactory(object): ...@@ -218,7 +225,8 @@ class PersistentReferenceFactory(object):
def persistent_load(self, ref): def persistent_load(self, ref):
if self.data is None: if self.data is None:
self.data = {} self.data = {}
key = tuple(ref) # lists are not hashable; formats are different enough # lists are not hashable; formats are different enough
key = tuple(ref)
# even after eliminating list/tuple distinction # even after eliminating list/tuple distinction
r = self.data.get(key, None) r = self.data.get(key, None)
if r is None: if r is None:
...@@ -227,12 +235,16 @@ class PersistentReferenceFactory(object): ...@@ -227,12 +235,16 @@ class PersistentReferenceFactory(object):
return r return r
def persistent_id(object): def persistent_id(object):
if getattr(object, '__class__', 0) is not PersistentReference: if getattr(object, '__class__', 0) is not PersistentReference:
return None return None
return object.data return object.data
_unresolvable = {} _unresolvable = {}
def tryToResolveConflict(self, oid, committedSerial, oldSerial, newpickle, def tryToResolveConflict(self, oid, committedSerial, oldSerial, newpickle,
committedData=b''): committedData=b''):
# class_tuple, old, committed, newstate = ('',''), 0, 0, 0 # class_tuple, old, committed, newstate = ('',''), 0, 0, 0
...@@ -264,13 +276,12 @@ def tryToResolveConflict(self, oid, committedSerial, oldSerial, newpickle, ...@@ -264,13 +276,12 @@ def tryToResolveConflict(self, oid, committedSerial, oldSerial, newpickle,
_unresolvable[klass] = 1 _unresolvable[klass] = 1
raise ConflictError raise ConflictError
oldData = self.loadSerial(oid, oldSerial) oldData = self.loadSerial(oid, oldSerial)
if not committedData: if not committedData:
committedData = self.loadSerial(oid, committedSerial) committedData = self.loadSerial(oid, committedSerial)
newstate = unpickler.load() newstate = unpickler.load()
old = state(self, oid, oldSerial, prfactory, oldData) old = state(self, oid, oldSerial, prfactory, oldData)
committed = state(self, oid, committedSerial, prfactory, committedData) committed = state(self, oid, committedSerial, prfactory, committedData)
resolved = resolve(old, committed, newstate) resolved = resolve(old, committed, newstate)
...@@ -284,7 +295,7 @@ def tryToResolveConflict(self, oid, committedSerial, oldSerial, newpickle, ...@@ -284,7 +295,7 @@ def tryToResolveConflict(self, oid, committedSerial, oldSerial, newpickle,
logger.debug( logger.debug(
"Conflict resolution on %s failed with %s: %s", "Conflict resolution on %s failed with %s: %s",
klass, e.__class__.__name__, str(e)) klass, e.__class__.__name__, str(e))
except: except: # noqa: E722 do not use bare 'except'
# If anything else went wrong, catch it here and avoid passing an # If anything else went wrong, catch it here and avoid passing an
# arbitrary exception back to the client. The error here will mask # arbitrary exception back to the client. The error here will mask
# the original ConflictError. A client can recover from a # the original ConflictError. A client can recover from a
...@@ -296,6 +307,7 @@ def tryToResolveConflict(self, oid, committedSerial, oldSerial, newpickle, ...@@ -296,6 +307,7 @@ def tryToResolveConflict(self, oid, committedSerial, oldSerial, newpickle,
raise ConflictError(oid=oid, serials=(committedSerial, oldSerial), raise ConflictError(oid=oid, serials=(committedSerial, oldSerial),
data=newpickle) data=newpickle)
class ConflictResolvingStorage(object): class ConflictResolvingStorage(object):
"Mix-in class that provides conflict resolution handling for storages" "Mix-in class that provides conflict resolution handling for storages"
......
This diff is collapsed.
...@@ -41,6 +41,7 @@ from ZODB import valuedoc ...@@ -41,6 +41,7 @@ from ZODB import valuedoc
logger = logging.getLogger('ZODB.DB') logger = logging.getLogger('ZODB.DB')
class AbstractConnectionPool(object): class AbstractConnectionPool(object):
"""Manage a pool of connections. """Manage a pool of connections.
...@@ -111,7 +112,7 @@ class AbstractConnectionPool(object): ...@@ -111,7 +112,7 @@ class AbstractConnectionPool(object):
class ConnectionPool(AbstractConnectionPool): class ConnectionPool(AbstractConnectionPool):
def __init__(self, size, timeout=1<<31): def __init__(self, size, timeout=1 << 31):
super(ConnectionPool, self).__init__(size, timeout) super(ConnectionPool, self).__init__(size, timeout)
# A stack of connections available to hand out. This is a subset # A stack of connections available to hand out. This is a subset
...@@ -127,9 +128,8 @@ class ConnectionPool(AbstractConnectionPool): ...@@ -127,9 +128,8 @@ class ConnectionPool(AbstractConnectionPool):
def _append(self, c): def _append(self, c):
available = self.available available = self.available
cactive = c._cache.cache_non_ghost_count cactive = c._cache.cache_non_ghost_count
if (available and if (available
(available[-1][1]._cache.cache_non_ghost_count > cactive) and (available[-1][1]._cache.cache_non_ghost_count > cactive)):
):
i = len(available) - 1 i = len(available) - 1
while (i and while (i and
(available[i-1][1]._cache.cache_non_ghost_count > cactive) (available[i-1][1]._cache.cache_non_ghost_count > cactive)
...@@ -185,7 +185,7 @@ class ConnectionPool(AbstractConnectionPool): ...@@ -185,7 +185,7 @@ class ConnectionPool(AbstractConnectionPool):
(len(available) > target) (len(available) > target)
or or
(available and available[0][0] < threshhold) (available and available[0][0] < threshhold)
): ):
t, c = available.pop(0) t, c = available.pop(0)
assert not c.opened assert not c.opened
self.all.remove(c) self.all.remove(c)
...@@ -244,7 +244,7 @@ class KeyedConnectionPool(AbstractConnectionPool): ...@@ -244,7 +244,7 @@ class KeyedConnectionPool(AbstractConnectionPool):
# see the comments in ConnectionPool for method descriptions. # see the comments in ConnectionPool for method descriptions.
def __init__(self, size, timeout=1<<31): def __init__(self, size, timeout=1 << 31):
super(KeyedConnectionPool, self).__init__(size, timeout) super(KeyedConnectionPool, self).__init__(size, timeout)
self.pools = {} self.pools = {}
...@@ -303,6 +303,7 @@ def toTimeStamp(dt): ...@@ -303,6 +303,7 @@ def toTimeStamp(dt):
args = utc_struct[:5]+(utc_struct[5] + dt.microsecond/1000000.0,) args = utc_struct[:5]+(utc_struct[5] + dt.microsecond/1000000.0,)
return TimeStamp(*args) return TimeStamp(*args)
def getTID(at, before): def getTID(at, before):
if at is not None: if at is not None:
if before is not None: if before is not None:
...@@ -319,6 +320,7 @@ def getTID(at, before): ...@@ -319,6 +320,7 @@ def getTID(at, before):
before = TimeStamp(before).raw() before = TimeStamp(before).raw()
return before return before
@implementer(IDatabase) @implementer(IDatabase)
class DB(object): class DB(object):
"""The Object Database """The Object Database
...@@ -348,7 +350,7 @@ class DB(object): ...@@ -348,7 +350,7 @@ class DB(object):
def __init__(self, def __init__(self,
storage, storage,
pool_size=7, pool_size=7,
pool_timeout=1<<31, pool_timeout=1 << 31,
cache_size=400, cache_size=400,
cache_size_bytes=0, cache_size_bytes=0,
historical_pool_size=3, historical_pool_size=3,
...@@ -358,7 +360,7 @@ class DB(object): ...@@ -358,7 +360,7 @@ class DB(object):
database_name='unnamed', database_name='unnamed',
databases=None, databases=None,
xrefs=True, xrefs=True,
large_record_size=1<<24, large_record_size=1 << 24,
**storage_args): **storage_args):
"""Create an object database. """Create an object database.
...@@ -425,10 +427,10 @@ class DB(object): ...@@ -425,10 +427,10 @@ class DB(object):
# Setup storage # Setup storage
if isinstance(storage, six.string_types): if isinstance(storage, six.string_types):
from ZODB import FileStorage from ZODB import FileStorage # noqa: F401 import unused
storage = ZODB.FileStorage.FileStorage(storage, **storage_args) storage = ZODB.FileStorage.FileStorage(storage, **storage_args)
elif storage is None: elif storage is None:
from ZODB import MappingStorage from ZODB import MappingStorage # noqa: F401 import unused
storage = ZODB.MappingStorage.MappingStorage(**storage_args) storage = ZODB.MappingStorage.MappingStorage(**storage_args)
else: else:
assert not storage_args assert not storage_args
...@@ -507,6 +509,7 @@ class DB(object): ...@@ -507,6 +509,7 @@ class DB(object):
""" """
detail = {} detail = {}
def f(con, detail=detail): def f(con, detail=detail):
for oid, ob in con._cache.items(): for oid, ob in con._cache.items():
module = getattr(ob.__class__, '__module__', '') module = getattr(ob.__class__, '__module__', '')
...@@ -570,17 +573,18 @@ class DB(object): ...@@ -570,17 +573,18 @@ class DB(object):
'rc': (rc(ob) - 3 - (ob._p_changed is not None) 'rc': (rc(ob) - 3 - (ob._p_changed is not None)
if rc else False), if rc else False),
'state': ob._p_changed, 'state': ob._p_changed,
#'references': con.references(oid), # 'references': con.references(oid),
}) })
self._connectionMap(f) self._connectionMap(f)
return detail return detail
def cacheFullSweep(self): # XXX this is the same as cacheMinimize def cacheFullSweep(self): # XXX this is the same as cacheMinimize
self._connectionMap(lambda c: c._cache.full_sweep()) self._connectionMap(lambda c: c._cache.full_sweep())
def cacheLastGCTime(self): def cacheLastGCTime(self):
m = [0] m = [0]
def f(con, m=m): def f(con, m=m):
t = con._cache.cache_last_gc_time t = con._cache.cache_last_gc_time
if t > m[0]: if t > m[0]:
...@@ -598,6 +602,7 @@ class DB(object): ...@@ -598,6 +602,7 @@ class DB(object):
"""Return the total count of non-ghost objects in all object caches """Return the total count of non-ghost objects in all object caches
""" """
m = [0] m = [0]
def f(con, m=m): def f(con, m=m):
m[0] += con._cache.cache_non_ghost_count m[0] += con._cache.cache_non_ghost_count
...@@ -608,6 +613,7 @@ class DB(object): ...@@ -608,6 +613,7 @@ class DB(object):
"""Return non-ghost counts sizes for all connections. """Return non-ghost counts sizes for all connections.
""" """
m = [] m = []
def f(con, m=m): def f(con, m=m):
m.append({'connection': repr(con), m.append({'connection': repr(con),
'ngsize': con._cache.cache_non_ghost_count, 'ngsize': con._cache.cache_non_ghost_count,
...@@ -731,7 +737,7 @@ class DB(object): ...@@ -731,7 +737,7 @@ class DB(object):
before = getTID(at, before) before = getTID(at, before)
if (before is not None and if (before is not None and
before > self.lastTransaction() and before > self.lastTransaction() and
before > getTID(self.lastTransaction(), None)): before > getTID(self.lastTransaction(), None)):
raise ValueError( raise ValueError(
'cannot open an historical connection in the future.') 'cannot open an historical connection in the future.')
...@@ -773,7 +779,6 @@ class DB(object): ...@@ -773,7 +779,6 @@ class DB(object):
self.pool.availableGC() self.pool.availableGC()
self.historical_pool.availableGC() self.historical_pool.availableGC()
result.open(transaction_manager) result.open(transaction_manager)
return result return result
...@@ -808,7 +813,7 @@ class DB(object): ...@@ -808,7 +813,7 @@ class DB(object):
t-o)), t-o)),
'info': d, 'info': d,
'before': c.before, 'before': c.before,
}) })
self._connectionMap(get_info) self._connectionMap(get_info)
return result return result
...@@ -836,7 +841,7 @@ class DB(object): ...@@ -836,7 +841,7 @@ class DB(object):
t -= days * 86400 t -= days * 86400
try: try:
self.storage.pack(t, self.references) self.storage.pack(t, self.references)
except: except: # noqa: E722 do not use bare 'except'
logger.exception("packing") logger.exception("packing")
raise raise
...@@ -994,7 +999,7 @@ class DB(object): ...@@ -994,7 +999,7 @@ class DB(object):
Kept for backwards compatibility only. New oids should be Kept for backwards compatibility only. New oids should be
allocated in a transaction using an open Connection. allocated in a transaction using an open Connection.
""" """
return self.storage.new_oid() # pragma: no cover return self.storage.new_oid() # pragma: no cover
def open_then_close_db_when_connection_closes(self): def open_then_close_db_when_connection_closes(self):
"""Create and return a connection. """Create and return a connection.
...@@ -1029,9 +1034,11 @@ class ContextManager(object): ...@@ -1029,9 +1034,11 @@ class ContextManager(object):
self.tm.abort() self.tm.abort()
self.conn.close() self.conn.close()
resource_counter_lock = utils.Lock() resource_counter_lock = utils.Lock()
resource_counter = 0 resource_counter = 0
class TransactionalUndo(object): class TransactionalUndo(object):
def __init__(self, db, tids): def __init__(self, db, tids):
...@@ -1064,9 +1071,10 @@ class TransactionalUndo(object): ...@@ -1064,9 +1071,10 @@ class TransactionalUndo(object):
# a new storage instance, and so we must close it to be sure # a new storage instance, and so we must close it to be sure
# to reclaim resources in a timely manner. # to reclaim resources in a timely manner.
# #
# Once the tpc_begin method has been called, the transaction manager will # Once the tpc_begin method has been called, the transaction manager
# guarantee to call either `tpc_finish` or `tpc_abort`, so those are the only # will guarantee to call either `tpc_finish` or `tpc_abort`, so those
# methods we need to be concerned about calling close() from. # are the only methods we need to be concerned about calling close()
# from.
db_mvcc_storage = self._db._mvcc_storage db_mvcc_storage = self._db._mvcc_storage
self._storage = getattr( self._storage = getattr(
db_mvcc_storage, db_mvcc_storage,
...@@ -1117,7 +1125,10 @@ def connection(*args, **kw): ...@@ -1117,7 +1125,10 @@ def connection(*args, **kw):
""" """
return DB(*args, **kw).open_then_close_db_when_connection_closes() return DB(*args, **kw).open_then_close_db_when_connection_closes()
_transaction_meta_data_text_variables = 'user_name', 'description' _transaction_meta_data_text_variables = 'user_name', 'description'
def _text_transaction_info(info): def _text_transaction_info(info):
for d in info: for d in info:
for name in _transaction_meta_data_text_variables: for name in _transaction_meta_data_text_variables:
......
...@@ -35,10 +35,11 @@ import zope.interface ...@@ -35,10 +35,11 @@ import zope.interface
from .ConflictResolution import ConflictResolvingStorage from .ConflictResolution import ConflictResolvingStorage
from .utils import load_current, maxtid from .utils import load_current, maxtid
@zope.interface.implementer( @zope.interface.implementer(
ZODB.interfaces.IStorage, ZODB.interfaces.IStorage,
ZODB.interfaces.IStorageIteration, ZODB.interfaces.IStorageIteration,
) )
class DemoStorage(ConflictResolvingStorage): class DemoStorage(ConflictResolvingStorage):
"""A storage that stores changes against a read-only base database """A storage that stores changes against a read-only base database
...@@ -99,7 +100,6 @@ class DemoStorage(ConflictResolvingStorage): ...@@ -99,7 +100,6 @@ class DemoStorage(ConflictResolvingStorage):
self.base = base self.base = base
self.close_base_on_close = close_base_on_close self.close_base_on_close = close_base_on_close
if changes is None: if changes is None:
self._temporary_changes = True self._temporary_changes = True
changes = ZODB.MappingStorage.MappingStorage() changes = ZODB.MappingStorage.MappingStorage()
...@@ -128,16 +128,15 @@ class DemoStorage(ConflictResolvingStorage): ...@@ -128,16 +128,15 @@ class DemoStorage(ConflictResolvingStorage):
self._copy_methods_from_changes(changes) self._copy_methods_from_changes(changes)
self._next_oid = random.randint(1, 1<<62) self._next_oid = random.randint(1, 1 << 62)
def _blobify(self): def _blobify(self):
if (self._temporary_changes and if (self._temporary_changes and
isinstance(self.changes, ZODB.MappingStorage.MappingStorage) isinstance(self.changes, ZODB.MappingStorage.MappingStorage)):
):
blob_dir = tempfile.mkdtemp('.demoblobs') blob_dir = tempfile.mkdtemp('.demoblobs')
_temporary_blobdirs[ _temporary_blobdirs[
weakref.ref(self, cleanup_temporary_blobdir) weakref.ref(self, cleanup_temporary_blobdir)
] = blob_dir ] = blob_dir
self.changes = ZODB.blob.BlobStorage(blob_dir, self.changes) self.changes = ZODB.blob.BlobStorage(blob_dir, self.changes)
self._copy_methods_from_changes(self.changes) self._copy_methods_from_changes(self.changes)
return True return True
...@@ -147,6 +146,7 @@ class DemoStorage(ConflictResolvingStorage): ...@@ -147,6 +146,7 @@ class DemoStorage(ConflictResolvingStorage):
self.changes.cleanup() self.changes.cleanup()
__opened = True __opened = True
def opened(self): def opened(self):
return self.__opened return self.__opened
...@@ -162,7 +162,7 @@ class DemoStorage(ConflictResolvingStorage): ...@@ -162,7 +162,7 @@ class DemoStorage(ConflictResolvingStorage):
'_lock', '_lock',
'getSize', 'isReadOnly', 'getSize', 'isReadOnly',
'sortKey', 'tpc_transaction', 'sortKey', 'tpc_transaction',
): ):
setattr(self, meth, getattr(changes, meth)) setattr(self, meth, getattr(changes, meth))
supportsUndo = getattr(changes, 'supportsUndo', None) supportsUndo = getattr(changes, 'supportsUndo', None)
...@@ -253,7 +253,7 @@ class DemoStorage(ConflictResolvingStorage): ...@@ -253,7 +253,7 @@ class DemoStorage(ConflictResolvingStorage):
t = self.changes.loadBefore(oid, end_tid) t = self.changes.loadBefore(oid, end_tid)
result = result[:2] + ( result = result[:2] + (
end_tid if end_tid != maxtid else None, end_tid if end_tid != maxtid else None,
) )
return result return result
...@@ -296,7 +296,7 @@ class DemoStorage(ConflictResolvingStorage): ...@@ -296,7 +296,7 @@ class DemoStorage(ConflictResolvingStorage):
def new_oid(self): def new_oid(self):
with self._lock: with self._lock:
while 1: while 1:
oid = ZODB.utils.p64(self._next_oid ) oid = ZODB.utils.p64(self._next_oid)
if oid not in self._issued_oids: if oid not in self._issued_oids:
try: try:
load_current(self.changes, oid) load_current(self.changes, oid)
...@@ -308,7 +308,7 @@ class DemoStorage(ConflictResolvingStorage): ...@@ -308,7 +308,7 @@ class DemoStorage(ConflictResolvingStorage):
self._issued_oids.add(oid) self._issued_oids.add(oid)
return oid return oid
self._next_oid = random.randint(1, 1<<62) self._next_oid = random.randint(1, 1 << 62)
def pack(self, t, referencesf, gc=None): def pack(self, t, referencesf, gc=None):
if gc is None: if gc is None:
...@@ -325,7 +325,7 @@ class DemoStorage(ConflictResolvingStorage): ...@@ -325,7 +325,7 @@ class DemoStorage(ConflictResolvingStorage):
self.changes.pack(t, referencesf, gc=False) self.changes.pack(t, referencesf, gc=False)
except TypeError as v: except TypeError as v:
if 'gc' in str(v): if 'gc' in str(v):
pass # The gc arg isn't supported. Don't pack pass # The gc arg isn't supported. Don't pack
raise raise
def pop(self): def pop(self):
...@@ -344,7 +344,7 @@ class DemoStorage(ConflictResolvingStorage): ...@@ -344,7 +344,7 @@ class DemoStorage(ConflictResolvingStorage):
close_base_on_close=False) close_base_on_close=False)
def store(self, oid, serial, data, version, transaction): def store(self, oid, serial, data, version, transaction):
assert version=='', "versions aren't supported" assert version == '', "versions aren't supported"
if transaction is not self._transaction: if transaction is not self._transaction:
raise ZODB.POSException.StorageTransactionError(self, transaction) raise ZODB.POSException.StorageTransactionError(self, transaction)
...@@ -367,7 +367,7 @@ class DemoStorage(ConflictResolvingStorage): ...@@ -367,7 +367,7 @@ class DemoStorage(ConflictResolvingStorage):
def storeBlob(self, oid, oldserial, data, blobfilename, version, def storeBlob(self, oid, oldserial, data, blobfilename, version,
transaction): transaction):
assert version=='', "versions aren't supported" assert version == '', "versions aren't supported"
if transaction is not self._transaction: if transaction is not self._transaction:
raise ZODB.POSException.StorageTransactionError(self, transaction) raise ZODB.POSException.StorageTransactionError(self, transaction)
...@@ -425,7 +425,7 @@ class DemoStorage(ConflictResolvingStorage): ...@@ -425,7 +425,7 @@ class DemoStorage(ConflictResolvingStorage):
"Unexpected resolved conflicts") "Unexpected resolved conflicts")
return self._resolved return self._resolved
def tpc_finish(self, transaction, func = lambda tid: None): def tpc_finish(self, transaction, func=lambda tid: None):
with self._lock: with self._lock:
if (transaction is not self._transaction): if (transaction is not self._transaction):
raise ZODB.POSException.StorageTransactionError( raise ZODB.POSException.StorageTransactionError(
...@@ -437,11 +437,14 @@ class DemoStorage(ConflictResolvingStorage): ...@@ -437,11 +437,14 @@ class DemoStorage(ConflictResolvingStorage):
self._commit_lock.release() self._commit_lock.release()
return tid return tid
_temporary_blobdirs = {} _temporary_blobdirs = {}
def cleanup_temporary_blobdir( def cleanup_temporary_blobdir(
ref, ref,
_temporary_blobdirs=_temporary_blobdirs, # Make sure it stays around _temporary_blobdirs=_temporary_blobdirs, # Make sure it stays around
): ):
blob_dir = _temporary_blobdirs.pop(ref, None) blob_dir = _temporary_blobdirs.pop(ref, None)
if blob_dir and os.path.exists(blob_dir): if blob_dir and os.path.exists(blob_dir):
ZODB.blob.remove_committed_dir(blob_dir) ZODB.blob.remove_committed_dir(blob_dir)
...@@ -29,17 +29,17 @@ from ZODB._compat import PersistentPickler, Unpickler, BytesIO, _protocol ...@@ -29,17 +29,17 @@ from ZODB._compat import PersistentPickler, Unpickler, BytesIO, _protocol
logger = logging.getLogger('ZODB.ExportImport') logger = logging.getLogger('ZODB.ExportImport')
class ExportImport(object): class ExportImport(object):
def exportFile(self, oid, f=None, bufsize=64 * 1024): def exportFile(self, oid, f=None, bufsize=64 * 1024):
if f is None: if f is None:
f = TemporaryFile(prefix="EXP") f = TemporaryFile(prefix="EXP")
elif isinstance(f, six.string_types): elif isinstance(f, six.string_types):
f = open(f,'w+b') f = open(f, 'w+b')
f.write(b'ZEXP') f.write(b'ZEXP')
oids = [oid] oids = [oid]
done_oids = {} done_oids = {}
done = done_oids.__contains__
load = self._storage.load load = self._storage.load
supports_blobs = IBlobStorage.providedBy(self._storage) supports_blobs = IBlobStorage.providedBy(self._storage)
while oids: while oids:
...@@ -49,7 +49,7 @@ class ExportImport(object): ...@@ -49,7 +49,7 @@ class ExportImport(object):
done_oids[oid] = True done_oids[oid] = True
try: try:
p, serial = load(oid) p, serial = load(oid)
except: except: # noqa: E722 do not use bare 'except'
logger.debug("broken reference for oid %s", repr(oid), logger.debug("broken reference for oid %s", repr(oid),
exc_info=True) exc_info=True)
else: else:
...@@ -58,7 +58,7 @@ class ExportImport(object): ...@@ -58,7 +58,7 @@ class ExportImport(object):
if supports_blobs: if supports_blobs:
if not isinstance(self._reader.getGhost(p), Blob): if not isinstance(self._reader.getGhost(p), Blob):
continue # not a blob continue # not a blob
blobfilename = self._storage.loadBlob(oid, serial) blobfilename = self._storage.loadBlob(oid, serial)
f.write(blob_begin_marker) f.write(blob_begin_marker)
...@@ -159,8 +159,7 @@ class ExportImport(object): ...@@ -159,8 +159,7 @@ class ExportImport(object):
return_oid_list.append(oid) return_oid_list.append(oid)
if (b'blob' in data and if (b'blob' in data and
isinstance(self._reader.getGhost(data), Blob) isinstance(self._reader.getGhost(data), Blob)):
):
# Blob support # Blob support
# Make sure we have a (redundant, overly) blob marker. # Make sure we have a (redundant, overly) blob marker.
...@@ -198,11 +197,14 @@ class ExportImport(object): ...@@ -198,11 +197,14 @@ class ExportImport(object):
export_end_marker = b'\377'*16 export_end_marker = b'\377'*16
blob_begin_marker = b'\000BLOBSTART' blob_begin_marker = b'\000BLOBSTART'
class Ghost(object): class Ghost(object):
__slots__ = ("oid",) __slots__ = ("oid",)
def __init__(self, oid): def __init__(self, oid):
self.oid = oid self.oid = oid
def persistent_id(obj): def persistent_id(obj):
if isinstance(obj, Ghost): if isinstance(obj, Ghost):
return obj.oid return obj.oid
This diff is collapsed.
...@@ -90,9 +90,11 @@ from ZODB.POSException import POSKeyError ...@@ -90,9 +90,11 @@ from ZODB.POSException import POSKeyError
from ZODB.utils import u64, oid_repr, as_bytes from ZODB.utils import u64, oid_repr, as_bytes
from ZODB._compat import PY3 from ZODB._compat import PY3
class CorruptedError(Exception): class CorruptedError(Exception):
pass pass
class CorruptedDataError(CorruptedError): class CorruptedDataError(CorruptedError):
def __init__(self, oid=None, buf=None, pos=None): def __init__(self, oid=None, buf=None, pos=None):
...@@ -110,6 +112,7 @@ class CorruptedDataError(CorruptedError): ...@@ -110,6 +112,7 @@ class CorruptedDataError(CorruptedError):
msg += " at %d" % self.pos msg += " at %d" % self.pos
return msg return msg
# the struct formats for the headers # the struct formats for the headers
TRANS_HDR = ">8sQcHHH" TRANS_HDR = ">8sQcHHH"
DATA_HDR = ">8s8sQQHQ" DATA_HDR = ">8s8sQQHQ"
...@@ -121,6 +124,7 @@ assert struct.calcsize(DATA_HDR) == DATA_HDR_LEN ...@@ -121,6 +124,7 @@ assert struct.calcsize(DATA_HDR) == DATA_HDR_LEN
logger = logging.getLogger('ZODB.FileStorage.format') logger = logging.getLogger('ZODB.FileStorage.format')
class FileStorageFormatter(object): class FileStorageFormatter(object):
"""Mixin class that can read and write the low-level format.""" """Mixin class that can read and write the low-level format."""
...@@ -211,7 +215,7 @@ class FileStorageFormatter(object): ...@@ -211,7 +215,7 @@ class FileStorageFormatter(object):
self.ltid = th.tid self.ltid = th.tid
if th.status == "c": if th.status == "c":
self.fail(pos, "transaction with checkpoint flag set") self.fail(pos, "transaction with checkpoint flag set")
if not th.status in " pu": # recognize " ", "p", and "u" as valid if th.status not in " pu": # recognize " ", "p", and "u" as valid
self.fail(pos, "invalid transaction status: %r", th.status) self.fail(pos, "invalid transaction status: %r", th.status)
if th.tlen < th.headerlen(): if th.tlen < th.headerlen():
self.fail(pos, "invalid transaction header: " self.fail(pos, "invalid transaction header: "
...@@ -232,9 +236,11 @@ class FileStorageFormatter(object): ...@@ -232,9 +236,11 @@ class FileStorageFormatter(object):
if dh.plen: if dh.plen:
self.fail(pos, "data record has back pointer and data") self.fail(pos, "data record has back pointer and data")
def DataHeaderFromString(s): def DataHeaderFromString(s):
return DataHeader(*struct.unpack(DATA_HDR, s)) return DataHeader(*struct.unpack(DATA_HDR, s))
class DataHeader(object): class DataHeader(object):
"""Header for a data record.""" """Header for a data record."""
...@@ -250,7 +256,7 @@ class DataHeader(object): ...@@ -250,7 +256,7 @@ class DataHeader(object):
self.prev = prev self.prev = prev
self.tloc = tloc self.tloc = tloc
self.plen = plen self.plen = plen
self.back = 0 # default self.back = 0 # default
def asString(self): def asString(self):
return struct.pack(DATA_HDR, self.oid, self.tid, self.prev, return struct.pack(DATA_HDR, self.oid, self.tid, self.prev,
...@@ -259,12 +265,14 @@ class DataHeader(object): ...@@ -259,12 +265,14 @@ class DataHeader(object):
def recordlen(self): def recordlen(self):
return DATA_HDR_LEN + (self.plen or 8) return DATA_HDR_LEN + (self.plen or 8)
def TxnHeaderFromString(s): def TxnHeaderFromString(s):
res = TxnHeader(*struct.unpack(TRANS_HDR, s)) res = TxnHeader(*struct.unpack(TRANS_HDR, s))
if PY3: if PY3:
res.status = res.status.decode('ascii') res.status = res.status.decode('ascii')
return res return res
class TxnHeader(object): class TxnHeader(object):
"""Header for a transaction record.""" """Header for a transaction record."""
......
...@@ -20,19 +20,20 @@ from ZODB.FileStorage.format import DATA_HDR, DATA_HDR_LEN ...@@ -20,19 +20,20 @@ from ZODB.FileStorage.format import DATA_HDR, DATA_HDR_LEN
from ZODB.TimeStamp import TimeStamp from ZODB.TimeStamp import TimeStamp
from ZODB.utils import u64, get_pickle_metadata from ZODB.utils import u64, get_pickle_metadata
def fsdump(path, file=None, with_offset=1): def fsdump(path, file=None, with_offset=1):
iter = FileIterator(path) iter = FileIterator(path)
for i, trans in enumerate(iter): for i, trans in enumerate(iter):
size = trans._tend - trans._tpos size = trans._tend - trans._tpos
if with_offset: if with_offset:
print(("Trans #%05d tid=%016x size=%d time=%s offset=%d" % print(("Trans #%05d tid=%016x size=%d time=%s offset=%d" %
(i, u64(trans.tid), size, (i, u64(trans.tid), size,
TimeStamp(trans.tid), trans._pos)), file=file) TimeStamp(trans.tid), trans._pos)), file=file)
else: else:
print(("Trans #%05d tid=%016x size=%d time=%s" % print(("Trans #%05d tid=%016x size=%d time=%s" %
(i, u64(trans.tid), size, TimeStamp(trans.tid))), file=file) (i, u64(trans.tid), size, TimeStamp(trans.tid))), file=file)
print((" status=%r user=%r description=%r" % print((" status=%r user=%r description=%r" %
(trans.status, trans.user, trans.description)), file=file) (trans.status, trans.user, trans.description)), file=file)
for j, rec in enumerate(trans): for j, rec in enumerate(trans):
if rec.data is None: if rec.data is None:
...@@ -51,13 +52,15 @@ def fsdump(path, file=None, with_offset=1): ...@@ -51,13 +52,15 @@ def fsdump(path, file=None, with_offset=1):
bp = "" bp = ""
print((" data #%05d oid=%016x%s class=%s%s" % print((" data #%05d oid=%016x%s class=%s%s" %
(j, u64(rec.oid), size, fullclass, bp)), file=file) (j, u64(rec.oid), size, fullclass, bp)), file=file)
iter.close() iter.close()
def fmt(p64): def fmt(p64):
# Return a nicely formatted string for a packaged 64-bit value # Return a nicely formatted string for a packaged 64-bit value
return "%016x" % u64(p64) return "%016x" % u64(p64)
class Dumper(object): class Dumper(object):
"""A very verbose dumper for debuggin FileStorage problems.""" """A very verbose dumper for debuggin FileStorage problems."""
...@@ -87,13 +90,13 @@ class Dumper(object): ...@@ -87,13 +90,13 @@ class Dumper(object):
print("transaction id: %s" % fmt(tid), file=self.dest) print("transaction id: %s" % fmt(tid), file=self.dest)
print("trec len: %d" % tlen, file=self.dest) print("trec len: %d" % tlen, file=self.dest)
print("status: %r" % status, file=self.dest) print("status: %r" % status, file=self.dest)
user = descr = extra = "" user = descr = ""
if ul: if ul:
user = self.file.read(ul) user = self.file.read(ul)
if dl: if dl:
descr = self.file.read(dl) descr = self.file.read(dl)
if el: if el:
extra = self.file.read(el) self.file.read(el)
print("user: %r" % user, file=self.dest) print("user: %r" % user, file=self.dest)
print("description: %r" % descr, file=self.dest) print("description: %r" % descr, file=self.dest)
print("len(extra): %d" % el, file=self.dest) print("len(extra): %d" % el, file=self.dest)
...@@ -121,6 +124,7 @@ class Dumper(object): ...@@ -121,6 +124,7 @@ class Dumper(object):
sbp = self.file.read(8) sbp = self.file.read(8)
print("backpointer: %d" % u64(sbp), file=self.dest) print("backpointer: %d" % u64(sbp), file=self.dest)
def main(): def main():
import sys import sys
fsdump(sys.argv[1]) fsdump(sys.argv[1])
......
...@@ -18,10 +18,14 @@ from ZODB.serialize import get_refs ...@@ -18,10 +18,14 @@ from ZODB.serialize import get_refs
from ZODB.TimeStamp import TimeStamp from ZODB.TimeStamp import TimeStamp
# Extract module.class string from pickle. # Extract module.class string from pickle.
def get_class(pickle): def get_class(pickle):
return "%s.%s" % get_pickle_metadata(pickle) return "%s.%s" % get_pickle_metadata(pickle)
# Shorten a string for display. # Shorten a string for display.
def shorten(s, size=50): def shorten(s, size=50):
if len(s) <= size: if len(s) <= size:
return s return s
...@@ -35,6 +39,7 @@ def shorten(s, size=50): ...@@ -35,6 +39,7 @@ def shorten(s, size=50):
sep = " ... " sep = " ... "
return s[:nleading] + sep + s[-ntrailing:] return s[:nleading] + sep + s[-ntrailing:]
class Tracer(object): class Tracer(object):
"""Trace all occurrences of a set of oids in a FileStorage. """Trace all occurrences of a set of oids in a FileStorage.
...@@ -84,7 +89,7 @@ class Tracer(object): ...@@ -84,7 +89,7 @@ class Tracer(object):
self.oids[oid] = 0 # 0 revisions seen so far self.oids[oid] = 0 # 0 revisions seen so far
def _msg(self, oid, tid, *args): def _msg(self, oid, tid, *args):
self.msgs.append( (oid, tid, ' '.join(map(str, args))) ) self.msgs.append((oid, tid, ' '.join(map(str, args))))
self._produced_msg = True self._produced_msg = True
def report(self): def report(self):
...@@ -98,9 +103,9 @@ class Tracer(object): ...@@ -98,9 +103,9 @@ class Tracer(object):
NOT_SEEN = "this oid was not defined (no data record for it found)" NOT_SEEN = "this oid was not defined (no data record for it found)"
for oid in oids: for oid in oids:
if oid not in oid2name: if oid not in oid2name:
msgs.append( (oid, None, NOT_SEEN) ) msgs.append((oid, None, NOT_SEEN))
msgs.sort() # oids are primary key, tids secondary msgs.sort() # oids are primary key, tids secondary
current_oid = current_tid = None current_oid = current_tid = None
for oid, tid, msg in msgs: for oid, tid, msg in msgs:
if oid != current_oid: if oid != current_oid:
......
...@@ -36,9 +36,11 @@ import ZODB.POSException ...@@ -36,9 +36,11 @@ import ZODB.POSException
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
class PackError(ZODB.POSException.POSError): class PackError(ZODB.POSException.POSError):
pass pass
class PackCopier(FileStorageFormatter): class PackCopier(FileStorageFormatter):
def __init__(self, f, index, tindex): def __init__(self, f, index, tindex):
...@@ -54,7 +56,7 @@ class PackCopier(FileStorageFormatter): ...@@ -54,7 +56,7 @@ class PackCopier(FileStorageFormatter):
self._file.seek(pos - 8) self._file.seek(pos - 8)
pos = pos - u64(self._file.read(8)) - 8 pos = pos - u64(self._file.read(8)) - 8
self._file.seek(pos) self._file.seek(pos)
h = self._file.read(TRANS_HDR_LEN) # XXX bytes h = self._file.read(TRANS_HDR_LEN) # XXX bytes
_tid = h[:8] _tid = h[:8]
if _tid == tid: if _tid == tid:
return pos return pos
...@@ -144,6 +146,7 @@ class PackCopier(FileStorageFormatter): ...@@ -144,6 +146,7 @@ class PackCopier(FileStorageFormatter):
finally: finally:
self._file.seek(pos) self._file.seek(pos)
class GC(FileStorageFormatter): class GC(FileStorageFormatter):
def __init__(self, file, eof, packtime, gc, referencesf): def __init__(self, file, eof, packtime, gc, referencesf):
...@@ -330,6 +333,7 @@ class GC(FileStorageFormatter): ...@@ -330,6 +333,7 @@ class GC(FileStorageFormatter):
else: else:
return [] return []
class FileStoragePacker(FileStorageFormatter): class FileStoragePacker(FileStorageFormatter):
# path is the storage file path. # path is the storage file path.
...@@ -409,15 +413,15 @@ class FileStoragePacker(FileStorageFormatter): ...@@ -409,15 +413,15 @@ class FileStoragePacker(FileStorageFormatter):
# try our best, but don't fail # try our best, but don't fail
try: try:
self._tfile.close() self._tfile.close()
except: except: # noqa: E722 do not use bare 'except'
pass pass
try: try:
self._file.close() self._file.close()
except: except: # noqa: E722 do not use bare 'except'
pass pass
try: try:
os.remove(self._name + ".pack") os.remove(self._name + ".pack")
except: except: # noqa: E722 do not use bare 'except'
pass pass
if self.blob_removed is not None: if self.blob_removed is not None:
self.blob_removed.close() self.blob_removed.close()
...@@ -459,8 +463,8 @@ class FileStoragePacker(FileStorageFormatter): ...@@ -459,8 +463,8 @@ class FileStoragePacker(FileStorageFormatter):
# argument, and then on every platform except native # argument, and then on every platform except native
# Windows it was observed that we could read stale # Windows it was observed that we could read stale
# data from the tail end of the file. # data from the tail end of the file.
self._file.close() # else self.gc keeps the original self._file.close() # else self.gc keeps the original
# alive & open # alive & open
self._file = open(self._path, "rb", 0) self._file = open(self._path, "rb", 0)
self._file.seek(0, 2) self._file.seek(0, 2)
self.file_end = self._file.tell() self.file_end = self._file.tell()
...@@ -483,13 +487,12 @@ class FileStoragePacker(FileStorageFormatter): ...@@ -483,13 +487,12 @@ class FileStoragePacker(FileStorageFormatter):
if self.locked: if self.locked:
self._commit_lock.release() self._commit_lock.release()
raise # don't succeed silently raise # don't succeed silently
except: except: # noqa: E722 do not use bare 'except'
if self.locked: if self.locked:
self._commit_lock.release() self._commit_lock.release()
raise raise
def copyToPacktime(self): def copyToPacktime(self):
offset = 0 # the amount of space freed by packing
pos = self._metadata_size pos = self._metadata_size
new_pos = pos new_pos = pos
...@@ -506,7 +509,6 @@ class FileStoragePacker(FileStorageFormatter): ...@@ -506,7 +509,6 @@ class FileStoragePacker(FileStorageFormatter):
self._tfile.seek(new_pos - 8) self._tfile.seek(new_pos - 8)
self._tfile.write(p64(tlen)) self._tfile.write(p64(tlen))
tlen = self._read_num(pos) tlen = self._read_num(pos)
if tlen != th.tlen: if tlen != th.tlen:
self.fail(pos, "redundant transaction length does not " self.fail(pos, "redundant transaction length does not "
...@@ -546,8 +548,8 @@ class FileStoragePacker(FileStorageFormatter): ...@@ -546,8 +548,8 @@ class FileStoragePacker(FileStorageFormatter):
# record. There's a bug in ZEO blob support that causes # record. There's a bug in ZEO blob support that causes
# duplicate data records. # duplicate data records.
rpos = self.gc.reachable.get(h.oid) rpos = self.gc.reachable.get(h.oid)
is_dup = (rpos is_dup = (
and self._read_data_header(rpos).tid == h.tid) rpos and self._read_data_header(rpos).tid == h.tid)
if not is_dup: if not is_dup:
if h.oid not in self.gc.reachable: if h.oid not in self.gc.reachable:
self.blob_removed.write( self.blob_removed.write(
...@@ -569,7 +571,6 @@ class FileStoragePacker(FileStorageFormatter): ...@@ -569,7 +571,6 @@ class FileStoragePacker(FileStorageFormatter):
s = th.asString() s = th.asString()
new_tpos = self._tfile.tell() new_tpos = self._tfile.tell()
self._tfile.write(s) self._tfile.write(s)
new_pos = new_tpos + len(s)
copy = 1 copy = 1
if h.plen: if h.plen:
...@@ -578,7 +579,6 @@ class FileStoragePacker(FileStorageFormatter): ...@@ -578,7 +579,6 @@ class FileStoragePacker(FileStorageFormatter):
data = self.fetchDataViaBackpointer(h.oid, h.back) data = self.fetchDataViaBackpointer(h.oid, h.back)
self.writePackedDataRecord(h, data, new_tpos) self.writePackedDataRecord(h, data, new_tpos)
new_pos = self._tfile.tell()
return new_tpos, pos return new_tpos, pos
......
...@@ -13,6 +13,7 @@ ...@@ -13,6 +13,7 @@
############################################################################## ##############################################################################
import zope.interface import zope.interface
class IFileStoragePacker(zope.interface.Interface): class IFileStoragePacker(zope.interface.Interface):
def __call__(storage, referencesf, stop, gc): def __call__(storage, referencesf, stop, gc):
...@@ -58,20 +59,21 @@ class IFileStoragePacker(zope.interface.Interface): ...@@ -58,20 +59,21 @@ class IFileStoragePacker(zope.interface.Interface):
corresponding to the file records. corresponding to the file records.
""" """
class IFileStorage(zope.interface.Interface): class IFileStorage(zope.interface.Interface):
packer = zope.interface.Attribute( packer = zope.interface.Attribute(
"The IFileStoragePacker to be used for packing." "The IFileStoragePacker to be used for packing."
) )
_file = zope.interface.Attribute( _file = zope.interface.Attribute(
"The file object used to access the underlying data." "The file object used to access the underlying data."
) )
_lock = zope.interface.Attribute( _lock = zope.interface.Attribute(
"The storage lock." "The storage lock."
) )
_commit_lock = zope.interface.Attribute( _commit_lock = zope.interface.Attribute(
"The storage commit lock." "The storage commit lock."
) )
...@@ -29,10 +29,11 @@ checker = renormalizing.RENormalizing([ ...@@ -29,10 +29,11 @@ checker = renormalizing.RENormalizing([
# Python 3 adds module name to exceptions. # Python 3 adds module name to exceptions.
(re.compile("ZODB.POSException.POSKeyError"), r"POSKeyError"), (re.compile("ZODB.POSException.POSKeyError"), r"POSKeyError"),
(re.compile("ZODB.FileStorage.FileStorage.FileStorageQuotaError"), (re.compile("ZODB.FileStorage.FileStorage.FileStorageQuotaError"),
"FileStorageQuotaError"), "FileStorageQuotaError"),
(re.compile('data.fs:[0-9]+'), 'data.fs:<OFFSET>'), (re.compile('data.fs:[0-9]+'), 'data.fs:<OFFSET>'),
]) ])
def pack_keep_old(): def pack_keep_old():
"""Should a copy of the database be kept? """Should a copy of the database be kept?
...@@ -106,6 +107,7 @@ directory for blobs is kept.) ...@@ -106,6 +107,7 @@ directory for blobs is kept.)
>>> db.close() >>> db.close()
""" """
def pack_with_repeated_blob_records(): def pack_with_repeated_blob_records():
""" """
There is a bug in ZEO that causes duplicate bloc database records There is a bug in ZEO that causes duplicate bloc database records
...@@ -144,6 +146,7 @@ def pack_with_repeated_blob_records(): ...@@ -144,6 +146,7 @@ def pack_with_repeated_blob_records():
>>> db.close() >>> db.close()
""" """
def _save_index(): def _save_index():
""" """
...@@ -187,6 +190,7 @@ cleanup ...@@ -187,6 +190,7 @@ cleanup
""" """
def pack_disk_full_copyToPacktime(): def pack_disk_full_copyToPacktime():
"""Recover from a disk full situation by removing the `.pack` file """Recover from a disk full situation by removing the `.pack` file
...@@ -239,6 +243,7 @@ check the data we added ...@@ -239,6 +243,7 @@ check the data we added
>>> db.close() >>> db.close()
""" """
def pack_disk_full_copyRest(): def pack_disk_full_copyRest():
"""Recover from a disk full situation by removing the `.pack` file """Recover from a disk full situation by removing the `.pack` file
...@@ -307,6 +312,7 @@ check the data we added ...@@ -307,6 +312,7 @@ check the data we added
>>> db.close() >>> db.close()
""" """
def test_suite(): def test_suite():
return unittest.TestSuite(( return unittest.TestSuite((
doctest.DocFileSuite( doctest.DocFileSuite(
...@@ -319,4 +325,4 @@ def test_suite(): ...@@ -319,4 +325,4 @@ def test_suite():
setUp=ZODB.tests.util.setUp, setUp=ZODB.tests.util.setUp,
tearDown=ZODB.tests.util.tearDown, tearDown=ZODB.tests.util.tearDown,
checker=checker), checker=checker),
)) ))
...@@ -28,9 +28,9 @@ import zope.interface ...@@ -28,9 +28,9 @@ import zope.interface
@zope.interface.implementer( @zope.interface.implementer(
ZODB.interfaces.IStorage, ZODB.interfaces.IStorage,
ZODB.interfaces.IStorageIteration, ZODB.interfaces.IStorageIteration,
) )
class MappingStorage(object): class MappingStorage(object):
"""In-memory storage implementation """In-memory storage implementation
...@@ -50,7 +50,8 @@ class MappingStorage(object): ...@@ -50,7 +50,8 @@ class MappingStorage(object):
""" """
self.__name__ = name self.__name__ = name
self._data = {} # {oid->{tid->pickle}} self._data = {} # {oid->{tid->pickle}}
self._transactions = BTrees.OOBTree.OOBTree() # {tid->TransactionRecord} # {tid->TransactionRecord}
self._transactions = BTrees.OOBTree.OOBTree()
self._ltid = ZODB.utils.z64 self._ltid = ZODB.utils.z64
self._last_pack = None self._last_pack = None
self._lock = ZODB.utils.RLock() self._lock = ZODB.utils.RLock()
...@@ -117,14 +118,14 @@ class MappingStorage(object): ...@@ -117,14 +118,14 @@ class MappingStorage(object):
tids.reverse() tids.reverse()
return [ return [
dict( dict(
time = ZODB.TimeStamp.TimeStamp(tid).timeTime(), time=ZODB.TimeStamp.TimeStamp(tid).timeTime(),
tid = tid, tid=tid,
serial = tid, serial=tid,
user_name = self._transactions[tid].user, user_name=self._transactions[tid].user,
description = self._transactions[tid].description, description=self._transactions[tid].description,
extension = self._transactions[tid].extension, extension=self._transactions[tid].extension,
size = len(tid_data[tid]) size=len(tid_data[tid])
) )
for tid in tids] for tid in tids]
# ZODB.interfaces.IStorage # ZODB.interfaces.IStorage
...@@ -167,8 +168,8 @@ class MappingStorage(object): ...@@ -167,8 +168,8 @@ class MappingStorage(object):
else: else:
raise ZODB.POSException.POSKeyError(oid) raise ZODB.POSException.POSKeyError(oid)
# ZODB.interfaces.IStorage # ZODB.interfaces.IStorage
@ZODB.utils.locked(opened) @ZODB.utils.locked(opened)
def loadSerial(self, oid, serial): def loadSerial(self, oid, serial):
tid_data = self._data.get(oid) tid_data = self._data.get(oid)
...@@ -192,7 +193,7 @@ class MappingStorage(object): ...@@ -192,7 +193,7 @@ class MappingStorage(object):
if not self._data: if not self._data:
return return
stop = ZODB.TimeStamp.TimeStamp(*time.gmtime(t)[:5]+(t%60,)).raw() stop = ZODB.TimeStamp.TimeStamp(*time.gmtime(t)[:5]+(t % 60,)).raw()
if self._last_pack is not None and self._last_pack >= stop: if self._last_pack is not None and self._last_pack >= stop:
if self._last_pack == stop: if self._last_pack == stop:
return return
...@@ -298,7 +299,7 @@ class MappingStorage(object): ...@@ -298,7 +299,7 @@ class MappingStorage(object):
# ZODB.interfaces.IStorage # ZODB.interfaces.IStorage
@ZODB.utils.locked(opened) @ZODB.utils.locked(opened)
def tpc_finish(self, transaction, func = lambda tid: None): def tpc_finish(self, transaction, func=lambda tid: None):
if (transaction is not self._transaction): if (transaction is not self._transaction):
raise ZODB.POSException.StorageTransactionError( raise ZODB.POSException.StorageTransactionError(
"tpc_finish called with wrong transaction") "tpc_finish called with wrong transaction")
...@@ -332,6 +333,7 @@ class MappingStorage(object): ...@@ -332,6 +333,7 @@ class MappingStorage(object):
raise ZODB.POSException.StorageTransactionError( raise ZODB.POSException.StorageTransactionError(
"tpc_vote called with wrong transaction") "tpc_vote called with wrong transaction")
class TransactionRecord(object): class TransactionRecord(object):
status = ' ' status = ' '
...@@ -357,11 +359,11 @@ class TransactionRecord(object): ...@@ -357,11 +359,11 @@ class TransactionRecord(object):
del self.data[oid] del self.data[oid]
return not self.data return not self.data
@zope.interface.implementer(ZODB.interfaces.IStorageRecordInformation) @zope.interface.implementer(ZODB.interfaces.IStorageRecordInformation)
class DataRecord(object): class DataRecord(object):
"""Abstract base class for iterator protocol""" """Abstract base class for iterator protocol"""
version = '' version = ''
data_txn = None data_txn = None
...@@ -370,5 +372,6 @@ class DataRecord(object): ...@@ -370,5 +372,6 @@ class DataRecord(object):
self.tid = tid self.tid = tid
self.data = data self.data = data
def DB(*args, **kw): def DB(*args, **kw):
return ZODB.DB(MappingStorage(), *args, **kw) return ZODB.DB(MappingStorage(), *args, **kw)
...@@ -18,20 +18,26 @@ $Id$""" ...@@ -18,20 +18,26 @@ $Id$"""
from ZODB.utils import oid_repr, readable_tid_repr from ZODB.utils import oid_repr, readable_tid_repr
# BBB: We moved the two transactions to the transaction package # BBB: We moved the two transactions to the transaction package
from transaction.interfaces import TransactionError, TransactionFailedError from transaction.interfaces import TransactionError # noqa: F401 import unused
from transaction.interfaces import TransactionFailedError # noqa: F401
import transaction.interfaces import transaction.interfaces
def _fmt_undo(oid, reason): def _fmt_undo(oid, reason):
s = reason and (": %s" % reason) or "" s = reason and (": %s" % reason) or ""
return "Undo error %s%s" % (oid_repr(oid), s) return "Undo error %s%s" % (oid_repr(oid), s)
def _recon(class_, state): def _recon(class_, state):
err = class_.__new__(class_) err = class_.__new__(class_)
err.__setstate__(state) err.__setstate__(state)
return err return err
_recon.__no_side_effects__ = True _recon.__no_side_effects__ = True
class POSError(Exception): class POSError(Exception):
"""Persistent object system error.""" """Persistent object system error."""
...@@ -49,9 +55,10 @@ class POSError(Exception): ...@@ -49,9 +55,10 @@ class POSError(Exception):
# the args would then get lost, leading to unprintable exceptions # the args would then get lost, leading to unprintable exceptions
# and worse. Manually assign to args from the state to be sure # and worse. Manually assign to args from the state to be sure
# this doesn't happen. # this doesn't happen.
super(POSError,self).__setstate__(state) super(POSError, self).__setstate__(state)
self.args = state['args'] self.args = state['args']
class POSKeyError(POSError, KeyError): class POSKeyError(POSError, KeyError):
"""Key not found in database.""" """Key not found in database."""
...@@ -143,6 +150,7 @@ class ConflictError(POSError, transaction.interfaces.TransientError): ...@@ -143,6 +150,7 @@ class ConflictError(POSError, transaction.interfaces.TransientError):
def get_serials(self): def get_serials(self):
return self.serials return self.serials
class ReadConflictError(ConflictError): class ReadConflictError(ConflictError):
"""Conflict detected when object was requested to stay unchanged. """Conflict detected when object was requested to stay unchanged.
...@@ -156,64 +164,67 @@ class ReadConflictError(ConflictError): ...@@ -156,64 +164,67 @@ class ReadConflictError(ConflictError):
- object is found to be removed, and - object is found to be removed, and
- there is possibility that database pack was running simultaneously. - there is possibility that database pack was running simultaneously.
""" """
def __init__(self, message=None, object=None, serials=None, **kw): def __init__(self, message=None, object=None, serials=None, **kw):
if message is None: if message is None:
message = "database read conflict error" message = "database read conflict error"
ConflictError.__init__(self, message=message, object=object, ConflictError.__init__(self, message=message, object=object,
serials=serials, **kw) serials=serials, **kw)
class BTreesConflictError(ConflictError): class BTreesConflictError(ConflictError):
"""A special subclass for BTrees conflict errors.""" """A special subclass for BTrees conflict errors."""
msgs = [# 0; i2 or i3 bucket split; positions are all -1 msgs = [
'Conflicting bucket split', # 0; i2 or i3 bucket split; positions are all -1
'Conflicting bucket split',
# 1; keys the same, but i2 and i3 values differ, and both values # 1; keys the same, but i2 and i3 values differ, and both values
# differ from i1's value # differ from i1's value
'Conflicting changes', 'Conflicting changes',
# 2; i1's value changed in i2, but key+value deleted in i3 # 2; i1's value changed in i2, but key+value deleted in i3
'Conflicting delete and change', 'Conflicting delete and change',
# 3; i1's value changed in i3, but key+value deleted in i2 # 3; i1's value changed in i3, but key+value deleted in i2
'Conflicting delete and change', 'Conflicting delete and change',
# 4; i1 and i2 both added the same key, or both deleted the # 4; i1 and i2 both added the same key, or both deleted the
# same key # same key
'Conflicting inserts or deletes', 'Conflicting inserts or deletes',
# 5; i2 and i3 both deleted the same key # 5; i2 and i3 both deleted the same key
'Conflicting deletes', 'Conflicting deletes',
# 6; i2 and i3 both added the same key # 6; i2 and i3 both added the same key
'Conflicting inserts', 'Conflicting inserts',
# 7; i2 and i3 both deleted the same key, or i2 changed the value # 7; i2 and i3 both deleted the same key, or i2 changed the value
# associated with a key and i3 deleted that key # associated with a key and i3 deleted that key
'Conflicting deletes, or delete and change', 'Conflicting deletes, or delete and change',
# 8; i2 and i3 both deleted the same key, or i3 changed the value # 8; i2 and i3 both deleted the same key, or i3 changed the value
# associated with a key and i2 deleted that key # associated with a key and i2 deleted that key
'Conflicting deletes, or delete and change', 'Conflicting deletes, or delete and change',
# 9; i2 and i3 both deleted the same key # 9; i2 and i3 both deleted the same key
'Conflicting deletes', 'Conflicting deletes',
# 10; i2 and i3 deleted all the keys, and didn't insert any, # 10; i2 and i3 deleted all the keys, and didn't insert any,
# leaving an empty bucket; conflict resolution doesn't have # leaving an empty bucket; conflict resolution doesn't have
# enough info to unlink an empty bucket from its containing # enough info to unlink an empty bucket from its containing
# BTree correctly # BTree correctly
'Empty bucket from deleting all keys', 'Empty bucket from deleting all keys',
# 11; conflicting changes in an internal BTree node # 11; conflicting changes in an internal BTree node
'Conflicting changes in an internal BTree node', 'Conflicting changes in an internal BTree node',
# 12; i2 or i3 was empty # 12; i2 or i3 was empty
'Empty bucket in a transaction', 'Empty bucket in a transaction',
# 13; delete of first key, which causes change to parent node # 13; delete of first key, which causes change to parent node
'Delete of first key', 'Delete of first key',
] ]
def __init__(self, p1, p2, p3, reason): def __init__(self, p1, p2, p3, reason):
self.p1 = p1 self.p1 = p1
...@@ -226,11 +237,14 @@ class BTreesConflictError(ConflictError): ...@@ -226,11 +237,14 @@ class BTreesConflictError(ConflictError):
self.p2, self.p2,
self.p3, self.p3,
self.reason) self.reason)
def __str__(self): def __str__(self):
return "BTrees conflict error at %d/%d/%d: %s" % ( return "BTrees conflict error at %d/%d/%d: %s" % (
self.p1, self.p2, self.p3, self.msgs[self.reason]) self.p1, self.p2, self.p3, self.msgs[self.reason])
class DanglingReferenceError(POSError, transaction.interfaces.TransactionError):
class DanglingReferenceError(
POSError, transaction.interfaces.TransactionError):
"""An object has a persistent reference to a missing object. """An object has a persistent reference to a missing object.
If an object is stored and it has a reference to another object If an object is stored and it has a reference to another object
...@@ -258,9 +272,11 @@ class DanglingReferenceError(POSError, transaction.interfaces.TransactionError): ...@@ -258,9 +272,11 @@ class DanglingReferenceError(POSError, transaction.interfaces.TransactionError):
class VersionError(POSError): class VersionError(POSError):
"""An error in handling versions occurred.""" """An error in handling versions occurred."""
class VersionCommitError(VersionError): class VersionCommitError(VersionError):
"""An invalid combination of versions was used in a version commit.""" """An invalid combination of versions was used in a version commit."""
class VersionLockError(VersionError, transaction.interfaces.TransactionError): class VersionLockError(VersionError, transaction.interfaces.TransactionError):
"""Modification to an object modified in an unsaved version. """Modification to an object modified in an unsaved version.
...@@ -269,6 +285,7 @@ class VersionLockError(VersionError, transaction.interfaces.TransactionError): ...@@ -269,6 +285,7 @@ class VersionLockError(VersionError, transaction.interfaces.TransactionError):
""" """
############################################################################ ############################################################################
class UndoError(POSError): class UndoError(POSError):
"""An attempt was made to undo a non-undoable transaction.""" """An attempt was made to undo a non-undoable transaction."""
...@@ -279,6 +296,7 @@ class UndoError(POSError): ...@@ -279,6 +296,7 @@ class UndoError(POSError):
def __str__(self): def __str__(self):
return _fmt_undo(self._oid, self._reason) return _fmt_undo(self._oid, self._reason)
class MultipleUndoErrors(UndoError): class MultipleUndoErrors(UndoError):
"""Several undo errors occurred during a single transaction.""" """Several undo errors occurred during a single transaction."""
...@@ -290,33 +308,43 @@ class MultipleUndoErrors(UndoError): ...@@ -290,33 +308,43 @@ class MultipleUndoErrors(UndoError):
def __str__(self): def __str__(self):
return "\n".join([_fmt_undo(*pair) for pair in self._errs]) return "\n".join([_fmt_undo(*pair) for pair in self._errs])
class StorageError(POSError): class StorageError(POSError):
"""Base class for storage based exceptions.""" """Base class for storage based exceptions."""
class StorageTransactionError(StorageError): class StorageTransactionError(StorageError):
"""An operation was invoked for an invalid transaction or state.""" """An operation was invoked for an invalid transaction or state."""
class StorageSystemError(StorageError): class StorageSystemError(StorageError):
"""Panic! Internal storage error!""" """Panic! Internal storage error!"""
class MountedStorageError(StorageError): class MountedStorageError(StorageError):
"""Unable to access mounted storage.""" """Unable to access mounted storage."""
class ReadOnlyError(StorageError): class ReadOnlyError(StorageError):
"""Unable to modify objects in a read-only storage.""" """Unable to modify objects in a read-only storage."""
class TransactionTooLargeError(StorageTransactionError): class TransactionTooLargeError(StorageTransactionError):
"""The transaction exhausted some finite storage resource.""" """The transaction exhausted some finite storage resource."""
class ExportError(POSError): class ExportError(POSError):
"""An export file doesn't have the right format.""" """An export file doesn't have the right format."""
class Unsupported(POSError): class Unsupported(POSError):
"""A feature was used that is not supported by the storage.""" """A feature was used that is not supported by the storage."""
class ReadOnlyHistoryError(POSError): class ReadOnlyHistoryError(POSError):
"""Unable to add or modify objects in an historical connection.""" """Unable to add or modify objects in an historical connection."""
class InvalidObjectReference(POSError): class InvalidObjectReference(POSError):
"""An object contains an invalid reference to another object. """An object contains an invalid reference to another object.
...@@ -329,6 +357,7 @@ class InvalidObjectReference(POSError): ...@@ -329,6 +357,7 @@ class InvalidObjectReference(POSError):
TODO: The exception ought to have a member that is the invalid object. TODO: The exception ought to have a member that is the invalid object.
""" """
class ConnectionStateError(POSError): class ConnectionStateError(POSError):
"""A Connection isn't in the required state for an operation. """A Connection isn't in the required state for an operation.
......
...@@ -12,6 +12,7 @@ ...@@ -12,6 +12,7 @@
# #
############################################################################## ##############################################################################
from ZODB.DB import DB, connection
import sys import sys
from persistent import TimeStamp from persistent import TimeStamp
...@@ -24,5 +25,3 @@ sys.modules['ZODB.PersistentMapping'] = sys.modules['persistent.mapping'] ...@@ -24,5 +25,3 @@ sys.modules['ZODB.PersistentMapping'] = sys.modules['persistent.mapping']
sys.modules['ZODB.PersistentList'] = sys.modules['persistent.list'] sys.modules['ZODB.PersistentList'] = sys.modules['persistent.list']
del mapping, list, sys del mapping, list, sys
from ZODB.DB import DB, connection
...@@ -11,13 +11,13 @@ ...@@ -11,13 +11,13 @@
# FOR A PARTICULAR PURPOSE # FOR A PARTICULAR PURPOSE
# #
############################################################################## ##############################################################################
from zodbpickle import binary # noqa: F401 import unused
import sys import sys
from six import PY3 from six import PY3
IS_JYTHON = sys.platform.startswith('java') IS_JYTHON = sys.platform.startswith('java')
_protocol = 3 _protocol = 3
from zodbpickle import binary
if not PY3: if not PY3:
# Python 2.x # Python 2.x
...@@ -42,7 +42,8 @@ else: ...@@ -42,7 +42,8 @@ else:
# http://bugs.python.org/issue6784 # http://bugs.python.org/issue6784
import zodbpickle.pickle import zodbpickle.pickle
HIGHEST_PROTOCOL = 3 HIGHEST_PROTOCOL = 3
from _compat_pickle import IMPORT_MAPPING, NAME_MAPPING from _compat_pickle import IMPORT_MAPPING # noqa: F401 import unused
from _compat_pickle import NAME_MAPPING # noqa: F401 import unused
class Pickler(zodbpickle.pickle.Pickler): class Pickler(zodbpickle.pickle.Pickler):
def __init__(self, f, protocol=None): def __init__(self, f, protocol=None):
...@@ -92,6 +93,7 @@ def PersistentPickler(persistent_id, *args, **kwargs): ...@@ -92,6 +93,7 @@ def PersistentPickler(persistent_id, *args, **kwargs):
p.persistent_id = persistent_id p.persistent_id = persistent_id
return p return p
def PersistentUnpickler(find_global, load_persistent, *args, **kwargs): def PersistentUnpickler(find_global, load_persistent, *args, **kwargs):
""" """
Returns a :class:`Unpickler` that will use the given `find_global` function Returns a :class:`Unpickler` that will use the given `find_global` function
...@@ -104,7 +106,8 @@ def PersistentUnpickler(find_global, load_persistent, *args, **kwargs): ...@@ -104,7 +106,8 @@ def PersistentUnpickler(find_global, load_persistent, *args, **kwargs):
if find_global is not None: if find_global is not None:
unpickler.find_global = find_global unpickler.find_global = find_global
try: try:
unpickler.find_class = find_global # PyPy, zodbpickle, the non-c-accelerated version # PyPy, zodbpickle, the non-c-accelerated version
unpickler.find_class = find_global
except AttributeError: except AttributeError:
pass pass
if load_persistent is not None: if load_persistent is not None:
...@@ -118,7 +121,7 @@ try: ...@@ -118,7 +121,7 @@ try:
from cStringIO import StringIO as BytesIO from cStringIO import StringIO as BytesIO
except ImportError: except ImportError:
# Python 3.x # Python 3.x
from io import BytesIO from io import BytesIO # noqa: F401 import unused
try: try:
...@@ -126,14 +129,15 @@ try: ...@@ -126,14 +129,15 @@ try:
from base64 import decodebytes, encodebytes from base64 import decodebytes, encodebytes
except ImportError: except ImportError:
# Python 2.x # Python 2.x
from base64 import decodestring as decodebytes, encodestring as encodebytes from base64 import decodestring as decodebytes # noqa: F401 import unused
from base64 import encodestring as encodebytes # noqa: F401 import unused
# Python 3.x: ``hasattr()`` swallows only AttributeError. # Python 3.x: ``hasattr()`` swallows only AttributeError.
def py2_hasattr(obj, name): def py2_hasattr(obj, name):
try: try:
getattr(obj, name) getattr(obj, name)
except: except: # noqa: E722 do not use bare 'except'
return False return False
return True return True
...@@ -151,9 +155,10 @@ else: ...@@ -151,9 +155,10 @@ else:
try: try:
TEXT = unicode TEXT = unicode
except NameError: #pragma NO COVER Py3k except NameError: # pragma NO COVER Py3k
TEXT = str TEXT = str
def ascii_bytes(x): def ascii_bytes(x):
if isinstance(x, TEXT): if isinstance(x, TEXT):
x = x.encode('ascii') x = x.encode('ascii')
......
...@@ -35,7 +35,6 @@ from ZODB._compat import BytesIO ...@@ -35,7 +35,6 @@ from ZODB._compat import BytesIO
from ZODB._compat import PersistentUnpickler from ZODB._compat import PersistentUnpickler
from ZODB._compat import decodebytes from ZODB._compat import decodebytes
from ZODB._compat import ascii_bytes from ZODB._compat import ascii_bytes
from ZODB._compat import INT_TYPES
from ZODB._compat import PY3 from ZODB._compat import PY3
...@@ -62,20 +61,21 @@ valid_modes = 'r', 'w', 'r+', 'a', 'c' ...@@ -62,20 +61,21 @@ valid_modes = 'r', 'w', 'r+', 'a', 'c'
# of a weakref when the weakref object dies at the same time # of a weakref when the weakref object dies at the same time
# as the object it refers to. In other words, this doesn't work: # as the object it refers to. In other words, this doesn't work:
# self._ref = weakref.ref(self, lambda ref: ...) # self._ref = weakref.ref(self, lambda ref: ...)
# because the function never gets called (https://bitbucket.org/pypy/pypy/issue/2030). # because the function never gets called
# (https://bitbucket.org/pypy/pypy/issue/2030).
# The Blob class used to use that pattern to clean up uncommitted # The Blob class used to use that pattern to clean up uncommitted
# files; now we use this module-level global (but still keep a # files; now we use this module-level global (but still keep a
# reference in the Blob in case we need premature cleanup). # reference in the Blob in case we need premature cleanup).
_blob_close_refs = [] _blob_close_refs = []
@zope.interface.implementer(ZODB.interfaces.IBlob) @zope.interface.implementer(ZODB.interfaces.IBlob)
class Blob(persistent.Persistent): class Blob(persistent.Persistent):
"""A BLOB supports efficient handling of large data within ZODB.""" """A BLOB supports efficient handling of large data within ZODB."""
_p_blob_uncommitted = None # Filename of the uncommitted (dirty) data _p_blob_uncommitted = None # Filename of the uncommitted (dirty) data
_p_blob_committed = None # Filename of the committed data _p_blob_committed = None # Filename of the committed data
_p_blob_ref = None # weakreference to self; also in _blob_close_refs _p_blob_ref = None # weakreference to self; also in _blob_close_refs
readers = writers = None readers = writers = None
...@@ -140,11 +140,10 @@ class Blob(persistent.Persistent): ...@@ -140,11 +140,10 @@ class Blob(persistent.Persistent):
if mode == 'c': if mode == 'c':
if (self._p_blob_uncommitted if (self._p_blob_uncommitted
or or
not self._p_blob_committed not self._p_blob_committed
or or
self._p_blob_committed.endswith(SAVEPOINT_SUFFIX) self._p_blob_committed.endswith(SAVEPOINT_SUFFIX)):
):
raise BlobError('Uncommitted changes') raise BlobError('Uncommitted changes')
return self._p_jar._storage.openCommittedBlobFile( return self._p_jar._storage.openCommittedBlobFile(
self._p_oid, self._p_serial) self._p_oid, self._p_serial)
...@@ -186,7 +185,7 @@ class Blob(persistent.Persistent): ...@@ -186,7 +185,7 @@ class Blob(persistent.Persistent):
if self._p_blob_uncommitted is None: if self._p_blob_uncommitted is None:
self._create_uncommitted_file() self._create_uncommitted_file()
result = BlobFile(self._p_blob_uncommitted, mode, self) result = BlobFile(self._p_blob_uncommitted, mode, self)
else: # 'r+' and 'a' else: # 'r+' and 'a'
if self._p_blob_uncommitted is None: if self._p_blob_uncommitted is None:
# Create a new working copy # Create a new working copy
self._create_uncommitted_file() self._create_uncommitted_file()
...@@ -214,11 +213,10 @@ class Blob(persistent.Persistent): ...@@ -214,11 +213,10 @@ class Blob(persistent.Persistent):
def committed(self): def committed(self):
if (self._p_blob_uncommitted if (self._p_blob_uncommitted
or or
not self._p_blob_committed not self._p_blob_committed
or or
self._p_blob_committed.endswith(SAVEPOINT_SUFFIX) self._p_blob_committed.endswith(SAVEPOINT_SUFFIX)):
):
raise BlobError('Uncommitted changes') raise BlobError('Uncommitted changes')
result = self._p_blob_committed result = self._p_blob_committed
...@@ -254,7 +252,7 @@ class Blob(persistent.Persistent): ...@@ -254,7 +252,7 @@ class Blob(persistent.Persistent):
try: try:
rename_or_copy_blob(filename, target, chmod=False) rename_or_copy_blob(filename, target, chmod=False)
except: except: # noqa: E722 do not use bare 'except'
# Recover from the failed consumption: First remove the file, it # Recover from the failed consumption: First remove the file, it
# might exist and mark the pointer to the uncommitted file. # might exist and mark the pointer to the uncommitted file.
self._p_blob_uncommitted = None self._p_blob_uncommitted = None
...@@ -317,6 +315,7 @@ class Blob(persistent.Persistent): ...@@ -317,6 +315,7 @@ class Blob(persistent.Persistent):
self._p_blob_uncommitted = self._p_blob_ref = None self._p_blob_uncommitted = self._p_blob_ref = None
return filename return filename
class BlobFile(file): class BlobFile(file):
"""A BlobFile that holds a file handle to actual blob data. """A BlobFile that holds a file handle to actual blob data.
...@@ -348,8 +347,10 @@ class BlobFile(file): ...@@ -348,8 +347,10 @@ class BlobFile(file):
# prohibit it on all versions. # prohibit it on all versions.
raise TypeError("Pickling a BlobFile is not allowed") raise TypeError("Pickling a BlobFile is not allowed")
_pid = str(os.getpid()) _pid = str(os.getpid())
def log(msg, level=logging.INFO, subsys=_pid, exc_info=False): def log(msg, level=logging.INFO, subsys=_pid, exc_info=False):
message = "(%s) %s" % (subsys, msg) message = "(%s) %s" % (subsys, msg)
logger.log(level, message, exc_info=exc_info) logger.log(level, message, exc_info=exc_info)
...@@ -394,8 +395,8 @@ class FilesystemHelper(object): ...@@ -394,8 +395,8 @@ class FilesystemHelper(object):
layout = layout_marker.read().strip() layout = layout_marker.read().strip()
if layout != self.layout_name: if layout != self.layout_name:
raise ValueError( raise ValueError(
"Directory layout `%s` selected for blob directory %s, but " "Directory layout `%s` selected for blob directory %s, but"
"marker found for layout `%s`" % " marker found for layout `%s`" %
(self.layout_name, self.base_dir, layout)) (self.layout_name, self.base_dir, layout))
def isSecure(self, path): def isSecure(self, path):
...@@ -541,6 +542,7 @@ class NoBlobsFileSystemHelper(object): ...@@ -541,6 +542,7 @@ class NoBlobsFileSystemHelper(object):
class BlobStorageError(Exception): class BlobStorageError(Exception):
"""The blob storage encountered an invalid state.""" """The blob storage encountered an invalid state."""
def auto_layout_select(path): def auto_layout_select(path):
# A heuristic to look at a path and determine which directory layout to # A heuristic to look at a path and determine which directory layout to
# use. # use.
...@@ -593,7 +595,7 @@ class BushyLayout(object): ...@@ -593,7 +595,7 @@ class BushyLayout(object):
directories = [b'0x' + hex_bytes[x:x+2] directories = [b'0x' + hex_bytes[x:x+2]
for x in range(0, 16, 2)] for x in range(0, 16, 2)]
if bytes is not str: # py3 if bytes is not str: # py3
sep_bytes = os.path.sep.encode('ascii') sep_bytes = os.path.sep.encode('ascii')
path_bytes = sep_bytes.join(directories) path_bytes = sep_bytes.join(directories)
return path_bytes.decode('ascii') return path_bytes.decode('ascii')
...@@ -618,8 +620,10 @@ class BushyLayout(object): ...@@ -618,8 +620,10 @@ class BushyLayout(object):
filename = "%s%s" % (utils.tid_repr(tid), BLOB_SUFFIX) filename = "%s%s" % (utils.tid_repr(tid), BLOB_SUFFIX)
return os.path.join(oid_path, filename) return os.path.join(oid_path, filename)
LAYOUTS['bushy'] = BushyLayout() LAYOUTS['bushy'] = BushyLayout()
class LawnLayout(BushyLayout): class LawnLayout(BushyLayout):
"""A shallow directory layout for blob directories. """A shallow directory layout for blob directories.
...@@ -640,8 +644,10 @@ class LawnLayout(BushyLayout): ...@@ -640,8 +644,10 @@ class LawnLayout(BushyLayout):
except (TypeError, binascii.Error): except (TypeError, binascii.Error):
raise ValueError('Not a valid OID path: `%s`' % path) raise ValueError('Not a valid OID path: `%s`' % path)
LAYOUTS['lawn'] = LawnLayout() LAYOUTS['lawn'] = LawnLayout()
class BlobStorageMixin(object): class BlobStorageMixin(object):
"""A mix-in to help storages support blobs.""" """A mix-in to help storages support blobs."""
...@@ -738,7 +744,6 @@ class BlobStorage(BlobStorageMixin): ...@@ -738,7 +744,6 @@ class BlobStorage(BlobStorageMixin):
"""A wrapper/proxy storage to support blobs. """A wrapper/proxy storage to support blobs.
""" """
def __init__(self, base_directory, storage, layout='automatic'): def __init__(self, base_directory, storage, layout='automatic'):
assert not ZODB.interfaces.IBlobStorage.providedBy(storage) assert not ZODB.interfaces.IBlobStorage.providedBy(storage)
self.__storage = storage self.__storage = storage
...@@ -780,8 +785,8 @@ class BlobStorage(BlobStorageMixin): ...@@ -780,8 +785,8 @@ class BlobStorage(BlobStorageMixin):
def tpc_abort(self, *arg, **kw): def tpc_abort(self, *arg, **kw):
# We need to override the base storage's abort instead of # We need to override the base storage's abort instead of
# providing an _abort method because methods found on the proxied object # providing an _abort method because methods found on the proxied
# aren't rebound to the proxy # object aren't rebound to the proxy
self.__storage.tpc_abort(*arg, **kw) self.__storage.tpc_abort(*arg, **kw)
self._blob_tpc_abort() self._blob_tpc_abort()
...@@ -814,7 +819,7 @@ class BlobStorage(BlobStorageMixin): ...@@ -814,7 +819,7 @@ class BlobStorage(BlobStorageMixin):
if exists: if exists:
files = os.listdir(oid_path) files = os.listdir(oid_path)
files.sort() files.sort()
latest = files[-1] # depends on ever-increasing tids latest = files[-1] # depends on ever-increasing tids
files.remove(latest) files.remove(latest)
for f in files: for f in files:
remove_committed(os.path.join(oid_path, f)) remove_committed(os.path.join(oid_path, f))
...@@ -905,7 +910,10 @@ class BlobStorage(BlobStorageMixin): ...@@ -905,7 +910,10 @@ class BlobStorage(BlobStorageMixin):
res = BlobStorage(base_dir, s) res = BlobStorage(base_dir, s)
return res return res
copied = logging.getLogger('ZODB.blob.copied').debug copied = logging.getLogger('ZODB.blob.copied').debug
def rename_or_copy_blob(f1, f2, chmod=True): def rename_or_copy_blob(f1, f2, chmod=True):
"""Try to rename f1 to f2, fallback to copy. """Try to rename f1 to f2, fallback to copy.
...@@ -926,6 +934,7 @@ def rename_or_copy_blob(f1, f2, chmod=True): ...@@ -926,6 +934,7 @@ def rename_or_copy_blob(f1, f2, chmod=True):
if chmod: if chmod:
set_not_writable(f2) set_not_writable(f2)
if sys.platform == 'win32': if sys.platform == 'win32':
# On Windows, you can't remove read-only files, so make the # On Windows, you can't remove read-only files, so make the
# file writable first. # file writable first.
...@@ -952,6 +961,7 @@ def find_global_Blob(module, class_): ...@@ -952,6 +961,7 @@ def find_global_Blob(module, class_):
if module == 'ZODB.blob' and class_ == 'Blob': if module == 'ZODB.blob' and class_ == 'Blob':
return Blob return Blob
def is_blob_record(record): def is_blob_record(record):
"""Check whether a database record is a blob record. """Check whether a database record is a blob record.
...@@ -960,7 +970,8 @@ def is_blob_record(record): ...@@ -960,7 +970,8 @@ def is_blob_record(record):
""" """
if record and (b'ZODB.blob' in record): if record and (b'ZODB.blob' in record):
unpickler = PersistentUnpickler(find_global_Blob, None, BytesIO(record)) unpickler = PersistentUnpickler(
find_global_Blob, None, BytesIO(record))
try: try:
return unpickler.load() is Blob return unpickler.load() is Blob
...@@ -971,6 +982,7 @@ def is_blob_record(record): ...@@ -971,6 +982,7 @@ def is_blob_record(record):
return False return False
def copyTransactionsFromTo(source, destination): def copyTransactionsFromTo(source, destination):
for trans in source.iterator(): for trans in source.iterator():
destination.tpc_begin(trans, trans.tid, trans.status) destination.tpc_begin(trans, trans.tid, trans.status)
...@@ -990,10 +1002,10 @@ def copyTransactionsFromTo(source, destination): ...@@ -990,10 +1002,10 @@ def copyTransactionsFromTo(source, destination):
with open(name, 'wb') as df: with open(name, 'wb') as df:
utils.cp(sf, df) utils.cp(sf, df)
destination.restoreBlob(record.oid, record.tid, record.data, destination.restoreBlob(record.oid, record.tid, record.data,
name, record.data_txn, trans) name, record.data_txn, trans)
else: else:
destination.restore(record.oid, record.tid, record.data, destination.restore(record.oid, record.tid, record.data,
'', record.data_txn, trans) '', record.data_txn, trans)
destination.tpc_vote(trans) destination.tpc_vote(trans)
destination.tpc_finish(trans) destination.tpc_finish(trans)
...@@ -1001,6 +1013,8 @@ def copyTransactionsFromTo(source, destination): ...@@ -1001,6 +1013,8 @@ def copyTransactionsFromTo(source, destination):
NO_WRITE = ~ (stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH) NO_WRITE = ~ (stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH)
READ_PERMS = stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH READ_PERMS = stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH
def set_not_writable(path): def set_not_writable(path):
perms = stat.S_IMODE(os.lstat(path).st_mode) perms = stat.S_IMODE(os.lstat(path).st_mode)
......
...@@ -25,6 +25,7 @@ from ZODB._compat import NAME_MAPPING ...@@ -25,6 +25,7 @@ from ZODB._compat import NAME_MAPPING
broken_cache = {} broken_cache = {}
@zope.interface.implementer(ZODB.interfaces.IBroken) @zope.interface.implementer(ZODB.interfaces.IBroken)
class Broken(object): class Broken(object):
"""Broken object base class """Broken object base class
...@@ -99,7 +100,6 @@ class Broken(object): ...@@ -99,7 +100,6 @@ class Broken(object):
>>> broken_cache.clear() >>> broken_cache.clear()
""" """
__Broken_state__ = __Broken_initargs__ = None __Broken_state__ = __Broken_initargs__ = None
__name__ = 'broken object' __name__ = 'broken object'
...@@ -131,6 +131,7 @@ class Broken(object): ...@@ -131,6 +131,7 @@ class Broken(object):
def __setattr__(self, name, value): def __setattr__(self, name, value):
raise BrokenModified("Can't change broken objects") raise BrokenModified("Can't change broken objects")
def find_global(modulename, globalname, def find_global(modulename, globalname,
# These are *not* optimizations. Callers can override these. # These are *not* optimizations. Callers can override these.
Broken=Broken, type=type, Broken=Broken, type=type,
...@@ -220,6 +221,7 @@ def find_global(modulename, globalname, ...@@ -220,6 +221,7 @@ def find_global(modulename, globalname,
broken_cache[(modulename, globalname)] = class_ broken_cache[(modulename, globalname)] = class_
return class_ return class_
def rebuild(modulename, globalname, *args): def rebuild(modulename, globalname, *args):
"""Recreate a broken object, possibly recreating the missing class """Recreate a broken object, possibly recreating the missing class
...@@ -257,10 +259,12 @@ def rebuild(modulename, globalname, *args): ...@@ -257,10 +259,12 @@ def rebuild(modulename, globalname, *args):
class_ = find_global(modulename, globalname) class_ = find_global(modulename, globalname)
return class_.__new__(class_, *args) return class_.__new__(class_, *args)
class BrokenModified(TypeError): class BrokenModified(TypeError):
"""Attempt to modify a broken object """Attempt to modify a broken object
""" """
class PersistentBroken(Broken, persistent.Persistent): class PersistentBroken(Broken, persistent.Persistent):
r"""Persistent broken objects r"""Persistent broken objects
...@@ -347,6 +351,7 @@ class PersistentBroken(Broken, persistent.Persistent): ...@@ -347,6 +351,7 @@ class PersistentBroken(Broken, persistent.Persistent):
def __getnewargs__(self): def __getnewargs__(self):
return self.__Broken_newargs__ return self.__Broken_newargs__
def persistentBroken(class_): def persistentBroken(class_):
try: try:
return class_.__dict__['__Broken_Persistent__'] return class_.__dict__['__Broken_Persistent__']
...@@ -356,5 +361,5 @@ def persistentBroken(class_): ...@@ -356,5 +361,5 @@ def persistentBroken(class_):
(PersistentBroken, class_), (PersistentBroken, class_),
{'__module__': class_.__module__}, {'__module__': class_.__module__},
) )
) )
return class_.__dict__['__Broken_Persistent__'] return class_.__dict__['__Broken_Persistent__']
...@@ -29,18 +29,21 @@ _db_schema = None ...@@ -29,18 +29,21 @@ _db_schema = None
s_schema_path = os.path.join(ZODB.__path__[0], "storage.xml") s_schema_path = os.path.join(ZODB.__path__[0], "storage.xml")
_s_schema = None _s_schema = None
def getDbSchema(): def getDbSchema():
global _db_schema global _db_schema
if _db_schema is None: if _db_schema is None:
_db_schema = ZConfig.loadSchema(db_schema_path) _db_schema = ZConfig.loadSchema(db_schema_path)
return _db_schema return _db_schema
def getStorageSchema(): def getStorageSchema():
global _s_schema global _s_schema
if _s_schema is None: if _s_schema is None:
_s_schema = ZConfig.loadSchema(s_schema_path) _s_schema = ZConfig.loadSchema(s_schema_path)
return _s_schema return _s_schema
def databaseFromString(s): def databaseFromString(s):
"""Create a database from a database-configuration string. """Create a database from a database-configuration string.
...@@ -56,6 +59,7 @@ def databaseFromString(s): ...@@ -56,6 +59,7 @@ def databaseFromString(s):
""" """
return databaseFromFile(StringIO(s)) return databaseFromFile(StringIO(s))
def databaseFromFile(f): def databaseFromFile(f):
"""Create a database from a file object that provides configuration. """Create a database from a file object that provides configuration.
...@@ -64,6 +68,7 @@ def databaseFromFile(f): ...@@ -64,6 +68,7 @@ def databaseFromFile(f):
config, handle = ZConfig.loadConfigFile(getDbSchema(), f) config, handle = ZConfig.loadConfigFile(getDbSchema(), f)
return databaseFromConfig(config.database) return databaseFromConfig(config.database)
def databaseFromURL(url): def databaseFromURL(url):
"""Load a database from URL (or file name) that provides configuration. """Load a database from URL (or file name) that provides configuration.
...@@ -72,6 +77,7 @@ def databaseFromURL(url): ...@@ -72,6 +77,7 @@ def databaseFromURL(url):
config, handler = ZConfig.loadConfig(getDbSchema(), url) config, handler = ZConfig.loadConfig(getDbSchema(), url)
return databaseFromConfig(config.database) return databaseFromConfig(config.database)
def databaseFromConfig(database_factories): def databaseFromConfig(database_factories):
databases = {} databases = {}
first = None first = None
...@@ -82,17 +88,20 @@ def databaseFromConfig(database_factories): ...@@ -82,17 +88,20 @@ def databaseFromConfig(database_factories):
return first return first
def storageFromString(s): def storageFromString(s):
"""Create a storage from a storage-configuration string. """Create a storage from a storage-configuration string.
""" """
return storageFromFile(StringIO(s)) return storageFromFile(StringIO(s))
def storageFromFile(f): def storageFromFile(f):
"""Create a storage from a file object providing storage-configuration. """Create a storage from a file object providing storage-configuration.
""" """
config, handle = ZConfig.loadConfigFile(getStorageSchema(), f) config, handle = ZConfig.loadConfigFile(getStorageSchema(), f)
return storageFromConfig(config.storage) return storageFromConfig(config.storage)
def storageFromURL(url): def storageFromURL(url):
"""\ """\
Create a storage from a URL (or file name) providing storage-configuration. Create a storage from a URL (or file name) providing storage-configuration.
...@@ -100,9 +109,11 @@ def storageFromURL(url): ...@@ -100,9 +109,11 @@ def storageFromURL(url):
config, handler = ZConfig.loadConfig(getStorageSchema(), url) config, handler = ZConfig.loadConfig(getStorageSchema(), url)
return storageFromConfig(config.storage) return storageFromConfig(config.storage)
def storageFromConfig(section): def storageFromConfig(section):
return section.open() return section.open()
class BaseConfig(object): class BaseConfig(object):
"""Object representing a configured storage or database. """Object representing a configured storage or database.
...@@ -124,6 +135,7 @@ class BaseConfig(object): ...@@ -124,6 +135,7 @@ class BaseConfig(object):
"""Open and return the storage object.""" """Open and return the storage object."""
raise NotImplementedError raise NotImplementedError
class ZODBDatabase(BaseConfig): class ZODBDatabase(BaseConfig):
def open(self, databases=None): def open(self, databases=None):
...@@ -150,21 +162,23 @@ class ZODBDatabase(BaseConfig): ...@@ -150,21 +162,23 @@ class ZODBDatabase(BaseConfig):
cache_size_bytes=section.cache_size_bytes, cache_size_bytes=section.cache_size_bytes,
historical_pool_size=section.historical_pool_size, historical_pool_size=section.historical_pool_size,
historical_cache_size=section.historical_cache_size, historical_cache_size=section.historical_cache_size,
historical_cache_size_bytes=section.historical_cache_size_bytes, historical_cache_size_bytes=section.historical_cache_size_bytes, # noqa: E501 line too long
historical_timeout=section.historical_timeout, historical_timeout=section.historical_timeout,
database_name=section.database_name or self.name or '', database_name=section.database_name or self.name or '',
databases=databases, databases=databases,
**options) **options)
except: except: # noqa: E722 do not use bare 'except'
storage.close() storage.close()
raise raise
class MappingStorage(BaseConfig): class MappingStorage(BaseConfig):
def open(self): def open(self):
from ZODB.MappingStorage import MappingStorage from ZODB.MappingStorage import MappingStorage
return MappingStorage(self.config.name) return MappingStorage(self.config.name)
class DemoStorage(BaseConfig): class DemoStorage(BaseConfig):
def open(self): def open(self):
...@@ -181,6 +195,7 @@ class DemoStorage(BaseConfig): ...@@ -181,6 +195,7 @@ class DemoStorage(BaseConfig):
from ZODB.DemoStorage import DemoStorage from ZODB.DemoStorage import DemoStorage
return DemoStorage(self.config.name, base=base, changes=changes) return DemoStorage(self.config.name, base=base, changes=changes)
class FileStorage(BaseConfig): class FileStorage(BaseConfig):
def open(self): def open(self):
...@@ -206,6 +221,7 @@ class FileStorage(BaseConfig): ...@@ -206,6 +221,7 @@ class FileStorage(BaseConfig):
return FileStorage(config.path, **options) return FileStorage(config.path, **options)
class BlobStorage(BaseConfig): class BlobStorage(BaseConfig):
def open(self): def open(self):
...@@ -225,7 +241,8 @@ class ZEOClient(BaseConfig): ...@@ -225,7 +241,8 @@ class ZEOClient(BaseConfig):
if self.config.blob_cache_size is not None: if self.config.blob_cache_size is not None:
options['blob_cache_size'] = self.config.blob_cache_size options['blob_cache_size'] = self.config.blob_cache_size
if self.config.blob_cache_size_check is not None: if self.config.blob_cache_size_check is not None:
options['blob_cache_size_check'] = self.config.blob_cache_size_check options['blob_cache_size_check'] = (
self.config.blob_cache_size_check)
if self.config.client_label is not None: if self.config.client_label is not None:
options['client_label'] = self.config.client_label options['client_label'] = self.config.client_label
...@@ -249,6 +266,7 @@ class ZEOClient(BaseConfig): ...@@ -249,6 +266,7 @@ class ZEOClient(BaseConfig):
realm=self.config.realm, realm=self.config.realm,
**options) **options)
class BDBStorage(BaseConfig): class BDBStorage(BaseConfig):
def open(self): def open(self):
...@@ -261,12 +279,14 @@ class BDBStorage(BaseConfig): ...@@ -261,12 +279,14 @@ class BDBStorage(BaseConfig):
setattr(bconf, name, getattr(self.config, name)) setattr(bconf, name, getattr(self.config, name))
return storageclass(self.config.envdir, config=bconf) return storageclass(self.config.envdir, config=bconf)
class BDBMinimalStorage(BDBStorage): class BDBMinimalStorage(BDBStorage):
def get_storageclass(self): def get_storageclass(self):
import BDBStorage.BDBMinimalStorage import BDBStorage.BDBMinimalStorage
return BDBStorage.BDBMinimalStorage.BDBMinimalStorage return BDBStorage.BDBMinimalStorage.BDBMinimalStorage
class BDBFullStorage(BDBStorage): class BDBFullStorage(BDBStorage):
def get_storageclass(self): def get_storageclass(self):
......
...@@ -14,21 +14,29 @@ ...@@ -14,21 +14,29 @@
import persistent.mapping import persistent.mapping
class fixer(object): class fixer(object):
def __of__(self, parent): def __of__(self, parent):
def __setstate__(state, self=parent): def __setstate__(state, self=parent):
self._container=state self._container = state
del self.__setstate__ del self.__setstate__
return __setstate__ return __setstate__
fixer=fixer()
class hack(object): pass fixer = fixer()
hack=hack()
class hack(object):
pass
hack = hack()
def __basicnew__(): def __basicnew__():
r=persistent.mapping.PersistentMapping() r = persistent.mapping.PersistentMapping()
r.__setstate__=fixer r.__setstate__ = fixer
return r return r
hack.__basicnew__=__basicnew__
hack.__basicnew__ = __basicnew__
...@@ -14,5 +14,5 @@ ...@@ -14,5 +14,5 @@
try: try:
from zope.event import notify from zope.event import notify
except ImportError: except ImportError:
notify = lambda event: None def notify(event):
return None
...@@ -55,17 +55,21 @@ from ZODB._compat import _protocol ...@@ -55,17 +55,21 @@ from ZODB._compat import _protocol
def num2str(n): def num2str(n):
return struct.pack(">Q", n)[2:] return struct.pack(">Q", n)[2:]
def str2num(s): def str2num(s):
return struct.unpack(">Q", b"\000\000" + s)[0] return struct.unpack(">Q", b"\000\000" + s)[0]
def prefix_plus_one(s): def prefix_plus_one(s):
num = str2num(s) num = str2num(s)
return num2str(num + 1) return num2str(num + 1)
def prefix_minus_one(s): def prefix_minus_one(s):
num = str2num(s) num = str2num(s)
return num2str(num - 1) return num2str(num - 1)
def ensure_bytes(s): def ensure_bytes(s):
# on Python 3 we might pickle bytes and unpickle unicode strings # on Python 3 we might pickle bytes and unpickle unicode strings
return s.encode('ascii') if not isinstance(s, bytes) else s return s.encode('ascii') if not isinstance(s, bytes) else s
...@@ -80,11 +84,11 @@ class fsIndex(object): ...@@ -80,11 +84,11 @@ class fsIndex(object):
def __getstate__(self): def __getstate__(self):
return dict( return dict(
state_version = 1, state_version=1,
_data = [(k, v.toString()) _data=[(k, v.toString())
for (k, v) in six.iteritems(self._data) for (k, v) in six.iteritems(self._data)
] ]
) )
def __setstate__(self, state): def __setstate__(self, state):
version = state.pop('state_version', 0) version = state.pop('state_version', 0)
...@@ -96,13 +100,13 @@ class fsIndex(object): ...@@ -96,13 +100,13 @@ class fsIndex(object):
self._data = OOBTree([ self._data = OOBTree([
(ensure_bytes(k), v) (ensure_bytes(k), v)
for (k, v) in self._data.items() for (k, v) in self._data.items()
]) ])
def _setstate_1(self, state): def _setstate_1(self, state):
self._data = OOBTree([ self._data = OOBTree([
(ensure_bytes(k), fsBucket().fromString(ensure_bytes(v))) (ensure_bytes(k), fsBucket().fromString(ensure_bytes(v)))
for (k, v) in state['_data'] for (k, v) in state['_data']
]) ])
def __getitem__(self, key): def __getitem__(self, key):
assert isinstance(key, bytes) assert isinstance(key, bytes)
...@@ -246,7 +250,7 @@ class fsIndex(object): ...@@ -246,7 +250,7 @@ class fsIndex(object):
else: else:
try: try:
smallest_suffix = tree.minKey(key[6:]) smallest_suffix = tree.minKey(key[6:])
except ValueError: # 'empty tree' (no suffix >= arg) except ValueError: # 'empty tree' (no suffix >= arg)
next_prefix = prefix_plus_one(smallest_prefix) next_prefix = prefix_plus_one(smallest_prefix)
smallest_prefix = self._data.minKey(next_prefix) smallest_prefix = self._data.minKey(next_prefix)
tree = self._data[smallest_prefix] tree = self._data[smallest_prefix]
...@@ -270,7 +274,7 @@ class fsIndex(object): ...@@ -270,7 +274,7 @@ class fsIndex(object):
else: else:
try: try:
biggest_suffix = tree.maxKey(key[6:]) biggest_suffix = tree.maxKey(key[6:])
except ValueError: # 'empty tree' (no suffix <= arg) except ValueError: # 'empty tree' (no suffix <= arg)
next_prefix = prefix_minus_one(biggest_prefix) next_prefix = prefix_minus_one(biggest_prefix)
biggest_prefix = self._data.maxKey(next_prefix) biggest_prefix = self._data.maxKey(next_prefix)
tree = self._data[biggest_prefix] tree = self._data[biggest_prefix]
......
...@@ -94,12 +94,15 @@ def die(mess='', show_docstring=False): ...@@ -94,12 +94,15 @@ def die(mess='', show_docstring=False):
print(__doc__ % sys.argv[0], file=sys.stderr) print(__doc__ % sys.argv[0], file=sys.stderr)
sys.exit(1) sys.exit(1)
class ErrorFound(Exception): class ErrorFound(Exception):
pass pass
def error(mess, *args): def error(mess, *args):
raise ErrorFound(mess % args) raise ErrorFound(mess % args)
def read_txn_header(f, pos, file_size, outp, ltid): def read_txn_header(f, pos, file_size, outp, ltid):
# Read the transaction record # Read the transaction record
f.seek(pos) f.seek(pos)
...@@ -107,7 +110,7 @@ def read_txn_header(f, pos, file_size, outp, ltid): ...@@ -107,7 +110,7 @@ def read_txn_header(f, pos, file_size, outp, ltid):
if len(h) < 23: if len(h) < 23:
raise EOFError raise EOFError
tid, stl, status, ul, dl, el = unpack(">8s8scHHH",h) tid, stl, status, ul, dl, el = unpack(">8s8scHHH", h)
status = as_text(status) status = as_text(status)
tl = u64(stl) tl = u64(stl)
...@@ -157,6 +160,7 @@ def read_txn_header(f, pos, file_size, outp, ltid): ...@@ -157,6 +160,7 @@ def read_txn_header(f, pos, file_size, outp, ltid):
return pos, result, tid return pos, result, tid
def truncate(f, pos, file_size, outp): def truncate(f, pos, file_size, outp):
"""Copy data from pos to end of f to a .trNNN file.""" """Copy data from pos to end of f to a .trNNN file."""
...@@ -176,6 +180,7 @@ def truncate(f, pos, file_size, outp): ...@@ -176,6 +180,7 @@ def truncate(f, pos, file_size, outp):
f.seek(pos) f.seek(pos)
tr.close() tr.close()
def copy(src, dst, n): def copy(src, dst, n):
while n: while n:
buf = src.read(8096) buf = src.read(8096)
...@@ -186,6 +191,7 @@ def copy(src, dst, n): ...@@ -186,6 +191,7 @@ def copy(src, dst, n):
dst.write(buf) dst.write(buf)
n -= len(buf) n -= len(buf)
def scan(f, pos): def scan(f, pos):
"""Return a potential transaction location following pos in f. """Return a potential transaction location following pos in f.
...@@ -206,20 +212,21 @@ def scan(f, pos): ...@@ -206,20 +212,21 @@ def scan(f, pos):
s = 0 s = 0
while 1: while 1:
l = data.find(b".", s) l_ = data.find(b".", s)
if l < 0: if l_ < 0:
pos += len(data) pos += len(data)
break break
# If we are less than 8 bytes from the end of the # If we are less than 8 bytes from the end of the
# string, we need to read more data. # string, we need to read more data.
s = l + 1 s = l_ + 1
if s > len(data) - 8: if s > len(data) - 8:
pos += l pos += l_
break break
tl = u64(data[s:s+8]) tl = u64(data[s:s+8])
if tl < pos: if tl < pos:
return pos + s + 8 return pos + s + 8
def iprogress(i): def iprogress(i):
if i % 2: if i % 2:
print(".", end=' ') print(".", end=' ')
...@@ -227,10 +234,12 @@ def iprogress(i): ...@@ -227,10 +234,12 @@ def iprogress(i):
print((i/2) % 10, end=' ') print((i/2) % 10, end=' ')
sys.stdout.flush() sys.stdout.flush()
def progress(p): def progress(p):
for i in range(p): for i in range(p):
iprogress(i) iprogress(i)
def main(): def main():
try: try:
opts, args = getopt.getopt(sys.argv[1:], "fv:pP:") opts, args = getopt.getopt(sys.argv[1:], "fv:pP:")
...@@ -256,6 +265,7 @@ def main(): ...@@ -256,6 +265,7 @@ def main():
recover(inp, outp, verbose, partial, force, pack) recover(inp, outp, verbose, partial, force, pack)
def recover(inp, outp, verbose=0, partial=False, force=False, pack=None): def recover(inp, outp, verbose=0, partial=False, force=False, pack=None):
print("Recovering", inp, "into", outp) print("Recovering", inp, "into", outp)
...@@ -266,7 +276,7 @@ def recover(inp, outp, verbose=0, partial=False, force=False, pack=None): ...@@ -266,7 +276,7 @@ def recover(inp, outp, verbose=0, partial=False, force=False, pack=None):
if f.read(4) != ZODB.FileStorage.packed_version: if f.read(4) != ZODB.FileStorage.packed_version:
die("input is not a file storage") die("input is not a file storage")
f.seek(0,2) f.seek(0, 2)
file_size = f.tell() file_size = f.tell()
ofs = ZODB.FileStorage.FileStorage(outp, create=1) ofs = ZODB.FileStorage.FileStorage(outp, create=1)
...@@ -332,11 +342,11 @@ def recover(inp, outp, verbose=0, partial=False, force=False, pack=None): ...@@ -332,11 +342,11 @@ def recover(inp, outp, verbose=0, partial=False, force=False, pack=None):
for r in txn: for r in txn:
if verbose > 1: if verbose > 1:
if r.data is None: if r.data is None:
l = "bp" l_ = "bp"
else: else:
l = len(r.data) l_ = len(r.data)
print("%7d %s %s" % (u64(r.oid), l)) print("%7d %s" % (u64(r.oid), l_))
ofs.restore(r.oid, r.tid, r.data, '', r.data_txn, ofs.restore(r.oid, r.tid, r.data, '', r.data_txn,
txn) txn)
nrec += 1 nrec += 1
...@@ -370,7 +380,6 @@ def recover(inp, outp, verbose=0, partial=False, force=False, pack=None): ...@@ -370,7 +380,6 @@ def recover(inp, outp, verbose=0, partial=False, force=False, pack=None):
prog1 = prog1 + 1 prog1 = prog1 + 1
iprogress(prog1) iprogress(prog1)
bad = file_size - undone - ofs._pos bad = file_size - undone - ofs._pos
print("\n%s bytes removed during recovery" % bad) print("\n%s bytes removed during recovery" % bad)
...@@ -385,5 +394,6 @@ def recover(inp, outp, verbose=0, partial=False, force=False, pack=None): ...@@ -385,5 +394,6 @@ def recover(inp, outp, verbose=0, partial=False, force=False, pack=None):
ofs.close() ofs.close()
f.close() f.close()
if __name__ == "__main__": if __name__ == "__main__":
main() main()
...@@ -100,6 +100,7 @@ class TxnHeader(object): ...@@ -100,6 +100,7 @@ class TxnHeader(object):
tlen = u64(self._file.read(8)) tlen = u64(self._file.read(8))
return TxnHeader(self._file, self._pos - (tlen + 8)) return TxnHeader(self._file, self._pos - (tlen + 8))
class DataHeader(object): class DataHeader(object):
"""Object representing a data record header. """Object representing a data record header.
...@@ -111,7 +112,7 @@ class DataHeader(object): ...@@ -111,7 +112,7 @@ class DataHeader(object):
txn_pos 24-32 position of txn header txn_pos 24-32 position of txn header
version_len 32-34 length of version (always 0) version_len 32-34 length of version (always 0)
data_len 34-42 length of data data_len 34-42 length of data
""" """
def __init__(self, file, pos): def __init__(self, file, pos):
...@@ -129,15 +130,16 @@ class DataHeader(object): ...@@ -129,15 +130,16 @@ class DataHeader(object):
self.prev_rec_pos = u64(prev_rec_pos) self.prev_rec_pos = u64(prev_rec_pos)
self.txn_pos = u64(txn_pos) self.txn_pos = u64(txn_pos)
self.data_len = u64(data_len) self.data_len = u64(data_len)
def next_offset(self): def next_offset(self):
"""Return offset of next record.""" """Return offset of next record."""
off = self._pos + self.data_len off = self._pos + self.data_len
off += DATA_HDR_LEN off += DATA_HDR_LEN
if self.data_len == 0: if self.data_len == 0:
off += 8 # backpointer off += 8 # backpointer
return off return off
def prev_txn(f): def prev_txn(f):
"""Return transaction located before current file position.""" """Return transaction located before current file position."""
f.seek(-8, 1) f.seek(-8, 1)
......
...@@ -267,6 +267,7 @@ class IConnection(Interface): ...@@ -267,6 +267,7 @@ class IConnection(Interface):
separate object. separate object.
""" """
class IStorageWrapper(Interface): class IStorageWrapper(Interface):
"""Storage wrapper interface """Storage wrapper interface
...@@ -296,7 +297,7 @@ class IStorageWrapper(Interface): ...@@ -296,7 +297,7 @@ class IStorageWrapper(Interface):
This interface may be implemented by storage adapters or other This interface may be implemented by storage adapters or other
intermediaries. For example, a storage adapter that provides intermediaries. For example, a storage adapter that provides
encryption and/or compresssion will apply record transformations encryption and/or compression will apply record transformations
in it's references method. in it's references method.
""" """
...@@ -343,7 +344,8 @@ class IStorageWrapper(Interface): ...@@ -343,7 +344,8 @@ class IStorageWrapper(Interface):
"""Return untransformed data """Return untransformed data
""" """
IStorageDB = IStorageWrapper # for backward compatibility
IStorageDB = IStorageWrapper # for backward compatibility
class IDatabase(IStorageDB): class IDatabase(IStorageDB):
...@@ -371,7 +373,6 @@ class IDatabase(IStorageDB): ...@@ -371,7 +373,6 @@ class IDatabase(IStorageDB):
this attribute. this attribute.
""") """)
def open(transaction_manager=None, serial=''): def open(transaction_manager=None, serial=''):
"""Return an IConnection object for use by application code. """Return an IConnection object for use by application code.
...@@ -421,7 +422,6 @@ class IDatabase(IStorageDB): ...@@ -421,7 +422,6 @@ class IDatabase(IStorageDB):
also included if they don't conflict with the keys above. also included if they don't conflict with the keys above.
""" """
def pack(t=None, days=0): def pack(t=None, days=0):
"""Pack the storage, deleting unused object revisions. """Pack the storage, deleting unused object revisions.
...@@ -433,7 +433,7 @@ class IDatabase(IStorageDB): ...@@ -433,7 +433,7 @@ class IDatabase(IStorageDB):
usually an expensive operation. usually an expensive operation.
There are two optional arguments that can be used to set the There are two optional arguments that can be used to set the
pack time: t, pack time in seconds since the epcoh, and days, pack time: t, pack time in seconds since the epoch, and days,
the number of days to subtract from t or from the current the number of days to subtract from t or from the current
time if t is not specified. time if t is not specified.
""" """
...@@ -539,6 +539,7 @@ class IDatabase(IStorageDB): ...@@ -539,6 +539,7 @@ class IDatabase(IStorageDB):
should also close all the Connections. should also close all the Connections.
""" """
class IStorageTransactionMetaData(Interface): class IStorageTransactionMetaData(Interface):
"""Provide storage transaction meta data. """Provide storage transaction meta data.
...@@ -628,13 +629,13 @@ class IStorage(Interface): ...@@ -628,13 +629,13 @@ class IStorage(Interface):
The format and interpretation of this name is storage The format and interpretation of this name is storage
dependent. It could be a file name, a database name, etc.. dependent. It could be a file name, a database name, etc..
This is used soley for informational purposes. This is used solely for informational purposes.
""" """
def getSize(): def getSize():
"""An approximate size of the database, in bytes. """An approximate size of the database, in bytes.
This is used soley for informational purposes. This is used solely for informational purposes.
""" """
def history(oid, size=1): def history(oid, size=1):
...@@ -660,7 +661,7 @@ class IStorage(Interface): ...@@ -660,7 +661,7 @@ class IStorage(Interface):
user_name user_name
The bytes user identifier, if any (or an empty string) of the The bytes user identifier, if any (or an empty string) of the
user on whos behalf the revision was committed. user on whose behalf the revision was committed.
description description
The bytes transaction description for the transaction that The bytes transaction description for the transaction that
...@@ -704,7 +705,7 @@ class IStorage(Interface): ...@@ -704,7 +705,7 @@ class IStorage(Interface):
def __len__(): def __len__():
"""The approximate number of objects in the storage """The approximate number of objects in the storage
This is used soley for informational purposes. This is used solely for informational purposes.
""" """
def loadBefore(oid, tid): def loadBefore(oid, tid):
...@@ -821,7 +822,7 @@ class IStorage(Interface): ...@@ -821,7 +822,7 @@ class IStorage(Interface):
This call is ignored is the storage is not participating in This call is ignored is the storage is not participating in
two-phase commit or if the given transaction is not the same two-phase commit or if the given transaction is not the same
as the transaction the storage is commiting. as the transaction the storage is committing.
""" """
def tpc_begin(transaction): def tpc_begin(transaction):
...@@ -837,7 +838,7 @@ class IStorage(Interface): ...@@ -837,7 +838,7 @@ class IStorage(Interface):
current transaction ends (commits or aborts). current transaction ends (commits or aborts).
""" """
def tpc_finish(transaction, func = lambda tid: None): def tpc_finish(transaction, func=lambda tid: None):
"""Finish the transaction, making any transaction changes permanent. """Finish the transaction, making any transaction changes permanent.
Changes must be made permanent at this point. Changes must be made permanent at this point.
...@@ -863,7 +864,7 @@ class IStorage(Interface): ...@@ -863,7 +864,7 @@ class IStorage(Interface):
The argument is the same object passed to tpc_begin. The argument is the same object passed to tpc_begin.
This call raises a StorageTransactionError if the storage This call raises a StorageTransactionError if the storage
isn't participating in two-phase commit or if it is commiting isn't participating in two-phase commit or if it is committing
a different transaction. a different transaction.
If a transaction can be committed by a storage, then the If a transaction can be committed by a storage, then the
...@@ -901,7 +902,7 @@ class IMultiCommitStorage(IStorage): ...@@ -901,7 +902,7 @@ class IMultiCommitStorage(IStorage):
the return value is always None. the return value is always None.
""" """
def tpc_finish(transaction, func = lambda tid: None): def tpc_finish(transaction, func=lambda tid: None):
"""Finish the transaction, making any transaction changes permanent. """Finish the transaction, making any transaction changes permanent.
See IStorage.store. For objects implementing this interface, See IStorage.store. For objects implementing this interface,
...@@ -954,7 +955,6 @@ class IStorageRestoreable(IStorage): ...@@ -954,7 +955,6 @@ class IStorageRestoreable(IStorage):
# including the existing FileStorage implementation), that # including the existing FileStorage implementation), that
# failed to take into account records after the pack time. # failed to take into account records after the pack time.
def restore(oid, serial, data, version, prev_txn, transaction): def restore(oid, serial, data, version, prev_txn, transaction):
"""Write data already committed in a separate database """Write data already committed in a separate database
...@@ -996,6 +996,7 @@ class IStorageRecordInformation(Interface): ...@@ -996,6 +996,7 @@ class IStorageRecordInformation(Interface):
data = Attribute("The data record, bytes") data = Attribute("The data record, bytes")
data_txn = Attribute("The previous transaction id, bytes") data_txn = Attribute("The previous transaction id, bytes")
class IStorageTransactionInformation(IStorageTransactionMetaData): class IStorageTransactionInformation(IStorageTransactionMetaData):
"""Provide information about a storage transaction. """Provide information about a storage transaction.
...@@ -1003,7 +1004,7 @@ class IStorageTransactionInformation(IStorageTransactionMetaData): ...@@ -1003,7 +1004,7 @@ class IStorageTransactionInformation(IStorageTransactionMetaData):
Note that this may contain a status field used by FileStorage to Note that this may contain a status field used by FileStorage to
support packing. At some point, this will go away when FileStorage support packing. At some point, this will go away when FileStorage
has a better pack algoritm. has a better pack algorithm.
""" """
tid = Attribute("Transaction id") tid = Attribute("Transaction id")
...@@ -1034,6 +1035,7 @@ class IStorageIteration(Interface): ...@@ -1034,6 +1035,7 @@ class IStorageIteration(Interface):
""" """
class IStorageUndoable(IStorage): class IStorageUndoable(IStorage):
"""A storage supporting transactional undo. """A storage supporting transactional undo.
""" """
...@@ -1245,6 +1247,7 @@ class IMVCCStorage(IStorage): ...@@ -1245,6 +1247,7 @@ class IMVCCStorage(IStorage):
A POSKeyError is raised if there is no record for the object id. A POSKeyError is raised if there is no record for the object id.
""" """
class IMVCCPrefetchStorage(IMVCCStorage): class IMVCCPrefetchStorage(IMVCCStorage):
def prefetch(oids): def prefetch(oids):
...@@ -1254,6 +1257,7 @@ class IMVCCPrefetchStorage(IMVCCStorage): ...@@ -1254,6 +1257,7 @@ class IMVCCPrefetchStorage(IMVCCStorage):
more than once. more than once.
""" """
class IMVCCAfterCompletionStorage(IMVCCStorage): class IMVCCAfterCompletionStorage(IMVCCStorage):
def afterCompletion(): def afterCompletion():
...@@ -1264,6 +1268,7 @@ class IMVCCAfterCompletionStorage(IMVCCStorage): ...@@ -1264,6 +1268,7 @@ class IMVCCAfterCompletionStorage(IMVCCStorage):
See ``transaction.interfaces.ISynchronizer.afterCompletion``. See ``transaction.interfaces.ISynchronizer.afterCompletion``.
""" """
class IStorageCurrentRecordIteration(IStorage): class IStorageCurrentRecordIteration(IStorage):
def record_iternext(next=None): def record_iternext(next=None):
...@@ -1271,6 +1276,7 @@ class IStorageCurrentRecordIteration(IStorage): ...@@ -1271,6 +1276,7 @@ class IStorageCurrentRecordIteration(IStorage):
Use like this: Use like this:
>>> storage = ...
>>> next = None >>> next = None
>>> while 1: >>> while 1:
... oid, tid, data, next = storage.record_iternext(next) ... oid, tid, data, next = storage.record_iternext(next)
...@@ -1280,24 +1286,26 @@ class IStorageCurrentRecordIteration(IStorage): ...@@ -1280,24 +1286,26 @@ class IStorageCurrentRecordIteration(IStorage):
""" """
class IExternalGC(IStorage): class IExternalGC(IStorage):
def deleteObject(oid, serial, transaction): def deleteObject(oid, serial, transaction):
"""Mark an object as deleted """Mark an object as deleted
This method marks an object as deleted via a new object This method marks an object as deleted via a new object
revision. Subsequent attempts to load current data for the revision. Subsequent attempts to load current data for the
object will fail with a POSKeyError, but loads for object will fail with a POSKeyError, but loads for
non-current data will suceed if there are previous non-current data will succeed if there are previous
non-delete records. The object will be removed from the non-delete records. The object will be removed from the
storage when all not-delete records are removed. storage when all not-delete records are removed.
The serial argument must match the most recently committed The serial argument must match the most recently committed
serial for the object. This is a seat belt. serial for the object. This is a seat belt.
This method can only be called in the first phase of 2-phase
commit.
"""
This method can only be called in the first phase of 2-phase
commit.
"""
class ReadVerifyingStorage(IStorage): class ReadVerifyingStorage(IStorage):
...@@ -1315,6 +1323,7 @@ class ReadVerifyingStorage(IStorage): ...@@ -1315,6 +1323,7 @@ class ReadVerifyingStorage(IStorage):
through the end of the transaction. through the end of the transaction.
""" """
class IBlob(Interface): class IBlob(Interface):
"""A BLOB supports efficient handling of large data within ZODB.""" """A BLOB supports efficient handling of large data within ZODB."""
...@@ -1325,7 +1334,7 @@ class IBlob(Interface): ...@@ -1325,7 +1334,7 @@ class IBlob(Interface):
mode: Mode to open the file with. Possible values: r,w,r+,a,c mode: Mode to open the file with. Possible values: r,w,r+,a,c
The mode 'c' is similar to 'r', except that an orinary file The mode 'c' is similar to 'r', except that an ordinary file
object is returned and may be used in a separate transaction object is returned and may be used in a separate transaction
and after the blob's database connection has been closed. and after the blob's database connection has been closed.
...@@ -1335,8 +1344,8 @@ class IBlob(Interface): ...@@ -1335,8 +1344,8 @@ class IBlob(Interface):
"""Return a file name for committed data. """Return a file name for committed data.
The returned file name may be opened for reading or handed to The returned file name may be opened for reading or handed to
other processes for reading. The file name isn't guarenteed other processes for reading. The file name isn't guaranteed
to be valid indefinately. The file may be removed in the to be valid indefinitely. The file may be removed in the
future as a result of garbage collection depending on system future as a result of garbage collection depending on system
configuration. configuration.
...@@ -1412,6 +1421,7 @@ class IBlobStorage(Interface): ...@@ -1412,6 +1421,7 @@ class IBlobStorage(Interface):
If Blobs use this, then commits can be performed with a simple rename. If Blobs use this, then commits can be performed with a simple rename.
""" """
class IBlobStorageRestoreable(IBlobStorage, IStorageRestoreable): class IBlobStorageRestoreable(IBlobStorage, IStorageRestoreable):
def restoreBlob(oid, serial, data, blobfilename, prev_txn, transaction): def restoreBlob(oid, serial, data, blobfilename, prev_txn, transaction):
...@@ -1446,6 +1456,7 @@ class IBroken(Interface): ...@@ -1446,6 +1456,7 @@ class IBroken(Interface):
__Broken_initargs__ = Attribute("Arguments passed to __init__.") __Broken_initargs__ = Attribute("Arguments passed to __init__.")
__Broken_state__ = Attribute("Value passed to __setstate__.") __Broken_state__ = Attribute("Value passed to __setstate__.")
class BlobError(Exception): class BlobError(Exception):
pass pass
......
...@@ -12,6 +12,7 @@ import zope.interface ...@@ -12,6 +12,7 @@ import zope.interface
from . import interfaces, serialize, POSException from . import interfaces, serialize, POSException
from .utils import p64, u64, Lock, oid_repr, tid_repr from .utils import p64, u64, Lock, oid_repr, tid_repr
class Base(object): class Base(object):
_copy_methods = ( _copy_methods = (
...@@ -19,7 +20,7 @@ class Base(object): ...@@ -19,7 +20,7 @@ class Base(object):
'loadBlob', 'openCommittedBlobFile', 'loadBlob', 'openCommittedBlobFile',
'isReadOnly', 'supportsUndo', 'undoLog', 'undoInfo', 'isReadOnly', 'supportsUndo', 'undoLog', 'undoInfo',
'temporaryDirectory', 'temporaryDirectory',
) )
def __init__(self, storage): def __init__(self, storage):
self._storage = storage self._storage = storage
...@@ -37,6 +38,7 @@ class Base(object): ...@@ -37,6 +38,7 @@ class Base(object):
def __len__(self): def __len__(self):
return len(self._storage) return len(self._storage)
class MVCCAdapter(Base): class MVCCAdapter(Base):
def __init__(self, storage): def __init__(self, storage):
...@@ -63,6 +65,7 @@ class MVCCAdapter(Base): ...@@ -63,6 +65,7 @@ class MVCCAdapter(Base):
self._instances.remove(instance) self._instances.remove(instance)
closed = False closed = False
def close(self): def close(self):
if not self.closed: if not self.closed:
self.closed = True self.closed = True
...@@ -92,14 +95,15 @@ class MVCCAdapter(Base): ...@@ -92,14 +95,15 @@ class MVCCAdapter(Base):
def pack(self, pack_time, referencesf): def pack(self, pack_time, referencesf):
return self._storage.pack(pack_time, referencesf) return self._storage.pack(pack_time, referencesf)
class MVCCAdapterInstance(Base): class MVCCAdapterInstance(Base):
_copy_methods = Base._copy_methods + ( _copy_methods = Base._copy_methods + (
'loadSerial', 'new_oid', 'tpc_vote', 'loadSerial', 'new_oid', 'tpc_vote',
'checkCurrentSerialInTransaction', 'tpc_abort', 'checkCurrentSerialInTransaction', 'tpc_abort',
) )
_start = None # Transaction start time _start = None # Transaction start time
_ltid = b'' # Last storage transaction id _ltid = b'' # Last storage transaction id
def __init__(self, base): def __init__(self, base):
...@@ -107,7 +111,7 @@ class MVCCAdapterInstance(Base): ...@@ -107,7 +111,7 @@ class MVCCAdapterInstance(Base):
Base.__init__(self, base._storage) Base.__init__(self, base._storage)
self._lock = Lock() self._lock = Lock()
self._invalidations = set() self._invalidations = set()
self._sync = getattr(self._storage, 'sync', lambda : None) self._sync = getattr(self._storage, 'sync', lambda: None)
def release(self): def release(self):
self._base._release(self) self._base._release(self)
...@@ -175,8 +179,8 @@ class MVCCAdapterInstance(Base): ...@@ -175,8 +179,8 @@ class MVCCAdapterInstance(Base):
# into account, and raise ReadConflictError only in the presence of # into account, and raise ReadConflictError only in the presence of
# database being simultaneously updated from back of its log. # database being simultaneously updated from back of its log.
raise POSException.ReadConflictError( raise POSException.ReadConflictError(
"load %s @%s: object deleted, likely by simultaneous pack" % "load %s @%s: object deleted, likely by simultaneous pack" %
(oid_repr(oid), tid_repr(p64(u64(self._start) - 1)))) (oid_repr(oid), tid_repr(p64(u64(self._start) - 1))))
return r[:2] return r[:2]
...@@ -189,8 +193,8 @@ class MVCCAdapterInstance(Base): ...@@ -189,8 +193,8 @@ class MVCCAdapterInstance(Base):
else: else:
raise raise
_modified = None # Used to keep track of oids modified within a _modified = None # Used to keep track of oids modified within a
# transaction, so we can invalidate them later. # transaction, so we can invalidate them later.
def tpc_begin(self, transaction): def tpc_begin(self, transaction):
self._storage.tpc_begin(transaction) self._storage.tpc_begin(transaction)
...@@ -205,7 +209,7 @@ class MVCCAdapterInstance(Base): ...@@ -205,7 +209,7 @@ class MVCCAdapterInstance(Base):
oid, serial, data, blobfilename, '', transaction) oid, serial, data, blobfilename, '', transaction)
self._modified.add(oid) self._modified.add(oid)
def tpc_finish(self, transaction, func = lambda tid: None): def tpc_finish(self, transaction, func=lambda tid: None):
modified = self._modified modified = self._modified
self._modified = None self._modified = None
...@@ -216,9 +220,11 @@ class MVCCAdapterInstance(Base): ...@@ -216,9 +220,11 @@ class MVCCAdapterInstance(Base):
return self._storage.tpc_finish(transaction, invalidate_finish) return self._storage.tpc_finish(transaction, invalidate_finish)
def read_only_writer(self, *a, **kw): def read_only_writer(self, *a, **kw):
raise POSException.ReadOnlyError raise POSException.ReadOnlyError
class HistoricalStorageAdapter(Base): class HistoricalStorageAdapter(Base):
"""Adapt a storage to a historical storage """Adapt a storage to a historical storage
""" """
...@@ -226,7 +232,7 @@ class HistoricalStorageAdapter(Base): ...@@ -226,7 +232,7 @@ class HistoricalStorageAdapter(Base):
_copy_methods = Base._copy_methods + ( _copy_methods = Base._copy_methods + (
'loadSerial', 'tpc_begin', 'tpc_finish', 'tpc_abort', 'tpc_vote', 'loadSerial', 'tpc_begin', 'tpc_finish', 'tpc_abort', 'tpc_vote',
'checkCurrentSerialInTransaction', 'checkCurrentSerialInTransaction',
) )
def __init__(self, storage, before=None): def __init__(self, storage, before=None):
Base.__init__(self, storage) Base.__init__(self, storage)
...@@ -267,7 +273,7 @@ class UndoAdapterInstance(Base): ...@@ -267,7 +273,7 @@ class UndoAdapterInstance(Base):
_copy_methods = Base._copy_methods + ( _copy_methods = Base._copy_methods + (
'tpc_abort', 'tpc_abort',
) )
def __init__(self, base): def __init__(self, base):
self._base = base self._base = base
...@@ -293,7 +299,7 @@ class UndoAdapterInstance(Base): ...@@ -293,7 +299,7 @@ class UndoAdapterInstance(Base):
if result: if result:
self._undone.update(result) self._undone.update(result)
def tpc_finish(self, transaction, func = lambda tid: None): def tpc_finish(self, transaction, func=lambda tid: None):
def invalidate_finish(tid): def invalidate_finish(tid):
self._base._invalidate_finish(tid, self._undone, None) self._base._invalidate_finish(tid, self._undone, None)
......
...@@ -63,6 +63,7 @@ class _p_DataDescr(object): ...@@ -63,6 +63,7 @@ class _p_DataDescr(object):
def __delete__(self, inst): def __delete__(self, inst):
raise AttributeError(self.__name__) raise AttributeError(self.__name__)
class _p_oid_or_jar_Descr(_p_DataDescr): class _p_oid_or_jar_Descr(_p_DataDescr):
# Special descr for _p_oid and _p_jar that loads # Special descr for _p_oid and _p_jar that loads
# state when set if both are set and _p_changed is None # state when set if both are set and _p_changed is None
...@@ -78,11 +79,11 @@ class _p_oid_or_jar_Descr(_p_DataDescr): ...@@ -78,11 +79,11 @@ class _p_oid_or_jar_Descr(_p_DataDescr):
jar = get('_p_jar') jar = get('_p_jar')
if (jar is not None if (jar is not None
and get('_p_oid') is not None and get('_p_oid') is not None
and get('_p_changed') is None and get('_p_changed') is None):
):
jar.setstate(inst) jar.setstate(inst)
class _p_ChangedDescr(object): class _p_ChangedDescr(object):
# descriptor to handle special weird semantics of _p_changed # descriptor to handle special weird semantics of _p_changed
...@@ -99,6 +100,7 @@ class _p_ChangedDescr(object): ...@@ -99,6 +100,7 @@ class _p_ChangedDescr(object):
def __delete__(self, inst): def __delete__(self, inst):
inst._p_invalidate() inst._p_invalidate()
class _p_MethodDescr(object): class _p_MethodDescr(object):
"""Provide unassignable class attributes """Provide unassignable class attributes
""" """
...@@ -120,6 +122,7 @@ class _p_MethodDescr(object): ...@@ -120,6 +122,7 @@ class _p_MethodDescr(object):
special_class_descrs = '__dict__', '__weakref__' special_class_descrs = '__dict__', '__weakref__'
class PersistentMetaClass(type): class PersistentMetaClass(type):
_p_jar = _p_oid_or_jar_Descr('_p_jar') _p_jar = _p_oid_or_jar_Descr('_p_jar')
...@@ -148,8 +151,8 @@ class PersistentMetaClass(type): ...@@ -148,8 +151,8 @@ class PersistentMetaClass(type):
and and
(get('_p_oid') is not None) (get('_p_oid') is not None)
and and
(get('_p_changed') == False) (get('_p_changed') is False)
): ):
self._p_changed = True self._p_changed = True
data_manager.register(self) data_manager.register(self)
...@@ -177,7 +180,6 @@ class PersistentMetaClass(type): ...@@ -177,7 +180,6 @@ class PersistentMetaClass(type):
_p_invalidate = _p_MethodDescr(_p_invalidate) _p_invalidate = _p_MethodDescr(_p_invalidate)
def __getstate__(self): def __getstate__(self):
return (self.__bases__, return (self.__bases__,
dict([(k, v) for (k, v) in self.__dict__.items() dict([(k, v) for (k, v) in self.__dict__.items()
...@@ -185,7 +187,7 @@ class PersistentMetaClass(type): ...@@ -185,7 +187,7 @@ class PersistentMetaClass(type):
or k.startswith('_v_') or k.startswith('_v_')
or k in special_class_descrs or k in special_class_descrs
) )
]), ]),
) )
__getstate__ = _p_MethodDescr(__getstate__) __getstate__ = _p_MethodDescr(__getstate__)
......
...@@ -9,7 +9,6 @@ from ZODB.FileStorage import FileStorage ...@@ -9,7 +9,6 @@ from ZODB.FileStorage import FileStorage
from ZODB._compat import PersistentUnpickler, BytesIO from ZODB._compat import PersistentUnpickler, BytesIO
class FakeError(Exception): class FakeError(Exception):
def __init__(self, module, name): def __init__(self, module, name):
Exception.__init__(self) Exception.__init__(self)
...@@ -41,20 +40,22 @@ class Report(object): ...@@ -41,20 +40,22 @@ class Report(object):
self.FOIDS = 0 self.FOIDS = 0
self.FBYTES = 0 self.FBYTES = 0
def shorten(s, n): def shorten(s, n):
l = len(s) length = len(s)
if l <= n: if length <= n:
return s return s
while len(s) + 3 > n: # account for ... while len(s) + 3 > n: # account for ...
i = s.find(".") i = s.find(".")
if i == -1: if i == -1:
# In the worst case, just return the rightmost n bytes # In the worst case, just return the rightmost n bytes
return s[-n:] return s[-n:]
else: else:
s = s[i + 1:] s = s[i + 1:]
l = len(s) length = len(s)
return "..." + s return "..." + s
def report(rep): def report(rep):
print("Processed %d records in %d transactions" % (rep.OIDS, rep.TIDS)) print("Processed %d records in %d transactions" % (rep.OIDS, rep.TIDS))
print("Average record size is %7.2f bytes" % (rep.DBYTES * 1.0 / rep.OIDS)) print("Average record size is %7.2f bytes" % (rep.DBYTES * 1.0 / rep.OIDS))
...@@ -63,8 +64,8 @@ def report(rep): ...@@ -63,8 +64,8 @@ def report(rep):
print("Types used:") print("Types used:")
fmt = "%-46s %7s %9s %6s %7s" fmt = "%-46s %7s %9s %6s %7s"
fmtp = "%-46s %7d %9d %5.1f%% %7.2f" # per-class format fmtp = "%-46s %7d %9d %5.1f%% %7.2f" # per-class format
fmts = "%46s %7d %8dk %5.1f%% %7.2f" # summary format fmts = "%46s %7d %8dk %5.1f%% %7.2f" # summary format
print(fmt % ("Class Name", "Count", "TBytes", "Pct", "AvgSize")) print(fmt % ("Class Name", "Count", "TBytes", "Pct", "AvgSize"))
print(fmt % ('-'*46, '-'*7, '-'*9, '-'*5, '-'*7)) print(fmt % ('-'*46, '-'*7, '-'*9, '-'*5, '-'*7))
typemap = sorted(rep.TYPEMAP) typemap = sorted(rep.TYPEMAP)
...@@ -76,8 +77,9 @@ def report(rep): ...@@ -76,8 +77,9 @@ def report(rep):
pct, rep.TYPESIZE[t] * 1.0 / rep.TYPEMAP[t])) pct, rep.TYPESIZE[t] * 1.0 / rep.TYPEMAP[t]))
print(fmt % ('='*46, '='*7, '='*9, '='*5, '='*7)) print(fmt % ('='*46, '='*7, '='*9, '='*5, '='*7))
print("%46s %7d %9s %6s %6.2fk" % ('Total Transactions', rep.TIDS, ' ', print("%46s %7d %9s %6s %6.2fk" % (
' ', rep.DBYTES * 1.0 / rep.TIDS / 1024.0)) 'Total Transactions', rep.TIDS, ' ', ' ',
rep.DBYTES * 1.0 / rep.TIDS / 1024.0))
print(fmts % ('Total Records', rep.OIDS, rep.DBYTES / 1024.0, cumpct, print(fmts % ('Total Records', rep.OIDS, rep.DBYTES / 1024.0, cumpct,
rep.DBYTES * 1.0 / rep.OIDS)) rep.DBYTES * 1.0 / rep.OIDS))
...@@ -89,6 +91,7 @@ def report(rep): ...@@ -89,6 +91,7 @@ def report(rep):
rep.FBYTES * 100.0 / rep.DBYTES, rep.FBYTES * 100.0 / rep.DBYTES,
rep.FBYTES * 1.0 / rep.FOIDS)) rep.FBYTES * 1.0 / rep.FOIDS))
def analyze(path): def analyze(path):
fs = FileStorage(path, read_only=1) fs = FileStorage(path, read_only=1)
fsi = fs.iterator() fsi = fs.iterator()
...@@ -97,11 +100,13 @@ def analyze(path): ...@@ -97,11 +100,13 @@ def analyze(path):
analyze_trans(report, txn) analyze_trans(report, txn)
return report return report
def analyze_trans(report, txn): def analyze_trans(report, txn):
report.TIDS += 1 report.TIDS += 1
for rec in txn: for rec in txn:
analyze_rec(report, rec) analyze_rec(report, rec)
def get_type(record): def get_type(record):
try: try:
unpickled = FakeUnpickler(BytesIO(record.data)).load() unpickled = FakeUnpickler(BytesIO(record.data)).load()
...@@ -114,6 +119,7 @@ def get_type(record): ...@@ -114,6 +119,7 @@ def get_type(record):
else: else:
return str(classinfo) return str(classinfo)
def analyze_rec(report, record): def analyze_rec(report, record):
oid = record.oid oid = record.oid
report.OIDS += 1 report.OIDS += 1
...@@ -121,7 +127,7 @@ def analyze_rec(report, record): ...@@ -121,7 +127,7 @@ def analyze_rec(report, record):
# No pickle -- aborted version or undo of object creation. # No pickle -- aborted version or undo of object creation.
return return
try: try:
size = len(record.data) # Ignores various overhead size = len(record.data) # Ignores various overhead
report.DBYTES += size report.DBYTES += size
if oid not in report.OIDMAP: if oid not in report.OIDMAP:
type = get_type(record) type = get_type(record)
...@@ -142,6 +148,7 @@ def analyze_rec(report, record): ...@@ -142,6 +148,7 @@ def analyze_rec(report, record):
except Exception as err: except Exception as err:
print(err) print(err)
if __name__ == "__main__": if __name__ == "__main__":
path = sys.argv[1] path = sys.argv[1]
report(analyze(path)) report(analyze(path))
...@@ -19,18 +19,21 @@ oids_seen = {} ...@@ -19,18 +19,21 @@ oids_seen = {}
# Append (obj, path) to L if and only if obj is a persistent object # Append (obj, path) to L if and only if obj is a persistent object
# and we haven't seen it before. # and we haven't seen it before.
def add_if_new_persistent(L, obj, path): def add_if_new_persistent(L, obj, path):
global oids_seen global oids_seen
getattr(obj, '_', None) # unghostify getattr(obj, '_', None) # unghostify
if hasattr(obj, '_p_oid'): if hasattr(obj, '_p_oid'):
oid = obj._p_oid oid = obj._p_oid
if oid not in oids_seen: if oid not in oids_seen:
L.append((obj, path)) L.append((obj, path))
oids_seen[oid] = 1 oids_seen[oid] = 1
def get_subobjects(obj): def get_subobjects(obj):
getattr(obj, '_', None) # unghostify getattr(obj, '_', None) # unghostify
sub = [] sub = []
try: try:
attrs = obj.__dict__.items() attrs = obj.__dict__.items()
...@@ -55,22 +58,23 @@ def get_subobjects(obj): ...@@ -55,22 +58,23 @@ def get_subobjects(obj):
while 1: while 1:
try: try:
elt = obj[i] elt = obj[i]
except: except: # noqa: E722 do not use bare 'except'
break break
sub.append(("[%d]" % i, elt)) sub.append(("[%d]" % i, elt))
i += 1 i += 1
return sub return sub
def main(fname=None): def main(fname=None):
if fname is None: if fname is None:
import sys import sys
try: try:
fname, = sys.argv[1:] fname, = sys.argv[1:]
except: except: # noqa: E722 do not use bare 'except'
print(__doc__) print(__doc__)
sys.exit(2) sys.exit(2)
fs = FileStorage(fname, read_only=1) fs = FileStorage(fname, read_only=1)
cn = ZODB.DB(fs).open() cn = ZODB.DB(fs).open()
rt = cn.root() rt = cn.root()
...@@ -116,5 +120,6 @@ def main(fname=None): ...@@ -116,5 +120,6 @@ def main(fname=None):
print("total", len(fs._index), "found", found) print("total", len(fs._index), "found", found)
if __name__ == "__main__": if __name__ == "__main__":
main() main()
...@@ -43,9 +43,11 @@ import sys ...@@ -43,9 +43,11 @@ import sys
from ZODB.FileStorage.fsoids import Tracer from ZODB.FileStorage.fsoids import Tracer
def usage(): def usage():
print(__doc__) print(__doc__)
def main(): def main():
import getopt import getopt
...@@ -64,7 +66,7 @@ def main(): ...@@ -64,7 +66,7 @@ def main():
c = Tracer(args[0]) c = Tracer(args[0])
for oid in args[1:]: for oid in args[1:]:
as_int = int(oid, 0) # 0 == auto-detect base as_int = int(oid, 0) # 0 == auto-detect base
c.register_oids(as_int) c.register_oids(as_int)
if path is not None: if path is not None:
for line in open(path): for line in open(path):
...@@ -75,5 +77,6 @@ def main(): ...@@ -75,5 +77,6 @@ def main():
c.run() c.run()
c.report() c.report()
if __name__ == "__main__": if __name__ == "__main__":
main() main()
...@@ -74,6 +74,8 @@ from BTrees.QQBTree import QQBTree ...@@ -74,6 +74,8 @@ from BTrees.QQBTree import QQBTree
# There's a problem with oid. 'data' is its pickle, and 'serial' its # There's a problem with oid. 'data' is its pickle, and 'serial' its
# serial number. 'missing' is a list of (oid, class, reason) triples, # serial number. 'missing' is a list of (oid, class, reason) triples,
# explaining what the problem(s) is(are). # explaining what the problem(s) is(are).
def report(oid, data, serial, missing): def report(oid, data, serial, missing):
from_mod, from_class = get_pickle_metadata(data) from_mod, from_class = get_pickle_metadata(data)
if len(missing) > 1: if len(missing) > 1:
...@@ -92,6 +94,7 @@ def report(oid, data, serial, missing): ...@@ -92,6 +94,7 @@ def report(oid, data, serial, missing):
print("\toid %s %s: %r" % (oid_repr(oid), reason, description)) print("\toid %s %s: %r" % (oid_repr(oid), reason, description))
print() print()
def main(path=None): def main(path=None):
verbose = 0 verbose = 0
if path is None: if path is None:
...@@ -105,7 +108,6 @@ def main(path=None): ...@@ -105,7 +108,6 @@ def main(path=None):
path, = args path, = args
fs = FileStorage(path, read_only=1) fs = FileStorage(path, read_only=1)
# Set of oids in the index that failed to load due to POSKeyError. # Set of oids in the index that failed to load due to POSKeyError.
...@@ -122,7 +124,7 @@ def main(path=None): ...@@ -122,7 +124,7 @@ def main(path=None):
# build {pos -> oid} index that is reverse to {oid -> pos} fs._index # build {pos -> oid} index that is reverse to {oid -> pos} fs._index
# we'll need this to iterate objects in order of ascending file position to # we'll need this to iterate objects in order of ascending file position to
# optimize disk IO. # optimize disk IO.
pos2oid = QQBTree() # pos -> u64(oid) pos2oid = QQBTree() # pos -> u64(oid)
for oid, pos in fs._index.iteritems(): for oid, pos in fs._index.iteritems():
pos2oid[pos] = u64(oid) pos2oid[pos] = u64(oid)
...@@ -137,14 +139,14 @@ def main(path=None): ...@@ -137,14 +139,14 @@ def main(path=None):
raise raise
except POSKeyError: except POSKeyError:
undone[oid] = 1 undone[oid] = 1
except: except: # noqa: E722 do not use bare 'except'
if verbose: if verbose:
traceback.print_exc() traceback.print_exc()
noload[oid] = 1 noload[oid] = 1
# pass 2: go through all objects again and verify that their references do # pass 2: go through all objects again and verify that their references do
# not point to problematic object set. Iterate objects in order of ascending # not point to problematic object set. Iterate objects in order of
# file position to optimize disk IO. # ascending file position to optimize disk IO.
inactive = noload.copy() inactive = noload.copy()
inactive.update(undone) inactive.update(undone)
for oid64 in pos2oid.itervalues(): for oid64 in pos2oid.itervalues():
...@@ -153,7 +155,7 @@ def main(path=None): ...@@ -153,7 +155,7 @@ def main(path=None):
continue continue
data, serial = load_current(fs, oid) data, serial = load_current(fs, oid)
refs = get_refs(data) refs = get_refs(data)
missing = [] # contains 3-tuples of oid, klass-metadata, reason missing = [] # contains 3-tuples of oid, klass-metadata, reason
for ref, klass in refs: for ref, klass in refs:
if klass is None: if klass is None:
klass = '<unknown>' klass = '<unknown>'
...@@ -166,5 +168,6 @@ def main(path=None): ...@@ -166,5 +168,6 @@ def main(path=None):
if missing: if missing:
report(oid, data, serial, missing) report(oid, data, serial, missing)
if __name__ == "__main__": if __name__ == "__main__":
main() main()
...@@ -9,6 +9,7 @@ from six.moves import filter ...@@ -9,6 +9,7 @@ from six.moves import filter
rx_txn = re.compile(r"tid=([0-9a-f]+).*size=(\d+)") rx_txn = re.compile(r"tid=([0-9a-f]+).*size=(\d+)")
rx_data = re.compile(r"oid=([0-9a-f]+) size=(\d+) class=(\S+)") rx_data = re.compile(r"oid=([0-9a-f]+) size=(\d+) class=(\S+)")
def sort_byhsize(seq, reverse=False): def sort_byhsize(seq, reverse=False):
L = [(v.size(), k, v) for k, v in seq] L = [(v.size(), k, v) for k, v in seq]
L.sort() L.sort()
...@@ -16,6 +17,7 @@ def sort_byhsize(seq, reverse=False): ...@@ -16,6 +17,7 @@ def sort_byhsize(seq, reverse=False):
L.reverse() L.reverse()
return [(k, v) for n, k, v in L] return [(k, v) for n, k, v in L]
class Histogram(dict): class Histogram(dict):
def add(self, size): def add(self, size):
...@@ -93,6 +95,7 @@ class Histogram(dict): ...@@ -93,6 +95,7 @@ class Histogram(dict):
i * binsize, n, p, pc, "*" * (n // dot))) i * binsize, n, p, pc, "*" * (n // dot)))
print() print()
def class_detail(class_size): def class_detail(class_size):
# summary of classes # summary of classes
fmt = "%5s %6s %6s %6s %-50.50s" fmt = "%5s %6s %6s %6s %-50.50s"
...@@ -110,6 +113,7 @@ def class_detail(class_size): ...@@ -110,6 +113,7 @@ def class_detail(class_size):
continue continue
h.report("Object size for %s" % klass, usebins=True) h.report("Object size for %s" % klass, usebins=True)
def revision_detail(lifetimes, classes): def revision_detail(lifetimes, classes):
# Report per-class details for any object modified more than once # Report per-class details for any object modified more than once
for name, oids in six.iteritems(classes): for name, oids in six.iteritems(classes):
...@@ -124,17 +128,18 @@ def revision_detail(lifetimes, classes): ...@@ -124,17 +128,18 @@ def revision_detail(lifetimes, classes):
if keep: if keep:
h.report("Number of revisions for %s" % name, binsize=10) h.report("Number of revisions for %s" % name, binsize=10)
def main(path=None): def main(path=None):
if path is None: if path is None:
path = sys.argv[1] path = sys.argv[1]
txn_objects = Histogram() # histogram of txn size in objects txn_objects = Histogram() # histogram of txn size in objects
txn_bytes = Histogram() # histogram of txn size in bytes txn_bytes = Histogram() # histogram of txn size in bytes
obj_size = Histogram() # histogram of object size obj_size = Histogram() # histogram of object size
n_updates = Histogram() # oid -> num updates n_updates = Histogram() # oid -> num updates
n_classes = Histogram() # class -> num objects n_classes = Histogram() # class -> num objects
lifetimes = {} # oid -> list of tids lifetimes = {} # oid -> list of tids
class_size = {} # class -> histogram of object size class_size = {} # class -> histogram of object size
classes = {} # class -> list of oids classes = {} # class -> list of oids
MAX = 0 MAX = 0
objects = 0 objects = 0
...@@ -203,5 +208,6 @@ def main(path=None): ...@@ -203,5 +208,6 @@ def main(path=None):
class_detail(class_size) class_detail(class_size)
if __name__ == "__main__": if __name__ == "__main__":
main() main()
...@@ -25,6 +25,7 @@ try: ...@@ -25,6 +25,7 @@ try:
except ImportError: except ImportError:
from sha import sha as sha1 from sha import sha as sha1
def main(path, ntxn): def main(path, ntxn):
with open(path, "rb") as f: with open(path, "rb") as f:
f.seek(0, 2) f.seek(0, 2)
...@@ -32,7 +33,6 @@ def main(path, ntxn): ...@@ -32,7 +33,6 @@ def main(path, ntxn):
i = ntxn i = ntxn
while th and i > 0: while th and i > 0:
hash = sha1(th.get_raw_data()).digest() hash = sha1(th.get_raw_data()).digest()
l = len(str(th.get_timestamp())) + 1
th.read_meta() th.read_meta()
print("%s: hash=%s" % (th.get_timestamp(), print("%s: hash=%s" % (th.get_timestamp(),
binascii.hexlify(hash).decode())) binascii.hexlify(hash).decode()))
...@@ -42,6 +42,7 @@ def main(path, ntxn): ...@@ -42,6 +42,7 @@ def main(path, ntxn):
th = th.prev_txn() th = th.prev_txn()
i -= 1 i -= 1
def Main(): def Main():
ntxn = 10 ntxn = 10
opts, args = getopt.getopt(sys.argv[1:], "n:") opts, args = getopt.getopt(sys.argv[1:], "n:")
...@@ -51,5 +52,6 @@ def Main(): ...@@ -51,5 +52,6 @@ def Main():
ntxn = int(v) ntxn = int(v)
main(path, ntxn) main(path, ntxn)
if __name__ == "__main__": if __name__ == "__main__":
Main() Main()
...@@ -41,13 +41,16 @@ import struct ...@@ -41,13 +41,16 @@ import struct
import sys import sys
from ZODB._compat import FILESTORAGE_MAGIC from ZODB._compat import FILESTORAGE_MAGIC
class FormatError(ValueError): class FormatError(ValueError):
"""There is a problem with the format of the FileStorage.""" """There is a problem with the format of the FileStorage."""
class Status(object): class Status(object):
checkpoint = b'c' checkpoint = b'c'
undone = b'u' undone = b'u'
packed_version = FILESTORAGE_MAGIC packed_version = FILESTORAGE_MAGIC
TREC_HDR_LEN = 23 TREC_HDR_LEN = 23
...@@ -55,6 +58,7 @@ DREC_HDR_LEN = 42 ...@@ -55,6 +58,7 @@ DREC_HDR_LEN = 42
VERBOSE = 0 VERBOSE = 0
def hexify(s): def hexify(s):
r"""Format an 8-bit string as hex r"""Format an 8-bit string as hex
...@@ -64,17 +68,20 @@ def hexify(s): ...@@ -64,17 +68,20 @@ def hexify(s):
""" """
return '0x' + binascii.hexlify(s).decode() return '0x' + binascii.hexlify(s).decode()
def chatter(msg, level=1): def chatter(msg, level=1):
if VERBOSE >= level: if VERBOSE >= level:
sys.stdout.write(msg) sys.stdout.write(msg)
def U64(v): def U64(v):
"""Unpack an 8-byte string as a 64-bit long""" """Unpack an 8-byte string as a 64-bit long"""
h, l = struct.unpack(">II", v) h, l_ = struct.unpack(">II", v)
if h: if h:
return (h << 32) + l return (h << 32) + l_
else: else:
return l return l_
def check(path): def check(path):
with open(path, 'rb') as file: with open(path, 'rb') as file:
...@@ -87,7 +94,7 @@ def check(path): ...@@ -87,7 +94,7 @@ def check(path):
raise FormatError("invalid file header") raise FormatError("invalid file header")
pos = 4 pos = 4
tid = b'\000' * 8 # lowest possible tid to start tid = b'\000' * 8 # lowest possible tid to start
i = 0 i = 0
while pos: while pos:
_pos = pos _pos = pos
...@@ -106,7 +113,7 @@ def check_trec(path, file, pos, ltid, file_size): ...@@ -106,7 +113,7 @@ def check_trec(path, file, pos, ltid, file_size):
used for generating error messages. used for generating error messages.
""" """
h = file.read(TREC_HDR_LEN) #XXX must be bytes under Py3k h = file.read(TREC_HDR_LEN) # XXX must be bytes under Py3k
if not h: if not h:
return None, None return None, None
if len(h) != TREC_HDR_LEN: if len(h) != TREC_HDR_LEN:
...@@ -120,7 +127,7 @@ def check_trec(path, file, pos, ltid, file_size): ...@@ -120,7 +127,7 @@ def check_trec(path, file, pos, ltid, file_size):
(path, pos, hexify(tid), hexify(ltid))) (path, pos, hexify(tid), hexify(ltid)))
ltid = tid ltid = tid
tl = U64(stl) # transaction record length - 8 tl = U64(stl) # transaction record length - 8
if pos + tl + 8 > file_size: if pos + tl + 8 > file_size:
raise FormatError("%s truncated possibly because of" raise FormatError("%s truncated possibly because of"
" damaged records at %s" % (path, pos)) " damaged records at %s" % (path, pos))
...@@ -140,7 +147,7 @@ def check_trec(path, file, pos, ltid, file_size): ...@@ -140,7 +147,7 @@ def check_trec(path, file, pos, ltid, file_size):
if status != Status.undone: if status != Status.undone:
pos = tpos + tmeta_len pos = tpos + tmeta_len
file.read(ul + dl + el) # skip transaction metadata file.read(ul + dl + el) # skip transaction metadata
i = 0 i = 0
while pos < tend: while pos < tend:
...@@ -162,6 +169,7 @@ def check_trec(path, file, pos, ltid, file_size): ...@@ -162,6 +169,7 @@ def check_trec(path, file, pos, ltid, file_size):
pos = tend + 8 pos = tend + 8
return pos, tid return pos, tid
def check_drec(path, file, pos, tpos, tid): def check_drec(path, file, pos, tpos, tid):
"""Check a data record for the current transaction record""" """Check a data record for the current transaction record"""
...@@ -170,7 +178,7 @@ def check_drec(path, file, pos, tpos, tid): ...@@ -170,7 +178,7 @@ def check_drec(path, file, pos, tpos, tid):
raise FormatError("%s truncated at %s" % (path, pos)) raise FormatError("%s truncated at %s" % (path, pos))
oid, serial, _prev, _tloc, vlen, _plen = ( oid, serial, _prev, _tloc, vlen, _plen = (
struct.unpack(">8s8s8s8sH8s", h)) struct.unpack(">8s8s8s8sH8s", h))
prev = U64(_prev) U64(_prev)
tloc = U64(_tloc) tloc = U64(_tloc)
plen = U64(_plen) plen = U64(_plen)
dlen = DREC_HDR_LEN + (plen or 8) dlen = DREC_HDR_LEN + (plen or 8)
...@@ -178,8 +186,8 @@ def check_drec(path, file, pos, tpos, tid): ...@@ -178,8 +186,8 @@ def check_drec(path, file, pos, tpos, tid):
if vlen: if vlen:
dlen = dlen + 16 + vlen dlen = dlen + 16 + vlen
file.seek(8, 1) file.seek(8, 1)
pv = U64(file.read(8)) U64(file.read(8))
file.seek(vlen, 1) # skip the version data file.seek(vlen, 1) # skip the version data
if tloc != tpos: if tloc != tpos:
raise FormatError("%s data record exceeds transaction record " raise FormatError("%s data record exceeds transaction record "
...@@ -195,9 +203,11 @@ def check_drec(path, file, pos, tpos, tid): ...@@ -195,9 +203,11 @@ def check_drec(path, file, pos, tpos, tid):
return pos, oid return pos, oid
def usage(): def usage():
sys.exit(__doc__) sys.exit(__doc__)
def main(args=None): def main(args=None):
if args is None: if args is None:
args = sys.argv[1:] args = sys.argv[1:]
...@@ -221,5 +231,6 @@ def main(args=None): ...@@ -221,5 +231,6 @@ def main(args=None):
chatter("no errors detected") chatter("no errors detected")
if __name__ == "__main__": if __name__ == "__main__":
main() main()
...@@ -6,12 +6,12 @@ Note: To run this test script fstest.py must be on your PYTHONPATH. ...@@ -6,12 +6,12 @@ Note: To run this test script fstest.py must be on your PYTHONPATH.
from cStringIO import StringIO from cStringIO import StringIO
import re import re
import struct import struct
import unittest
import ZODB.tests.util import ZODB.tests.util
import fstest import fstest
from fstest import FormatError, U64 from fstest import FormatError, U64
class TestCorruptedFS(ZODB.tests.util.TestCase): class TestCorruptedFS(ZODB.tests.util.TestCase):
f = open('test-checker.fs', 'rb') f = open('test-checker.fs', 'rb')
...@@ -117,7 +117,7 @@ class TestCorruptedFS(ZODB.tests.util.TestCase): ...@@ -117,7 +117,7 @@ class TestCorruptedFS(ZODB.tests.util.TestCase):
self._file.write(data) self._file.write(data)
buf = self._datafs.read(tl - 8) buf = self._datafs.read(tl - 8)
self._file.write(buf[0]) self._file.write(buf[0])
assert tl <= 1<<16, "can't use this transaction for this test" assert tl <= 1 << 16, "can't use this transaction for this test"
self._file.write("\777\777") self._file.write("\777\777")
self._file.write(buf[3:]) self._file.write(buf[3:])
self.detectsError("invalid transaction header") self.detectsError("invalid transaction header")
...@@ -172,6 +172,3 @@ class TestCorruptedFS(ZODB.tests.util.TestCase): ...@@ -172,6 +172,3 @@ class TestCorruptedFS(ZODB.tests.util.TestCase):
self._file.write("\000" * 4 + "\077" + "\000" * 3) self._file.write("\000" * 4 + "\077" + "\000" * 3)
self._file.write(data[32:]) self._file.write(data[32:])
self.detectsError("record exceeds transaction") self.detectsError("record exceeds transaction")
if __name__ == "__main__":
unittest.main()
...@@ -82,7 +82,7 @@ import profile ...@@ -82,7 +82,7 @@ import profile
from persistent.timestamp import TimeStamp from persistent.timestamp import TimeStamp
from ZODB import utils from ZODB import utils
from ZODB import StorageTypes # XXX: This import does not exist from ZODB import StorageTypes # XXX: This import does not exist
PROGRAM = sys.argv[0] PROGRAM = sys.argv[0]
...@@ -130,7 +130,7 @@ def main(): ...@@ -130,7 +130,7 @@ def main():
elif opt in ('-v', '--verbose'): elif opt in ('-v', '--verbose'):
options.verbose += 1 options.verbose += 1
elif opt in ('-T', '--storage_types'): elif opt in ('-T', '--storage_types'):
print_types() print('Unknown option.')
sys.exit(0) sys.exit(0)
elif opt in ('-S', '--stype'): elif opt in ('-S', '--stype'):
options.stype = arg options.stype = arg
...@@ -247,16 +247,16 @@ def doit(srcdb, dstdb, options): ...@@ -247,16 +247,16 @@ def doit(srcdb, dstdb, options):
t = TimeStamp(tid) t = TimeStamp(tid)
if t <= ts: if t <= ts:
if ok: if ok:
print(( print('Time stamps are out of order %s, %s' % (ts, t),
'Time stamps are out of order %s, %s' % (ts, t)), file=sys.stderr) file=sys.stderr)
ok = False ok = False
ts = t.laterThan(ts) ts = t.laterThan(ts)
tid = ts.raw() tid = ts.raw()
else: else:
ts = t ts = t
if not ok: if not ok:
print(( print('Time stamps are back in order %s' % t,
'Time stamps are back in order %s' % t), file=sys.stderr) file=sys.stderr)
ok = True ok = True
if verbose > 1: if verbose > 1:
print(ts) print(ts)
...@@ -310,7 +310,7 @@ def doit(srcdb, dstdb, options): ...@@ -310,7 +310,7 @@ def doit(srcdb, dstdb, options):
tidstr = utils.U64(tid) tidstr = utils.U64(tid)
format = "%4d. %20s %6d %8d %6.4f %6.4f %6.4f %6.4f %6.4f" format = "%4d. %20s %6d %8d %6.4f %6.4f %6.4f %6.4f %6.4f"
print(format % (skipper, tidstr, objects, size, print(format % (skipper, tidstr, objects, size,
t4-t0, t1-t0, t2-t1, t3-t2, t4-t3), file=outfp) t4-t0, t1-t0, t2-t1, t3-t2, t4-t3), file=outfp)
total_pickle_size += size total_pickle_size += size
total_object_count += objects total_object_count += objects
......
...@@ -23,17 +23,17 @@ from ZODB.blob import FilesystemHelper ...@@ -23,17 +23,17 @@ from ZODB.blob import FilesystemHelper
from ZODB.utils import oid_repr from ZODB.utils import oid_repr
def link_or_copy(f1, f2):
try:
os.link(f1, f2)
except OSError:
shutil.copy(f1, f2)
# Check if we actually have link # Check if we actually have link
try: try:
os.link os.link
except AttributeError: except AttributeError:
link_or_copy = shutil.copy link_or_copy = shutil.copy
else:
def link_or_copy(f1, f2):
try:
os.link(f1, f2)
except OSError:
shutil.copy(f1, f2)
def migrate(source, dest, layout): def migrate(source, dest, layout):
......
...@@ -13,6 +13,7 @@ from ZODB.utils import U64, get_pickle_metadata, load_current ...@@ -13,6 +13,7 @@ from ZODB.utils import U64, get_pickle_metadata, load_current
from ZODB.serialize import referencesf from ZODB.serialize import referencesf
from six.moves import filter from six.moves import filter
def find_paths(root, maxdist): def find_paths(root, maxdist):
"""Find Python attribute traversal paths for objects to maxdist distance. """Find Python attribute traversal paths for objects to maxdist distance.
...@@ -37,7 +38,7 @@ def find_paths(root, maxdist): ...@@ -37,7 +38,7 @@ def find_paths(root, maxdist):
if oid is not None: if oid is not None:
paths[oid] = path paths[oid] = path
if dist < maxdist: if dist < maxdist:
getattr(obj, 'foo', None) # unghostify getattr(obj, 'foo', None) # unghostify
try: try:
items = obj.__dict__.items() items = obj.__dict__.items()
except AttributeError: except AttributeError:
...@@ -48,6 +49,7 @@ def find_paths(root, maxdist): ...@@ -48,6 +49,7 @@ def find_paths(root, maxdist):
return paths return paths
def main(path): def main(path):
fs = FileStorage(path, read_only=1) fs = FileStorage(path, read_only=1)
if PACK: if PACK:
...@@ -60,6 +62,7 @@ def main(path): ...@@ -60,6 +62,7 @@ def main(path):
def total_size(oid): def total_size(oid):
cache = {} cache = {}
cache_size = 1000 cache_size = 1000
def _total_size(oid, seen): def _total_size(oid, seen):
v = cache.get(oid) v = cache.get(oid)
if v is not None: if v is not None:
...@@ -91,10 +94,11 @@ def main(path): ...@@ -91,10 +94,11 @@ def main(path):
for oid in keys: for oid in keys:
data, serialno = load_current(fs, oid) data, serialno = load_current(fs, oid)
mod, klass = get_pickle_metadata(data) mod, klass = get_pickle_metadata(data)
refs = referencesf(data) referencesf(data)
path = paths.get(oid, '-') path = paths.get(oid, '-')
print(fmt % (U64(oid), len(data), total_size(oid), path, mod, klass)) print(fmt % (U64(oid), len(data), total_size(oid), path, mod, klass))
def Main(): def Main():
import sys import sys
import getopt import getopt
...@@ -122,5 +126,6 @@ def Main(): ...@@ -122,5 +126,6 @@ def Main():
VERBOSE += 1 VERBOSE += 1
main(path) main(path)
if __name__ == "__main__": if __name__ == "__main__":
Main() Main()
...@@ -18,6 +18,7 @@ $Id$ ...@@ -18,6 +18,7 @@ $Id$
from ZODB.serialize import referencesf from ZODB.serialize import referencesf
def referrers(storage): def referrers(storage):
result = {} result = {}
for transaction in storage.iterator(): for transaction in storage.iterator():
......
...@@ -85,6 +85,7 @@ Options for -V/--verify: ...@@ -85,6 +85,7 @@ Options for -V/--verify:
Verify file sizes only (skip md5 checksums). Verify file sizes only (skip md5 checksums).
""" """
from __future__ import print_function from __future__ import print_function
import re
import os import os
import shutil import shutil
import sys import sys
...@@ -176,7 +177,7 @@ def parseargs(argv): ...@@ -176,7 +177,7 @@ def parseargs(argv):
'date=', 'date=',
'output=', 'output=',
'with-verification', 'with-verification',
]) ])
except getopt.error as msg: except getopt.error as msg:
usage(1, msg) usage(1, msg)
...@@ -299,6 +300,8 @@ def fsync(afile): ...@@ -299,6 +300,8 @@ def fsync(afile):
# Return the total number of bytes read == the total number of bytes # Return the total number of bytes read == the total number of bytes
# passed in all to func(). Leaves the file position just after the # passed in all to func(). Leaves the file position just after the
# last byte read. # last byte read.
def dofile(func, fp, n=None): def dofile(func, fp, n=None):
bytesread = 0 bytesread = 0
while n is None or n > 0: while n is None or n > 0:
...@@ -320,6 +323,7 @@ def dofile(func, fp, n=None): ...@@ -320,6 +323,7 @@ def dofile(func, fp, n=None):
def checksum(fp, n): def checksum(fp, n):
# Checksum the first n bytes of the specified file # Checksum the first n bytes of the specified file
sum = md5() sum = md5()
def func(data): def func(data):
sum.update(data) sum.update(data)
dofile(func, fp, n) dofile(func, fp, n)
...@@ -336,6 +340,7 @@ def file_size(fp): ...@@ -336,6 +340,7 @@ def file_size(fp):
def checksum_and_size(fp): def checksum_and_size(fp):
# Checksum and return it with the size of the file # Checksum and return it with the size of the file
sum = md5() sum = md5()
def func(data): def func(data):
sum.update(data) sum.update(data)
size = dofile(func, fp, None) size = dofile(func, fp, None)
...@@ -374,6 +379,7 @@ def concat(files, ofp=None): ...@@ -374,6 +379,7 @@ def concat(files, ofp=None):
# given. Return the number of bytes written and the md5 checksum of the # given. Return the number of bytes written and the md5 checksum of the
# bytes. # bytes.
sum = md5() sum = md5()
def func(data): def func(data):
sum.update(data) sum.update(data)
if ofp: if ofp:
...@@ -393,6 +399,7 @@ def concat(files, ofp=None): ...@@ -393,6 +399,7 @@ def concat(files, ofp=None):
def gen_filedate(options): def gen_filedate(options):
return getattr(options, 'test_now', time.gmtime()[:6]) return getattr(options, 'test_now', time.gmtime()[:6])
def gen_filename(options, ext=None, now=None): def gen_filename(options, ext=None, now=None):
if ext is None: if ext is None:
if options.full: if options.full:
...@@ -412,10 +419,11 @@ def gen_filename(options, ext=None, now=None): ...@@ -412,10 +419,11 @@ def gen_filename(options, ext=None, now=None):
# files, from the time of the most recent full backup preceding # files, from the time of the most recent full backup preceding
# options.date, up to options.date. # options.date, up to options.date.
import re
is_data_file = re.compile(r'\d{4}(?:-\d\d){5}\.(?:delta)?fsz?$').match is_data_file = re.compile(r'\d{4}(?:-\d\d){5}\.(?:delta)?fsz?$').match
del re del re
def find_files(options): def find_files(options):
when = options.date when = options.date
if not when: if not when:
...@@ -455,10 +463,11 @@ def find_files(options): ...@@ -455,10 +463,11 @@ def find_files(options):
# #
# None, None, None, None # None, None, None, None
def scandat(repofiles): def scandat(repofiles):
fullfile = repofiles[0] fullfile = repofiles[0]
datfile = os.path.splitext(fullfile)[0] + '.dat' datfile = os.path.splitext(fullfile)[0] + '.dat'
fn = startpos = endpos = sum = None # assume .dat file missing or empty fn = startpos = endpos = sum = None # assume .dat file missing or empty
try: try:
fp = open(datfile) fp = open(datfile)
except IOError as e: except IOError as e:
...@@ -475,6 +484,7 @@ def scandat(repofiles): ...@@ -475,6 +484,7 @@ def scandat(repofiles):
return fn, startpos, endpos, sum return fn, startpos, endpos, sum
def delete_old_backups(options): def delete_old_backups(options):
# Delete all full backup files except for the most recent full backup file # Delete all full backup files except for the most recent full backup file
all = sorted(filter(is_data_file, os.listdir(options.repository))) all = sorted(filter(is_data_file, os.listdir(options.repository)))
...@@ -515,6 +525,7 @@ def delete_old_backups(options): ...@@ -515,6 +525,7 @@ def delete_old_backups(options):
pass pass
os.unlink(os.path.join(options.repository, fname)) os.unlink(os.path.join(options.repository, fname))
def do_full_backup(options): def do_full_backup(options):
options.full = True options.full = True
tnow = gen_filedate(options) tnow = gen_filedate(options)
...@@ -714,7 +725,8 @@ def do_recover(options): ...@@ -714,7 +725,8 @@ def do_recover(options):
"%s has checksum %s instead of %s" % ( "%s has checksum %s instead of %s" % (
repofile, reposum, expected_truth['sum'])) repofile, reposum, expected_truth['sum']))
totalsz += reposz totalsz += reposz
log("Recovered chunk %s : %s bytes, md5: %s", repofile, reposz, reposum) log("Recovered chunk %s : %s bytes, md5: %s",
repofile, reposz, reposum)
log("Recovered a total of %s bytes", totalsz) log("Recovered a total of %s bytes", totalsz)
else: else:
reposz, reposum = concat(repofiles, outfp) reposz, reposum = concat(repofiles, outfp)
...@@ -725,7 +737,8 @@ def do_recover(options): ...@@ -725,7 +737,8 @@ def do_recover(options):
source_index = '%s.index' % last_base source_index = '%s.index' % last_base
target_index = '%s.index' % options.output target_index = '%s.index' % options.output
if os.path.exists(source_index): if os.path.exists(source_index):
log('Restoring index file %s to %s', source_index, target_index) log('Restoring index file %s to %s',
source_index, target_index)
shutil.copyfile(source_index, target_index) shutil.copyfile(source_index, target_index)
else: else:
log('No index file to restore: %s', source_index) log('No index file to restore: %s', source_index)
...@@ -737,8 +750,8 @@ def do_recover(options): ...@@ -737,8 +750,8 @@ def do_recover(options):
try: try:
os.rename(temporary_output_file, options.output) os.rename(temporary_output_file, options.output)
except OSError: except OSError:
log("ZODB has been fully recovered as %s, but it cannot be renamed into : %s", log("ZODB has been fully recovered as %s, but it cannot be renamed"
temporary_output_file, options.output) " into : %s", temporary_output_file, options.output)
raise raise
...@@ -759,10 +772,12 @@ def do_verify(options): ...@@ -759,10 +772,12 @@ def do_verify(options):
log("Verifying %s", filename) log("Verifying %s", filename)
try: try:
if filename.endswith('fsz'): if filename.endswith('fsz'):
actual_sum, size = get_checksum_and_size_of_gzipped_file(filename, options.quick) actual_sum, size = get_checksum_and_size_of_gzipped_file(
filename, options.quick)
when_uncompressed = ' (when uncompressed)' when_uncompressed = ' (when uncompressed)'
else: else:
actual_sum, size = get_checksum_and_size_of_file(filename, options.quick) actual_sum, size = get_checksum_and_size_of_file(
filename, options.quick)
when_uncompressed = '' when_uncompressed = ''
except IOError: except IOError:
error("%s is missing", filename) error("%s is missing", filename)
......
...@@ -12,6 +12,7 @@ from ZODB.FileStorage import FileStorage ...@@ -12,6 +12,7 @@ from ZODB.FileStorage import FileStorage
from ZODB.utils import U64, get_pickle_metadata, load_current from ZODB.utils import U64, get_pickle_metadata, load_current
import six import six
def run(path, v=0): def run(path, v=0):
fs = FileStorage(path, read_only=1) fs = FileStorage(path, read_only=1)
# break into the file implementation # break into the file implementation
...@@ -31,12 +32,13 @@ def run(path, v=0): ...@@ -31,12 +32,13 @@ def run(path, v=0):
if v: if v:
print("%8s %5d %s" % (U64(oid), len(data), key)) print("%8s %5d %s" % (U64(oid), len(data), key))
L = totals.items() L = totals.items()
L.sort(lambda a, b: cmp(a[1], b[1])) L.sort(key=lambda x: x[1])
L.reverse() L.reverse()
print("Totals per object class:") print("Totals per object class:")
for key, (bytes, count) in L: for key, (bytes, count) in L:
print("%8d %8d %s" % (count, bytes, key)) print("%8d %8d %s" % (count, bytes, key))
def main(): def main():
import sys import sys
import getopt import getopt
...@@ -56,5 +58,6 @@ def main(): ...@@ -56,5 +58,6 @@ def main():
path = args[0] path = args[0]
run(path, v) run(path, v)
if __name__ == "__main__": if __name__ == "__main__":
main() main()
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment