Commit 960c5e17 authored by Kirill Smelkov's avatar Kirill Smelkov

zodbcommit - Tool to commit new transaction into ZODB

Zodbcommit reads transaction description from stdin and commits read data into
ZODB. The transaction to be committed is read in zodbdump format, but without
first 'txn' header line. For example:

    user "author"
    description "change 123"
    extension ""
    obj 0000000000000001 4 null:00
    ZZZZ

This tool could be useful for testing and for low-level database
maintenance. Please see zodbcommit.py docstring for more details.
parent dd959b28
......@@ -10,5 +10,6 @@ __ https://github.com/zopefoundation/ZODB/pull/128#issuecomment-260970932
- `zodb analyze` - analyze FileStorage or repozo deltafs usage.
- `zodb cmp` - compare content of two ZODB databases bit-to-bit.
- `zodb commit` - commit new transaction into a ZODB database.
- `zodb dump` - dump content of a ZODB database.
- `zodb info` - print general information about a ZODB database.
# Copyright (C) 2018 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
# option) any later version, as published by the Free Software Foundation.
#
# You can also Link and Combine this program with other software covered by
# the terms of any of the Free Software licenses or any of the Open Source
# Initiative approved licenses and Convey the resulting work. Corresponding
# source of such a combination shall include the source code for all other
# software used.
#
# This program is distributed WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See COPYING file for full licensing terms.
# See https://www.nexedi.com/licensing for rationale and options.
from zodbtools.zodbcommit import zodbcommit
from zodbtools.zodbdump import zodbdump, Transaction, ObjectData, ObjectDelete, ObjectCopy
from zodbtools.util import storageFromURL, sha1
from ZODB.utils import p64, u64, z64, maxtid
from ZODB._compat import BytesIO, dumps, _protocol # XXX can't yet commit with arbitrary ext.bytes
from tempfile import mkdtemp
from shutil import rmtree
from golang import func, defer
# verify zodbcommit.
@func
def test_zodbcommit():
tmpd = mkdtemp('', 'zodbcommit.')
defer(lambda: rmtree(tmpd))
stor = storageFromURL('%s/2.fs' % tmpd)
defer(stor.close)
head = stor.lastTransaction()
# commit some transactions via zodbcommit and verify if storage dump gives
# what is expected.
t1 = Transaction(z64, ' ', b'user name', b'description ...', dumps({'a': 'b'}, _protocol), [
ObjectData(p64(1), b'data1', 'sha1', sha1('data1')),
ObjectData(p64(2), b'data2', 'sha1', sha1('data2'))])
t1.tid = zodbcommit(stor, head, t1)
t2 = Transaction(z64, ' ', b'user2', b'desc2', b'', [
ObjectDelete(p64(2))])
t2.tid = zodbcommit(stor, t1.tid, t2)
buf = BytesIO()
zodbdump(stor, p64(u64(head)+1), maxtid, out=buf)
dumped = buf.getvalue()
assert dumped == ''.join([_.zdump() for _ in (t1, t2)])
# ObjectCopy. XXX zodbcommit handled ObjectCopy by actually copying data,
# not referencing previous transaction via backpointer.
t3 = Transaction(z64, ' ', b'user3', b'desc3', b'', [
ObjectCopy(p64(1), t1.tid)])
t3.tid = zodbcommit(stor, t2.tid, t3)
data1_1, _, _ = stor.loadBefore(p64(1), p64(u64(t1.tid)+1))
data1_3, _, _ = stor.loadBefore(p64(1), p64(u64(t3.tid)+1))
assert data1_1 == data1_3
assert data1_1 == b'data1' # just in case
#!/usr/bin/env python
# Copyright (C) 2017 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
# Copyright (C) 2017-2018 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
......@@ -35,7 +35,7 @@ def register_command(cmdname):
command_module = importlib.import_module('zodbtools.zodb' + cmdname)
command_dict[cmdname] = command_module
for _ in ('analyze', 'cmp', 'dump', 'info'):
for _ in ('analyze', 'cmp', 'commit', 'dump', 'info'):
register_command(_)
......
# Copyright (C) 2018 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
# option) any later version, as published by the Free Software Foundation.
#
# You can also Link and Combine this program with other software covered by
# the terms of any of the Free Software licenses or any of the Open Source
# Initiative approved licenses and Convey the resulting work. Corresponding
# source of such a combination shall include the source code for all other
# software used.
#
# This program is distributed WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See COPYING file for full licensing terms.
# See https://www.nexedi.com/licensing for rationale and options.
"""Zodbcommit - Commit new transaction into a ZODB database
Zodbcommit reads transaction description from stdin and commits read data into
ZODB. The transaction to be committed is read in zodbdump format, but without
first 'txn' header line. For example::
user "author"
description "change 123"
extension ""
obj 0000000000000001 4 null:00
ZZZZ
On success the ID of committed transaction is printed to stdout.
On conflict or other problem - the error is printed to stderr exit code is !0.
Zodbcommit requires `at` parameter to be given. This specifies caller idea
about its current database view and is used to detect conflicting simultaneous
commits. `at` is required because zodbcommit is plumbing-level command and
implicitly using storage last_tid instead of it could hide bugs. In scripts one
can query current database head (last_tid) with `zodb info <stor> last_tid`.
"""
from __future__ import print_function
from zodbtools import zodbdump
from zodbtools.util import ashex, storageFromURL
from ZODB.utils import p64, u64, z64
from ZODB.POSException import POSKeyError
from ZODB._compat import BytesIO
from golang import panic
# zodbcommit commits new transaction into ZODB storage with data specified by
# zodbdump transaction.
#
# txn.tid is ignored.
# tid of committed transaction is returned.
def zodbcommit(stor, at, txn):
assert isinstance(txn, zodbdump.Transaction)
before = p64(u64(at)+1)
stor.tpc_begin(txn)
for obj in txn.objv:
data = None # data do be committed - setup vvv
if isinstance(obj, zodbdump.ObjectCopy):
# NEO does not support restore, and even if stor supports restore,
# going that way requires to already know tid for transaction we are
# committing. -> we just imitate copy by actually copying data and
# letting the storage deduplicate it.
data, _, _ = stor.loadBefore(obj.oid, p64(u64(obj.copy_from)+1))
elif isinstance(obj, zodbdump.ObjectDelete):
data = None
elif isinstance(obj, zodbdump.ObjectData):
if isinstance(obj.data, zodbdump.HashOnly):
raise ValueError('cannot commit transaction with hashonly object')
data = obj.data
else:
panic('invalid object record: %r' % (obj,))
# now we have the data.
# find out what is oid's serial as of <before state
try:
xdata = stor.loadBefore(obj.oid, before)
except POSKeyError:
serial_prev = z64
else:
if xdata is None:
serial_prev = z64
else:
_, serial_prev, _ = xdata
# store the object.
# if it will be ConflictError - we just fail and let the caller retry.
if data is None:
stor.deleteObject(obj.oid, serial_prev, txn)
else:
stor.store(obj.oid, serial_prev, data, '', txn)
stor.tpc_vote(txn)
# in ZODB >= 5 tpc_finish returns tid directly, but on ZODB 4 and ZODB 3 it
# does not do so. Since we still need to support ZODB 4, utilize tpc_finish
# callback to know with which tid the transaction was committed.
_ = []
stor.tpc_finish(txn, lambda tid: _.append(tid))
assert len(_) == 1
tid = _[0]
return tid
# ----------------------------------------
import sys, getopt
summary = "commit new transaction into a ZODB database"
def usage(out):
print("""\
Usage: zodb commit [OPTIONS] <storage> <at> < input
Commit new transaction into a ZODB database.
The transaction to be committed is read from stdin in zodbdump format without
first 'txn' header line.
<storage> is an URL (see 'zodb help zurl') of a ZODB-storage.
<at> is transaction ID of what is caller idea about its current database view.
On success the ID of committed transaction is printed to stdout.
Options:
-h --help show this help
""", file=out)
def main(argv):
try:
optv, argv = getopt.getopt(argv[1:], "h", ["help"])
except getopt.GetoptError as e:
print(e, file=sys.stderr)
usage(sys.stderr)
sys.exit(2)
for opt, _ in optv:
if opt in ("-h", "--help"):
usage(sys.stdout)
sys.exit(0)
if len(argv) != 2:
usage(sys.stderr)
sys.exit(2)
storurl = argv[0]
at = argv[1].decode('hex')
stor = storageFromURL(storurl)
zin = 'txn 0000000000000000 " "\n' # artificial transaction header
zin += sys.stdin.read()
zin = BytesIO(zin)
zr = zodbdump.DumpReader(zin)
zr.lineno -= 1 # we prepended txn header
txn = zr.readtxn()
tail = zin.read()
if tail:
print('E: +%d: garbage after transaction' % zr.lineno, file=sys.stderr)
sys.exit(1)
tid = zodbcommit(stor, at, txn)
print(ashex(tid))
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment