Skip to content

N
neoppod
Commit 1eb56978 authored by Kirill Smelkov's avatar Kirill Smelkov
Browse files
Options

X got python quoting right

parent fa3a552c
Expand all Show whitespace changes
Inline Side-by-side
Showing with 253 additions and 208 deletions
t/neo/storage/fs1/cmd/fstail/fstail.go
View file @ 1eb56978
...@@ -29,6 +29,7 @@ import ( ...@@ -29,6 +29,7 @@ import (
"log" "log"
"os" "os"
"strconv" "strconv"
"strings"
"unicode/utf8" "unicode/utf8"
"../../../../storage/fs1" "../../../../storage/fs1"
...@@ -36,9 +37,7 @@ import ( ...@@ -36,9 +37,7 @@ import (
"lab.nexedi.com/kirr/go123/mem" "lab.nexedi.com/kirr/go123/mem"
) )
// pyQuote quotes string the way python would do it // pyQuote quotes string the way python repr(str) would do it
// specifically quote char is ' (not " as in go)
// XXX s = 'a\'bc'; print repr(s) -> "a'bc" (not 'a\'bc' <- XXX fix this
func pyQuote(s string) string { func pyQuote(s string) string {
out := pyQuoteBytes(mem.Bytes(s)) out := pyQuoteBytes(mem.Bytes(s))
return mem.String(out) return mem.String(out)
...@@ -47,19 +46,52 @@ func pyQuote(s string) string { ...@@ -47,19 +46,52 @@ func pyQuote(s string) string {
func pyQuoteBytes(b []byte) []byte { func pyQuoteBytes(b []byte) []byte {
s := mem.String(b) s := mem.String(b)
buf := make([]byte, 0, len(s)) buf := make([]byte, 0, len(s))
buf = append(buf, '\'')
// smartquotes: choose ' or " as quoting character
// https://github.com/python/cpython/blob/v2.7.13-116-g1aa1803b3d/Objects/stringobject.c#L947
quote := byte('\'')
noquote := byte('"')
if strings.ContainsRune(s, '\'') && !strings.ContainsRune(s, '"') {
quote, noquote = noquote, quote
}
buf = append(buf, quote)
for i, r := range s { for i, r := range s {
if r == utf8.RuneError { switch r {
buf = append(buf, []byte(fmt.Sprintf("\\x%0x", s[i]))...) case utf8.RuneError:
} else { buf = append(buf, []byte(fmt.Sprintf("\\x%02x", s[i]))...)
case '\\', rune(quote):
buf = append(buf, '\\', byte(r))
case rune(noquote):
buf = append(buf, noquote)
// NOTE python converts to \<letter> only \t \n \r (not e.g. \v)
// https://github.com/python/cpython/blob/v2.7.13-116-g1aa1803b3d/Objects/stringobject.c#L963
case '\t':
buf = append(buf, `\t`...)
case '\n':
buf = append(buf, `\n`...)
case '\r':
buf = append(buf, `\r`...)
default:
switch {
case r < ' ':
// we already converted to \<letter> what python represents as such above
buf = append(buf, []byte(fmt.Sprintf("\\x%02x", s[i]))...)
default:
// we aleady handled ', " and (< ' ') above, so now it
// should be safe to reuse strconv.QuoteRune
rq := strconv.QuoteRune(r) // "'\x01'" rq := strconv.QuoteRune(r) // "'\x01'"
rq = rq[1:len(rq)-1] // "\x01" rq = rq[1:len(rq)-1] // "\x01"
buf = append(buf, rq...) buf = append(buf, rq...)
} }
} }
}
buf = append(buf, '\'') buf = append(buf, quote)
return buf return buf
} }
...@@ -141,7 +173,11 @@ func fsDump(w io.Writer, path string, ntxn int) (err error) { ...@@ -141,7 +173,11 @@ func fsDump(w io.Writer, path string, ntxn int) (err error) {
// print information about read txn record // print information about read txn record
_, err = fmt.Fprintf(w, "%s: hash=%x\nuser=%s description=%s length=%d offset=%d (+%d)\n\n", _, err = fmt.Fprintf(w, "%s: hash=%x\nuser=%s description=%s length=%d offset=%d (+%d)\n\n",
txnh.Tid.Time(), sha1.Sum(data), txnh.Tid.Time(), sha1.Sum(data),
// fstail.py uses repr to print user/description:
// https://github.com/zopefoundation/ZODB/blob/5.2.0-4-g359f40ec7/src/ZODB/scripts/fstail.py#L39
pyQuoteBytes(txnh.User), pyQuoteBytes(txnh.Description), pyQuoteBytes(txnh.User), pyQuoteBytes(txnh.Description),
// NOTE in zodb/py .length is len - 8, in zodb/go - whole txn record length // NOTE in zodb/py .length is len - 8, in zodb/go - whole txn record length
txnh.Len - 8, txnh.Len - 8,
txnh.Pos, txnh.HeaderLen()) txnh.Pos, txnh.HeaderLen())
...@@ -157,7 +193,7 @@ func fsDump(w io.Writer, path string, ntxn int) (err error) { ...@@ -157,7 +193,7 @@ func fsDump(w io.Writer, path string, ntxn int) (err error) {
func main() { func main() {
ntxn := 10 ntxn := 10
usage := func() { flag.Usage = func() {
fmt.Fprintf(os.Stderr, fmt.Fprintf(os.Stderr,
`fstail [options] <storage> `fstail [options] <storage>
Dump transactions from a FileStorage in reverse order Dump transactions from a FileStorage in reverse order
...@@ -171,13 +207,12 @@ Dump transactions from a FileStorage in reverse order ...@@ -171,13 +207,12 @@ Dump transactions from a FileStorage in reverse order
`, ntxn) `, ntxn)
} }
flag.Usage = usage
flag.IntVar(&ntxn, "n", ntxn, "output the last <N> transactions") flag.IntVar(&ntxn, "n", ntxn, "output the last <N> transactions")
flag.Parse() flag.Parse()
argv := flag.Args() argv := flag.Args()
if len(argv) < 1 { if len(argv) < 1 {
usage() flag.Usage()
os.Exit(2) // XXX recheck it is same as from flag.Parse on -zzz os.Exit(2) // XXX recheck it is same as from flag.Parse on -zzz
} }
storPath := argv[0] storPath := argv[0]
......
t/neo/storage/fs1/cmd/fstail/testdata/1.fsdump.ok
View file @ 1eb56978
1979-01-03 21:01:31.300002: hash=9176cad01fe2751a2317912476da043f2cd27085 1979-01-03 21:01:31.300002: hash=132dacc98500dc2f6d3ed7339fb9f8e820719477
user="root1\nYour\nRoyal\nMagesty' " description='delete 1\nalpha beta gamma\'delta"lambda\n\nqqq ...' length=206 offset=11948 (+156) user="root1\nYour\nRoyal\nMagesty' \x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f" description='delete 1\nalpha beta gamma\'delta"lambda\n\nqqq ...' length=238 offset=11980 (+188)
1979-01-03 21:01:30.200002: hash=4c2bcec885530156e134f089e7255a0ad1f2eab0 1979-01-03 21:01:30.200002: hash=d783be6d164baac8786e235590903bc67e949b61
user='' description='predelete 6' length=362 offset=11578 (+34) user='' description='predelete 6' length=362 offset=11610 (+34)
1979-01-03 21:01:29.100002: hash=fd3148ac612b6ec793747165c73ce236a3b61702 1979-01-03 21:01:29.100002: hash=7ec0737f615cf705cebfbda1a0b89412adb52d0a
user='root1.1\nYour\nMagesty ' description='undo 1.1\nmore detailed description\n\nzzz ...\t\t' length=208 offset=11362 (+158) user='root1.1\nYour\nMagesty ' description='undo 1.1\nmore detailed description\n\nzzz ...\t\t' length=208 offset=11394 (+158)
1979-01-03 21:01:28.000002: hash=ef9928d3049ad902a5d98293d46358cea8f44319 1979-01-03 21:01:28.000002: hash=c2fcc684a7916bdcce0a8ba9cb9c9391ba13654d
user='root1.0\nYour\nMagesty ' description='undo 1.0\nmore detailed description\n\nzzz ...\t' length=207 offset=11147 (+157) user='root1.0\nYour\nMagesty ' description='undo 1.0\nmore detailed description\n\nzzz ...\t' length=207 offset=11179 (+157)
1979-01-03 21:01:25.800002: hash=bc4c742d2821d463ffad5348f6c86d4ed44996cf 1979-01-03 21:01:25.800002: hash=392f2216c2c2ae37d1a396bfe0394d85aad0f1dc
user='user1.24' description='step 1.24' length=165 offset=10974 (+93) user='user1.24' description='step 1.24' length=165 offset=11006 (+93)
1979-01-03 21:01:24.700002: hash=ad2bfa76c1c7288f99831397715d120a84ac3e63 1979-01-03 21:01:24.700002: hash=89f5e0ad5af7fa46d418412ecb5ccf8ec5d550f4
user='user1.23' description='step 1.23' length=165 offset=10801 (+93) user='user1.23' description='step 1.23' length=165 offset=10833 (+93)
1979-01-03 21:01:23.600002: hash=722921c9dd036f270f505d4e1645a63b62f6bac1 1979-01-03 21:01:23.600002: hash=422d4514a5a0cf08afc427b2a589264fa798f0fd
user='user1.22' description='step 1.22' length=165 offset=10628 (+93) user='user1.22' description='step 1.22' length=165 offset=10660 (+93)
1979-01-03 21:01:22.500002: hash=f22d4cdd01d17a66d53e8727b1305c227983b194 1979-01-03 21:01:22.500002: hash=11ecff1984ddf48e3da386bc4d689900fb70c9b1
user='user1.21' description='step 1.21' length=165 offset=10455 (+93) user='user1.21' description='step 1.21' length=165 offset=10487 (+93)
1979-01-03 21:01:21.400002: hash=e3c412c26b6522c046bdfe630eef32451b50ba69 1979-01-03 21:01:21.400002: hash=1057ae69f5f40b49fb610f8672698ecd31dd3501
user='user1.20' description='step 1.20' length=165 offset=10282 (+93) user='user1.20' description='step 1.20' length=165 offset=10314 (+93)
1979-01-03 21:01:20.300002: hash=54cdd59a7131c0784bb3236471d857ed6f33f9b9 1979-01-03 21:01:20.300002: hash=916cb146cafdc4094f57ec64b32256195e6bd0b5
user='user1.19' description='step 1.19' length=165 offset=10109 (+93) user='user1.19' description='step 1.19' length=165 offset=10141 (+93)
1979-01-03 21:01:19.200002: hash=20a4abf3c8a0984798374af5ed890e2910f46265 1979-01-03 21:01:19.200002: hash=422ddafb8ba35869bb887c583cb15cb35ee90d0a
user='user1.18' description='step 1.18' length=165 offset=9936 (+93) user='user1.18' description='step 1.18' length=165 offset=9968 (+93)
1979-01-03 21:01:18.100002: hash=a7c66e66a6e25cafe9b046b5a41993287ba3e296 1979-01-03 21:01:18.100002: hash=5c0b9f3fc2cb8da9de23109cbb05b47600209a3f
user='user1.17' description='step 1.17' length=165 offset=9763 (+93) user='user1.17' description='step 1.17' length=165 offset=9795 (+93)
1979-01-03 21:01:17.000002: hash=a69f094230ca875a963524c93a35d23f0687308d 1979-01-03 21:01:17.000002: hash=7bc7b122bc5b19fc5956ea1fadff26bddd6d76ef
user='user1.16' description='step 1.16' length=165 offset=9590 (+93) user='user1.16' description='step 1.16' length=165 offset=9622 (+93)
1979-01-03 21:01:15.900002: hash=96316e0682da99e24c5d587b4bd4d2f3150eea8b 1979-01-03 21:01:15.900002: hash=c883974a78a88618132cf3152d29db1ee3626da9
user='user1.15' description='step 1.15' length=165 offset=9417 (+93) user='user1.15' description='step 1.15' length=165 offset=9449 (+93)
1979-01-03 21:01:14.800002: hash=da20e7ad9c0f55e7576dba5ffd9912c37d059232 1979-01-03 21:01:14.800002: hash=7f3e20b9ba460593174bb9f1a9901aa6f8b559e2
user='user1.14' description='step 1.14' length=165 offset=9244 (+93) user='user1.14' description='step 1.14' length=165 offset=9276 (+93)
1979-01-03 21:01:13.700002: hash=877241dc4a5f14d24a7779917310332f6c2e414e 1979-01-03 21:01:13.700002: hash=3818e96310cf6e6b67709fc486858fc223113be5
user='user1.13' description='step 1.13' length=165 offset=9071 (+93) user='user1.13' description='step 1.13' length=165 offset=9103 (+93)
1979-01-03 21:01:12.600002: hash=321ea792b7e67ac9339de238e817ca639d7f42ec 1979-01-03 21:01:12.600002: hash=97a27eaa6771b0fc8a0473251ef2f06a65866c08
user='user1.12' description='step 1.12' length=165 offset=8898 (+93) user='user1.12' description='step 1.12' length=165 offset=8930 (+93)
1979-01-03 21:01:11.500002: hash=6b64527e3ba109266a8d68321147d54af8d6edc7 1979-01-03 21:01:11.500002: hash=ca201ecaf94a4809b6c87b9c22e827cc8e901617
user='user1.11' description='step 1.11' length=165 offset=8725 (+93) user='user1.11' description='step 1.11' length=165 offset=8757 (+93)
1979-01-03 21:01:10.400002: hash=a5db043429fc63d48df1922ab60f3e6990d3561c 1979-01-03 21:01:10.400002: hash=f9cd2cadaef25f9854e589965d5a6dfee9a42b2e
user='user1.10' description='step 1.10' length=165 offset=8552 (+93) user='user1.10' description='step 1.10' length=165 offset=8584 (+93)
1979-01-03 21:01:09.300001: hash=200139e7913a7ff53a34c6f5484b92128b246cde 1979-01-03 21:01:09.300001: hash=3cc0c694df5aaf607331a3b5f0f7246353d2cea5
user='user1.9' description='step 1.9' length=162 offset=8382 (+91) user='user1.9' description='step 1.9' length=162 offset=8414 (+91)
1979-01-03 21:01:08.200001: hash=39dfd11d1f24fb97387aeabf2a013f6ede6385dd 1979-01-03 21:01:08.200001: hash=a2c2c7e3fd68f0f464e8df2e2d657403ebe998b1
user='user1.8' description='step 1.8' length=162 offset=8212 (+91) user='user1.8' description='step 1.8' length=162 offset=8244 (+91)
1979-01-03 21:01:07.100001: hash=9f7327cd475602bca99bf085aabf6051f2573313 1979-01-03 21:01:07.100001: hash=8aaae32a640e94ec46c56604ef727d819174dadf
user='user1.7' description='step 1.7' length=162 offset=8042 (+91) user='user1.7' description='step 1.7' length=162 offset=8074 (+91)
1979-01-03 21:01:06.000001: hash=735edd1e61556098b0268dee9df1cae6a66aa938 1979-01-03 21:01:06.000001: hash=d6a491262e79be892f77442401851e97057a5e1b
user='user1.6' description='step 1.6' length=162 offset=7872 (+91) user='user1.6' description='step 1.6' length=162 offset=7904 (+91)
1979-01-03 21:01:04.900001: hash=43863e7c4b4fb3f1bd400248d71fabbdac9456b6 1979-01-03 21:01:04.900001: hash=467a7e8eb8c9c42d6dd07ce66869a6b549864899
user='user1.5' description='step 1.5' length=162 offset=7702 (+91) user='user1.5' description='step 1.5' length=162 offset=7734 (+91)
1979-01-03 21:01:03.800001: hash=22ee061f2372fd74d03cf5dd54197bc3aa12a816 1979-01-03 21:01:03.800001: hash=5e3e9b8032ac4a1343b115fbfb662bdc5c5778e4
user='user1.4' description='step 1.4' length=162 offset=7532 (+91) user='user1.4' description='step 1.4' length=162 offset=7564 (+91)
1979-01-03 21:01:02.700001: hash=be00cfff67f8f186cce0efb4be7af5525a8fdda7 1979-01-03 21:01:02.700001: hash=cef316831ff36f956440a7586df1a0170e7b8111
user='user1.3' description='step 1.3' length=162 offset=7362 (+91) user='user1.3' description='step 1.3' length=162 offset=7394 (+91)
1979-01-03 21:01:01.600001: hash=152e5811cc260c0e9c28cf6bc0c5fb5c147b4daa 1979-01-03 21:01:01.600001: hash=9844714205a885699405eb241cdb65f8bc08a503
user='user1.2' description='step 1.2' length=162 offset=7192 (+91) user='user1.2' description='step 1.2' length=162 offset=7224 (+91)
1979-01-03 21:01:00.500001: hash=16e51f130683d896e8d182ebdd3bb80ae35d6050 1979-01-03 21:01:00.500001: hash=aa5020c400ecf153ec97c3565fb2948641830fd0
user='user1.1' description='step 1.1' length=162 offset=7022 (+91) user='user1.1' description='step 1.1' length=162 offset=7054 (+91)
1979-01-03 21:00:59.400001: hash=17453ac2d8f8044d959f763516b56eda268e96ad 1979-01-03 21:00:59.400001: hash=3781d0578e07d668c46a4d446425c37924261501
user='user1.0' description='step 1.0' length=162 offset=6852 (+91) user='user1.0' description='step 1.0' length=162 offset=6884 (+91)
1979-01-03 21:00:45.100001: hash=51db603ba9898da20dbe162dd20e4a76aeacbfeb 1979-01-03 21:00:45.100001: hash=51db603ba9898da20dbe162dd20e4a76aeacbfeb
user="root0\nYour\nRoyal\nMagesty' " description='delete 0\nalpha beta gamma\'delta"lambda\n\nqqq ...' length=206 offset=6638 (+156) user="root0\nYour\nRoyal\nMagesty' \x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f" description='delete 0\nalpha beta gamma\'delta"lambda\n\nqqq ...' length=238 offset=6638 (+188)
1979-01-03 21:00:44.000001: hash=469f380f59f9a72dae5ee6351fa2e38efb170d86 1979-01-03 21:00:44.000001: hash=469f380f59f9a72dae5ee6351fa2e38efb170d86
user='' description='predelete 7' length=362 offset=6268 (+34) user='' description='predelete 7' length=362 offset=6268 (+34)
......
t/neo/storage/fs1/py/gen-testdata
View file @ 1eb56978
...@@ -33,6 +33,8 @@ def escapeqq(s): ...@@ -33,6 +33,8 @@ def escapeqq(s):
# we don't want ' to be escaped # we don't want ' to be escaped
for _ in s.split("'"): for _ in s.split("'"):
# this escape almost everything except " character # this escape almost everything except " character
# NOTE string_escape does not do smartquotes and always uses ' for quoting
# (repr(str) is the same except it does smartquoting picking ' or " automatically)
q = _.encode("string_escape") q = _.encode("string_escape")
q = q.replace('"', r'\"') q = q.replace('"', r'\"')
outv.append(q) outv.append(q)
...@@ -154,7 +156,8 @@ def main(): ...@@ -154,7 +156,8 @@ def main():
# Get serial via history. # Get serial via history.
obj_tid_lastchange = db.history(obj._p_oid)[0]['tid'] obj_tid_lastchange = db.history(obj._p_oid)[0]['tid']
txn = precommit(u"root%i\nYour\nRoyal\nMagesty' " % i, txn = precommit(u"root%i\nYour\nRoyal\nMagesty' " % i +
''.join(chr(_) for _ in range(32)), # <- NOTE all control characters
u"delete %i\nalpha beta gamma'delta\"lambda\n\nqqq ..." % i, u"delete %i\nalpha beta gamma'delta\"lambda\n\nqqq ..." % i,
ext("delete %s" % unpack64(obj._p_oid))) ext("delete %s" % unpack64(obj._p_oid)))
stor.tpc_begin(txn) stor.tpc_begin(txn)
......
t/neo/storage/fs1/testdata_expect_test.go
View file @ 1eb56978
This diff is collapsed.
t/neo/zodb/cmd/zodbdump/zodbdump_test.go
View file @ 1eb56978
...@@ -34,16 +34,23 @@ func loadZdumpPy(t *testing.T, path string) string { ...@@ -34,16 +34,23 @@ func loadZdumpPy(t *testing.T, path string) string {
t.Fatal(err) t.Fatal(err)
} }
// python qoutes "\v" as "\x0b", go as "\v"; same for "\f" vs "\x0c" // python qoutes "\v" as "\x0b", go as "\v"; same for "\f", "\a", "\b".
// XXX this is a bit hacky. We could compare quoted strings as decoded, // XXX this is a bit hacky. We could compare quoted strings as decoded,
// but this would need zdump format parser which could contain other // but this would need zdump format parser which could contain other
// bugs. Here we want to compare output ideally bit-to-bit but those // bugs. Here we want to compare output ideally bit-to-bit but those
// \v vs \x0b glitches prevents that to be done directly. So here we // \v vs \x0b glitches prevents that to be done directly. So here we
// are with this ugly hack: // are with this ugly hack:
r0b := regexp.MustCompile(`\\x0b`) var pyNoBackLetter = []struct {backNoLetterRe, backLetter string} {
r0c := regexp.MustCompile(`\\x0c`) {`\\x07`, `\a`},
dump = r0b.ReplaceAllLiteral(dump, []byte(`\v`)) {`\\x08`, `\b`},
dump = r0c.ReplaceAllLiteral(dump, []byte(`\f`)) {`\\x0b`, `\v`},
{`\\x0c`, `\f`},
}
for _, __ := range pyNoBackLetter {
re := regexp.MustCompile(__.backNoLetterRe)
dump = re.ReplaceAllLiteral(dump, []byte(__.backLetter))
}
return string(dump) return string(dump)
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7