Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
erp5
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Labels
Merge Requests
137
Merge Requests
137
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Jobs
Commits
Open sidebar
nexedi
erp5
Commits
d9be3480
Commit
d9be3480
authored
Dec 15, 2016
by
Kazuhiko Shiozaki
1
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
erp5-util: remove zodbanalyze, that is now a part of
https://lab.nexedi.com/nexedi/zodbtools
.
parent
96cb6422
Changes
2
Hide whitespace changes
Inline
Side-by-side
Showing
2 changed files
with
0 additions
and
212 deletions
+0
-212
erp5/util/README.zodbanalyze.txt
erp5/util/README.zodbanalyze.txt
+0
-10
erp5/util/zodbanalyze/__init__.py
erp5/util/zodbanalyze/__init__.py
+0
-202
No files found.
erp5/util/README.zodbanalyze.txt
deleted
100644 → 0
View file @
96cb6422
zodbanalyze
-----------
This is an improved version of ZODB's ZODB/scripts/analyze.py.
* faster
* does not require Products.
* display current/old information for each class
* supports csv output by '-c' or '--csv'
* supports using DBM temporary storage to limit memory usage by '-d' or '--dbm', otherwise memory usage is O(n) of number of OIDs.
erp5/util/zodbanalyze/__init__.py
deleted
100644 → 0
View file @
96cb6422
#!/usr/bin/env python2.4
# Based on a transaction analyzer by Matt Kromer.
import
sys
import
os
import
getopt
import
anydbm
as
dbm
import
tempfile
import
shutil
from
ZODB.FileStorage
import
FileStorage
from
ZODB.utils
import
get_pickle_metadata
class
Report
:
def
__init__
(
self
,
use_dbm
=
False
):
self
.
use_dbm
=
use_dbm
if
use_dbm
:
self
.
temp_dir
=
tempfile
.
mkdtemp
()
self
.
OIDMAP
=
dbm
.
open
(
os
.
path
.
join
(
self
.
temp_dir
,
'oidmap.db'
),
'nf'
)
self
.
USEDMAP
=
dbm
.
open
(
os
.
path
.
join
(
self
.
temp_dir
,
'usedmap.db'
),
'nf'
)
else
:
self
.
OIDMAP
=
{}
self
.
USEDMAP
=
{}
self
.
TYPEMAP
=
{}
self
.
TYPESIZE
=
{}
self
.
TIDS
=
0
self
.
OIDS
=
0
self
.
DBYTES
=
0
self
.
COIDS
=
0
self
.
CBYTES
=
0
self
.
FOIDS
=
0
self
.
FBYTES
=
0
self
.
COIDSMAP
=
{}
self
.
CBYTESMAP
=
{}
self
.
FOIDSMAP
=
{}
self
.
FBYTESMAP
=
{}
def
shorten
(
s
,
n
):
l
=
len
(
s
)
if
l
<=
n
:
return
s
while
len
(
s
)
+
3
>
n
:
# account for ...
i
=
s
.
find
(
"."
)
if
i
==
-
1
:
# In the worst case, just return the rightmost n bytes
return
s
[
-
n
:]
else
:
s
=
s
[
i
+
1
:]
l
=
len
(
s
)
return
"..."
+
s
def
report
(
rep
,
csv
=
False
):
if
not
csv
:
print
"Processed %d records in %d transactions"
%
(
rep
.
OIDS
,
rep
.
TIDS
)
print
"Average record size is %7.2f bytes"
%
(
rep
.
DBYTES
*
1.0
/
rep
.
OIDS
)
print
(
"Average transaction size is %7.2f bytes"
%
(
rep
.
DBYTES
*
1.0
/
rep
.
TIDS
))
print
"Types used:"
if
csv
:
fmt
=
"%s,%s,%s,%s,%s,%s,%s,%s,%s"
fmtp
=
"%s,%d,%d,%f%%,%f,%d,%d,%d,%d"
# per-class format
else
:
fmt
=
"%-46s %7s %9s %6s %7s %7s %9s %7s %9s"
fmtp
=
"%-46s %7d %9d %5.1f%% %7.2f %7d %9d %7d %9d"
# per-class format
fmts
=
"%46s %7d %8dk %5.1f%% %7.2f"
# summary format
print
fmt
%
(
"Class Name"
,
"T.Count"
,
"T.Bytes"
,
"Pct"
,
"AvgSize"
,
"C.Count"
,
"C.Bytes"
,
"O.Count"
,
"O.Bytes"
)
if
not
csv
:
print
fmt
%
(
'-'
*
46
,
'-'
*
7
,
'-'
*
9
,
'-'
*
5
,
'-'
*
7
,
'-'
*
7
,
'-'
*
9
,
'-'
*
7
,
'-'
*
9
)
typemap
=
rep
.
TYPEMAP
.
keys
()
typemap
.
sort
(
key
=
lambda
a
:
rep
.
TYPESIZE
[
a
])
cumpct
=
0.0
for
t
in
typemap
:
pct
=
rep
.
TYPESIZE
[
t
]
*
100.0
/
rep
.
DBYTES
cumpct
+=
pct
if
csv
:
t_display
=
t
else
:
t_display
=
shorten
(
t
,
46
)
print
fmtp
%
(
t_display
,
rep
.
TYPEMAP
[
t
],
rep
.
TYPESIZE
[
t
],
pct
,
rep
.
TYPESIZE
[
t
]
*
1.0
/
rep
.
TYPEMAP
[
t
],
rep
.
COIDSMAP
[
t
],
rep
.
CBYTESMAP
[
t
],
rep
.
FOIDSMAP
.
get
(
t
,
0
),
rep
.
FBYTESMAP
.
get
(
t
,
0
))
if
csv
:
return
print
fmt
%
(
'='
*
46
,
'='
*
7
,
'='
*
9
,
'='
*
5
,
'='
*
7
,
'='
*
7
,
'='
*
9
,
'='
*
7
,
'='
*
9
)
print
"%46s %7d %9s %6s %6.2fk"
%
(
'Total Transactions'
,
rep
.
TIDS
,
' '
,
' '
,
rep
.
DBYTES
*
1.0
/
rep
.
TIDS
/
1024.0
)
print
fmts
%
(
'Total Records'
,
rep
.
OIDS
,
rep
.
DBYTES
/
1024.0
,
cumpct
,
rep
.
DBYTES
*
1.0
/
rep
.
OIDS
)
print
fmts
%
(
'Current Objects'
,
rep
.
COIDS
,
rep
.
CBYTES
/
1024.0
,
rep
.
CBYTES
*
100.0
/
rep
.
DBYTES
,
rep
.
CBYTES
*
1.0
/
rep
.
COIDS
)
if
rep
.
FOIDS
:
print
fmts
%
(
'Old Objects'
,
rep
.
FOIDS
,
rep
.
FBYTES
/
1024.0
,
rep
.
FBYTES
*
100.0
/
rep
.
DBYTES
,
rep
.
FBYTES
*
1.0
/
rep
.
FOIDS
)
def
analyze
(
path
,
use_dbm
):
fs
=
FileStorage
(
path
,
read_only
=
1
)
fsi
=
fs
.
iterator
()
report
=
Report
(
use_dbm
)
for
txn
in
fsi
:
analyze_trans
(
report
,
txn
)
if
use_dbm
:
shutil
.
rmtree
(
report
.
temp_dir
)
return
report
def
analyze_trans
(
report
,
txn
):
report
.
TIDS
+=
1
for
rec
in
txn
:
analyze_rec
(
report
,
rec
)
def
get_type
(
record
):
mod
,
klass
=
get_pickle_metadata
(
record
.
data
)
return
"%s.%s"
%
(
mod
,
klass
)
def
analyze_rec
(
report
,
record
):
oid
=
record
.
oid
report
.
OIDS
+=
1
if
record
.
data
is
None
:
# No pickle -- aborted version or undo of object creation.
return
try
:
size
=
len
(
record
.
data
)
# Ignores various overhead
report
.
DBYTES
+=
size
if
oid
not
in
report
.
OIDMAP
:
type
=
get_type
(
record
)
report
.
OIDMAP
[
oid
]
=
type
if
report
.
use_dbm
:
report
.
USEDMAP
[
oid
]
=
str
(
size
)
else
:
report
.
USEDMAP
[
oid
]
=
size
report
.
COIDS
+=
1
report
.
CBYTES
+=
size
report
.
COIDSMAP
[
type
]
=
report
.
COIDSMAP
.
get
(
type
,
0
)
+
1
report
.
CBYTESMAP
[
type
]
=
report
.
CBYTESMAP
.
get
(
type
,
0
)
+
size
else
:
type
=
report
.
OIDMAP
[
oid
]
if
report
.
use_dbm
:
fsize
=
int
(
report
.
USEDMAP
[
oid
])
report
.
USEDMAP
[
oid
]
=
str
(
size
)
else
:
fsize
=
report
.
USEDMAP
[
oid
]
report
.
USEDMAP
[
oid
]
=
size
report
.
FOIDS
+=
1
report
.
FBYTES
+=
fsize
report
.
CBYTES
+=
size
-
fsize
report
.
FOIDSMAP
[
type
]
=
report
.
FOIDSMAP
.
get
(
type
,
0
)
+
1
report
.
FBYTESMAP
[
type
]
=
report
.
FBYTESMAP
.
get
(
type
,
0
)
+
fsize
report
.
CBYTESMAP
[
type
]
=
report
.
CBYTESMAP
.
get
(
type
,
0
)
+
size
-
fsize
report
.
TYPEMAP
[
type
]
=
report
.
TYPEMAP
.
get
(
type
,
0
)
+
1
report
.
TYPESIZE
[
type
]
=
report
.
TYPESIZE
.
get
(
type
,
0
)
+
size
except
Exception
,
err
:
print
err
__doc__
=
"""%(program)s: Data.fs analyzer
usage: %(program)s [options] /path/to/Data.fs
Options:
-h, --help this help screen
-c, --csv output CSV
-d, --dbm use DBM as temporary storage to limit memory usage
"""
def
usage
(
stream
,
msg
=
None
):
if
msg
:
print
>>
stream
,
msg
print
>>
stream
program
=
os
.
path
.
basename
(
sys
.
argv
[
0
])
print
>>
stream
,
__doc__
%
{
"program"
:
program
}
def
main
():
try
:
opts
,
args
=
getopt
.
getopt
(
sys
.
argv
[
1
:],
'hcd'
,
[
'help'
,
'csv'
,
'dbm'
])
path
=
args
[
0
]
except
(
getopt
.
GetoptError
,
IndexError
),
msg
:
usage
(
sys
.
stderr
,
msg
)
sys
.
exit
(
2
)
csv
=
False
use_dbm
=
False
for
opt
,
args
in
opts
:
if
opt
in
(
'-c'
,
'--csv'
):
csv
=
True
if
opt
in
(
'-d'
,
'--dbm'
):
use_dbm
=
True
if
opt
in
(
'-h'
,
'--help'
):
usage
(
sys
.
stdout
)
sys
.
exit
()
report
(
analyze
(
path
,
use_dbm
),
csv
)
if
__name__
==
"__main__"
:
main
()
Kazuhiko Shiozaki
@kazuhiko
mentioned in merge request
slapos!121 (merged)
·
Dec 15, 2016
mentioned in merge request
slapos!121 (merged)
mentioned in merge request nexedi/slapos!121
Toggle commit list
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment