Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
W
wendelin
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
Levin Zimmermann
wendelin
Commits
a3f99c16
Commit
a3f99c16
authored
May 17, 2018
by
Ivan Tyagov
Committed by
Klaus Wölfel
May 29, 2018
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Add Data Bucket Stream to consistency checks.
parent
c101f1ac
Changes
4
Show whitespace changes
Inline
Side-by-side
Showing
4 changed files
with
106 additions
and
21 deletions
+106
-21
bt5/erp5_neo_administration/ExtensionTemplateItem/portal_components/extension.erp5.NEOAdministration.py
...tem/portal_components/extension.erp5.NEOAdministration.py
+42
-0
bt5/erp5_neo_administration/SkinTemplateItem/portal_skins/erp5_neo_administration/DataBucketStream_getChecksumListFromNEONodeListForKey.xml
...DataBucketStream_getChecksumListFromNEONodeListForKey.xml
+28
-0
bt5/erp5_neo_administration/SkinTemplateItem/portal_skins/erp5_neo_administration/DataStream_checkIfNEOCloneBackupIsConsistent.py
...istration/DataStream_checkIfNEOCloneBackupIsConsistent.py
+29
-15
bt5/erp5_neo_administration/SkinTemplateItem/portal_skins/erp5_neo_administration/ERP5Site_checkNEOCloneBackupConsistency.py
...administration/ERP5Site_checkNEOCloneBackupConsistency.py
+7
-6
No files found.
bt5/erp5_neo_administration/ExtensionTemplateItem/portal_components/extension.erp5.NEOAdministration.py
View file @
a3f99c16
...
...
@@ -6,6 +6,48 @@ from ZODB import DB
from
neo.client.Storage
import
Storage
import
hashlib
def
DataBucketStream_getChecksumListFromNEONodeListForKey
(
self
,
\
node_list
,
\
ca_file
,
\
cert_file
,
\
key_file
,
\
key
,
\
threshold
):
"""
Directly connect to NEO backends and check checksums of this Data Bucket Stream for this key.
"""
checksum_list
=
[]
# get directly checksum as we have access to data stream over self
data
=
self
.
getBucket
(
key
)
data
=
data
[:
threshold
]
checksum
=
hashlib
.
sha256
(
data
).
hexdigest
()
checksum_list
.
append
(
checksum
)
for
node
in
node_list
:
kw
=
{
'master_nodes'
:
node
[
0
],
'name'
:
node
[
1
],
'ca'
:
ca_file
,
'cert'
:
cert_file
,
'key'
:
key_file
}
# make a direct connection
stor
=
Storage
(
**
kw
)
db
=
DB
(
stor
)
conn
=
db
.
open
()
root
=
conn
.
root
()
data_stream_id
=
self
.
getId
()
data_stream
=
root
[
'Application'
].
erp5
.
data_stream_module
[
data_stream_id
]
data
=
data_stream
.
getBucket
(
key
)
data
=
data
[:
threshold
]
conn
.
close
()
db
.
close
()
checksum
=
hashlib
.
sha256
(
data
).
hexdigest
()
checksum_list
.
append
(
checksum
)
return
checksum_list
def
DataStream_getChecksumListFromNEONodeListForStartStopOffset
(
self
,
\
node_list
,
\
ca_file
,
\
...
...
bt5/erp5_neo_administration/SkinTemplateItem/portal_skins/erp5_neo_administration/DataBucketStream_getChecksumListFromNEONodeListForKey.xml
0 → 100644
View file @
a3f99c16
<?xml version="1.0"?>
<ZopeData>
<record
id=
"1"
aka=
"AAAAAAAAAAE="
>
<pickle>
<global
name=
"ExternalMethod"
module=
"Products.ExternalMethod.ExternalMethod"
/>
</pickle>
<pickle>
<dictionary>
<item>
<key>
<string>
_function
</string>
</key>
<value>
<string>
DataBucketStream_getChecksumListFromNEONodeListForKey
</string>
</value>
</item>
<item>
<key>
<string>
_module
</string>
</key>
<value>
<string>
NEOAdministration
</string>
</value>
</item>
<item>
<key>
<string>
id
</string>
</key>
<value>
<string>
DataBucketStream_getChecksumListFromNEONodeListForKey
</string>
</value>
</item>
<item>
<key>
<string>
title
</string>
</key>
<value>
<string></string>
</value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
bt5/erp5_neo_administration/SkinTemplateItem/portal_skins/erp5_neo_administration/DataStream_checkIfNEOCloneBackupIsConsistent.py
View file @
a3f99c16
"""
Check replication of Data Stream's content amongst NEO clones.
Check replication of Data Stream's
or Data Bucket Stream's
content amongst NEO clones.
"""
import
random
# sometimes we have HUGE streams so we need to check some randomly generated
# portion of a Data Stream
max_stop_offset
=
context
.
getSize
()
start_offset
=
int
(
random
.
random
()
*
max_stop_offset
)
stop_offset
=
start_offset
+
threshold
if
stop_offset
>
max_stop_offset
:
portal_type
=
context
.
getPortalType
()
if
portal_type
==
'Data Stream'
:
# sometimes we have HUGE streams so we need to check some randomly generated
# portion of a Data Stream
max_stop_offset
=
context
.
getSize
()
start_offset
=
int
(
random
.
random
()
*
max_stop_offset
)
stop_offset
=
start_offset
+
threshold
if
stop_offset
>
max_stop_offset
:
stop_offset
=
max_stop_offset
checksum_list
=
context
.
DataStream_getChecksumListFromNEONodeListForStartStopOffset
(
checksum_list
=
context
.
DataStream_getChecksumListFromNEONodeListForStartStopOffset
(
neo_node_list
,
neo_cert_list
[
0
],
neo_cert_list
[
1
],
neo_cert_list
[
2
],
start_offset
,
stop_offset
)
elif
portal_type
==
'Data Bucket Stream'
:
# choose a random bucket key as raw data source
key_list
=
context
.
getKeyList
()
# choose random key
key
=
random
.
choice
(
key_list
)
checksum_list
=
context
.
DataBucketStream_getChecksumListFromNEONodeListForKey
(
neo_node_list
,
neo_cert_list
[
0
],
neo_cert_list
[
1
],
neo_cert_list
[
2
],
key
,
threshold
)
if
len
(
set
(
checksum_list
))
>
1
:
# one of checksums didn't match
print
"PROBLEM:"
,
context
.
getRelativeUrl
(),
checksum_list
...
...
bt5/erp5_neo_administration/SkinTemplateItem/portal_skins/erp5_neo_administration/ERP5Site_checkNEOCloneBackupConsistency.py
View file @
a3f99c16
...
...
@@ -15,10 +15,11 @@ active_process.setTitle("NEO_Clone_check")
# which Data Streams to check ...
data_stream_list
=
context
.
ERP5Site_getDataStreamListToCheck
()
# we can only check if we have some data inside thus filter out
data_stream_list
=
[
x
for
x
in
data_stream_list
if
x
.
getSize
()
>
0
]
for
data_stream
in
data_stream_list
:
portal_type
=
data_stream
.
getPortalType
()
if
((
portal_type
==
"Data Stream"
and
data_stream
.
getSize
()
>
0
)
or
(
portal_type
==
"Data Bucket Stream"
and
len
(
data_stream
.
getKeyList
())
>
0
)):
# Data Stream or Data Bucket Stream needs to have data ...
tag
=
'%s_consistency_check'
%
data_stream
.
getPath
()
data_stream
.
activate
(
tag
=
tag
,
active_process
=
active_process
.
getPath
()).
DataStream_checkIfNEOCloneBackupIsConsistent
(
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment