Skip to content

W
wendelin
Commit 925f3deb authored by Ivan Tyagov's avatar Ivan Tyagov
Browse files
Options

Remove an example that stored twice the RAM memory zeros in ZODB's root as... 

Remove an example that stored twice the RAM memory zeros in ZODB's root as this can quickly run out of space any Wendelin instance.
Use ERP5-ish API to work with Data Arrays
parent 7293de58
Hide whitespace changes
Inline Side-by-side
Showing with 37 additions and 292 deletions
bt5/erp5_wendelin_examples/ExtensionTemplateItem/portal_components/extension.erp5.WendelinExamples.py
View file @ 925f3deb
...@@ -2,21 +2,17 @@ ...@@ -2,21 +2,17 @@
""" """
Sample Wendelin example test code. Sample Wendelin example test code.
""" """
from wendelin.bigarray.array_zodb import ZBigArray
import transaction import transaction
import numpy as np import numpy as np
from numpy import float64, dtype, cumsum, sin, uint8 from numpy import uint8
import psutil import random
import os import string
# generate signal S(t) = M⋅sin(f⋅t)
f = 0.2
M = 15
KB = 1024
MB = 1024*KB
GB = 1024*MB
def getRandomString():
return 'test_%s' %''.join([random.choice(string.ascii_letters + string.digits) \
for n in xrange(32)])
# Game of Life examples # Game of Life examples
default_input_ndarray = np.array([[0,0,0,0,0,0], default_input_ndarray = np.array([[0,0,0,0,0,0],
[0,0,0,1,0,0], [0,0,0,1,0,0],
...@@ -24,81 +20,6 @@ default_input_ndarray = np.array([[0,0,0,0,0,0], ...@@ -24,81 +20,6 @@ default_input_ndarray = np.array([[0,0,0,0,0,0],
[0,0,1,1,0,0], [0,0,1,1,0,0],
[0,0,0,0,0,0], [0,0,0,0,0,0],
[0,0,0,0,0,0]]) [0,0,0,0,0,0]])
def read(signalv):
"""
Read zbigarray from ZODB and compute its average/var/sum.
To run use ERP5Site_readZBigArray external method.
"""
message_list = []
message_list.append('sig: %s %s (= %.2fGB)' % \
('x'.join('%s' % _ for _ in signalv.shape),
signalv.dtype, float(signalv.nbytes) / GB))
a = signalv[:] # BigArray -> ndarray
message_list.append('<sig>:\t%s' % a.mean())
#print('δ(sig):\t%s' % a.var()) # XXX wants to produce temps (var = S (a - <a>)^2
message_list.append('S(sig):\t%s' % a.sum())
#cumsum(a) # cummulative sum which will cause memor errors thus disabled!
message_list.append('S(sig):\t%s' % a.sum())
return message_list
def generate(signalv):
"""
Generate a ZBigArray instance bigger than RAM and saves to ZODB.
To run use external method ERP5Site_generateBigArray
"""
message_list = []
message_list.append('gen signal t=0...%.2e %s (= %.2fGB) ' % \
(len(signalv), signalv.dtype, float(signalv.nbytes) / GB))
a = signalv[:] # BigArray -> ndarray
blocksize = 32*MB//a.itemsize # will write out signal in such blocks
for t0 in xrange(0, len(a), blocksize):
ablk = a[t0:t0+blocksize]
ablk[:] = 1 # at = 1
cumsum(ablk, out=ablk) # at = t-t0+1
ablk += (t0-1) # at = t
ablk *= f # at = f⋅t
sin(ablk, out=ablk) # at = sin(f⋅t)
ablk *= M # at = M⋅sin(f⋅t)
note = 'gen signal blk [%s:%s] (%.1f%%)' % (t0, t0+len(ablk), 100. * (t0+len(ablk)) / len(a))
txn = transaction.get()
txn.note(note)
txn.commit()
message_list.append(note)
return message_list
def zbig_array(self, act):
message_list = []
ram_nbytes = psutil.virtual_memory().total
message_list.append('I: RAM: %.2fGB' % (float(ram_nbytes) / GB))
# let us store in the very root of ZODB
root = self.getPhysicalRoot()
if act == 'generate':
sig_dtype = dtype(float64)
sig_len = (2*ram_nbytes) // sig_dtype.itemsize
sig = ZBigArray((sig_len,), sig_dtype)
root.big_array = sig
# ZBigArray requirement: before we can compute it (with subobject
# .zfile) have to be made explicitly known to connection or current
# transaction committed
transaction.commit()
message_list.extend(generate(sig))
elif act == 'read':
message_list.extend(read(root.big_array))
p = psutil.Process(os.getpid())
m = p.memory_info()
message_list.append('VIRT: %i MB\tRSS: %iMB' % (m.vms//MB, m.rss//MB))
return '\n'.join(message_list)
def iterate_2(Z): def iterate_2(Z):
# Count neighbours # Count neighbours
N = (Z[0:-2,0:-2] + Z[0:-2,1:-1] + Z[0:-2,2:] + N = (Z[0:-2,0:-2] + Z[0:-2,1:-1] + Z[0:-2,2:] +
...@@ -119,27 +40,19 @@ def game_of_life_out_of_core(self): ...@@ -119,27 +40,19 @@ def game_of_life_out_of_core(self):
Using "out-of-core" ndarray stored in Zope's ZODB. Using "out-of-core" ndarray stored in Zope's ZODB.
""" """
print_list = [] print_list = []
array_reference = 'game_of_life_out_of_core_%s' %getRandomString()
# let us store in the very root of ZODB data_array = self.data_array_module.newContent(
root = self.getPhysicalRoot() portal_type='Data Array',
reference = array_reference,
# create ZBigArray and hook it into DB. version = '001')
# please note that it's persistent and never clean up from ZODB so next data_array.initArray((6,6), uint8)
# execution will raise unless removed from ZODB!
zlife_area = ZBigArray((6,6), uint8) life_area = data_array.getArray()[:,:] # ZBigArray -> ndarray view of it
root.life_area_out_of_core = zlife_area
# ZBigArray requirement: before we can compute it (with subobject
# .zfile) have to be made explicitly known to connection or current
# transaction committed
transaction.commit()
life_area = root.life_area_out_of_core # our ZBigArray
life_area = zlife_area[:,:] # ZBigArray -> ndarray view of it
# change the data, exactly as if working with numpy.ndarray but # change the data, exactly as if working with numpy.ndarray but
# still referencing the ndarray of ZBigArray # still referencing the ndarray of ZBigArray
life_area[:,:] = default_input_ndarray life_area[:,:] = default_input_ndarray
transaction.commit()
print_list.append(str(life_area)) print_list.append(str(life_area))
for i in range(4): for i in range(4):
...@@ -148,13 +61,15 @@ def game_of_life_out_of_core(self): ...@@ -148,13 +61,15 @@ def game_of_life_out_of_core(self):
return '\n\n'.join(print_list) return '\n\n'.join(print_list)
def ERP5Site_gameOfLife(self, ndarray_id): def ERP5Site_gameOfLife(self, array_reference):
""" """
Run inside an activity and manipulate directly ndarray inside ZODB. Run inside an activity and manipulate directly ndarray inside ZODB.
""" """
root = self.getPhysicalRoot() data_array = self.portal_catalog.getResultValue(
zlife_area = getattr(root, ndarray_id) # our ZBigArray portal_type = 'Data Array',
life_area = zlife_area[:,:] # ZBigArray -> ndarray view of it reference = array_reference)
life_area = data_array.getArray()[:,:] # ZBigArray -> ndarray view of it
# we make a copy, with big volumes maybe not wise # we make a copy, with big volumes maybe not wise
input_life_area = np.copy(life_area) input_life_area = np.copy(life_area)
...@@ -165,7 +80,7 @@ def ERP5Site_gameOfLife(self, ndarray_id): ...@@ -165,7 +80,7 @@ def ERP5Site_gameOfLife(self, ndarray_id):
if not (input_life_area==life_area).all(): if not (input_life_area==life_area).all():
# input array not equals output array, in this case we can continue # input array not equals output array, in this case we can continue
# until we find a solution or an end stage # until we find a solution or an end stage
self.portal_activities.activate().ERP5Site_gameOfLife(ndarray_id) self.portal_activities.activate().ERP5Site_gameOfLife(array_reference)
def game_of_life_out_of_core_activities(self): def game_of_life_out_of_core_activities(self):
""" """
...@@ -174,29 +89,22 @@ def game_of_life_out_of_core_activities(self): ...@@ -174,29 +89,22 @@ def game_of_life_out_of_core_activities(self):
Using "out-of-core" ndarray stored in Zope's ZODB and activities for load Using "out-of-core" ndarray stored in Zope's ZODB and activities for load
balancing. balancing.
""" """
# let us store in the very root of ZODB array_reference = 'life_area_out_of_core_persistent_%s' %getRandomString()
root = self.getPhysicalRoot() data_array = self.data_array_module.newContent(
portal_type='Data Array',
# create ZBigArray and hook it into DB. reference = array_reference,
# please note that it's persistent and never clean up from ZODB so next version = '001')
# execution will raise unless removed from ZODB! data_array.initArray((6,6), uint8)
zlife_area = ZBigArray((6,6), uint8)
root.life_area_out_of_core_persistent = zlife_area life_area = data_array.getArray()[:,:] # ZBigArray -> ndarray view of it
# ZBigArray requirement: before we can compute it (with subobject
# .zfile) have to be made explicitly known to connection or current
# transaction committed
transaction.commit()
array_id = 'life_area_out_of_core_persistent'
life_area = getattr(root, array_id) # our ZBigArray
life_area = zlife_area[:,:] # ZBigArray -> ndarray view of it
# initialise # initialise
life_area[:,:] = default_input_ndarray life_area[:,:] = default_input_ndarray
transaction.commit()
# start calculation in background using activities # start calculation in background using activities
self.portal_activities.activate().ERP5Site_gameOfLife(array_id) self.portal_activities.activate().ERP5Site_gameOfLife(array_reference)
return 'Started' return 'Started'
......
bt5/erp5_wendelin_examples/ExtensionTemplateItem/portal_components/extension.erp5.WendelinExamples.xml
View file @ 925f3deb
...@@ -46,8 +46,9 @@ ...@@ -46,8 +46,9 @@
<key> <string>text_content_warning_message</string> </key> <key> <string>text_content_warning_message</string> </key>
<value> <value>
<tuple> <tuple>
<string>W:145, 6: Unused variable \'i\' (unused-variable)</string> <string>W: 14, 8: Unused variable \'n\' (unused-variable)</string>
<string>W:212, 6: Unused variable \'i\' (unused-variable)</string> <string>W: 58, 6: Unused variable \'i\' (unused-variable)</string>
<string>W:120, 6: Unused variable \'i\' (unused-variable)</string>
</tuple> </tuple>
</value> </value>
</item> </item>
......
bt5/erp5_wendelin_examples/SkinTemplateItem/portal_skins/erp5_wendelin_examples/ERP5Site_generateZBigArray.xml deleted 100644 → 0
View file @ 7293de58
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>Script_magic</string> </key>
<value> <int>3</int> </value>
</item>
<item>
<key> <string>_bind_names</string> </key>
<value>
<object>
<klass>
<global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key> <string>_asgns</string> </key>
<value>
<dictionary>
<item>
<key> <string>name_container</string> </key>
<value> <string>container</string> </value>
</item>
<item>
<key> <string>name_context</string> </key>
<value> <string>context</string> </value>
</item>
<item>
<key> <string>name_m_self</string> </key>
<value> <string>script</string> </value>
</item>
<item>
<key> <string>name_subpath</string> </key>
<value> <string>traverse_subpath</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>_body</string> </key>
<value> <string>print context.ERP5Site_zBigArray(\'generate\')\n
return printed\n
</string> </value>
</item>
<item>
<key> <string>_params</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>ERP5Site_generateZBigArray</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
bt5/erp5_wendelin_examples/SkinTemplateItem/portal_skins/erp5_wendelin_examples/ERP5Site_readZBigArray.xml deleted 100644 → 0
View file @ 7293de58
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>Script_magic</string> </key>
<value> <int>3</int> </value>
</item>
<item>
<key> <string>_bind_names</string> </key>
<value>
<object>
<klass>
<global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key> <string>_asgns</string> </key>
<value>
<dictionary>
<item>
<key> <string>name_container</string> </key>
<value> <string>container</string> </value>
</item>
<item>
<key> <string>name_context</string> </key>
<value> <string>context</string> </value>
</item>
<item>
<key> <string>name_m_self</string> </key>
<value> <string>script</string> </value>
</item>
<item>
<key> <string>name_subpath</string> </key>
<value> <string>traverse_subpath</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>_body</string> </key>
<value> <string>print context.ERP5Site_zBigArray(\'read\')\n
return printed\n
</string> </value>
</item>
<item>
<key> <string>_params</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>ERP5Site_readZBigArray</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
bt5/erp5_wendelin_examples/SkinTemplateItem/portal_skins/erp5_wendelin_examples/ERP5Site_zBigArray.xml deleted 100644 → 0
View file @ 7293de58
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="ExternalMethod" module="Products.ExternalMethod.ExternalMethod"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_function</string> </key>
<value> <string>zbig_array</string> </value>
</item>
<item>
<key> <string>_module</string> </key>
<value> <string>WendelinExamples</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>ERP5Site_zBigArray</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7