Skip to content

slapos
slapos
Commit 816840c4 authored by Julien Muchembled's avatar Julien Muchembled
Browse files
Options

wendelin: 2 new scalability tests

parent dbb51ac9
Show whitespace changes
Inline Side-by-side
Showing with 709 additions and 0 deletions
software/wendelin-scalability/test-fluentd-1pb.cfg 0 → 100644
View file @ 816840c4
[buildout]
extends = test-fluentd-common.cfg
[template-erp5-patched]
extra =
{%- for family_name, zope_section_id_list in zope_family_dict.items() %}
{%- for zope_section_id in zope_section_id_list %}
[{{zope_section_id}}]
config-family = {{family_name}}
{%- endfor %}
{%- endfor %}
[template-zope]
recipe =
target = ${template-zope-patched:location}
[template-zope-base]
<= template-zope
recipe = slapos.recipe.build:download
[template-zope-patched]
<= template-fluentd
base = ${template-zope-base:target}
method = unpack
tags =
{%- if slapparameter_dict['family'] == 'fluentd' %}
{%- for i, zope in enumerate(publish_list) %}
{%- do tags.__setitem__(
node_id_base ~ (node_id_index_format % i) ~ '-stream', zope[0]) %}
{%- endfor %}
{%- endif %}
[feeder]
feeder =
#
import os, struct
from random import lognormvariate
pack = struct.Struct('!d').pack
size = 65536
data = os.urandom(size - 8)
while True:
emit('',
''.join((pack(time.time()) + data[:int(lognormvariate(10, 1))]
).ljust(size, '\0') for i in xrange(16)))
time.sleep(interval)
software/wendelin-scalability/test-fluentd-common.cfg 0 → 100644
View file @ 816840c4
[buildout]
extends =
../../component/gnupg/buildout.cfg
../fluentd/software.cfg
../wendelin/software.cfg
../../component/mariadb/mariarocks.cfg
[local-bt5-repository]
list += ${slapos.cookbook-repository:location}/software/wendelin-scalability
[patch-template]
recipe = slapos.recipe.build
location = ${buildout:directory}/${:_buildout_section_name_}.cfg.in
script =
with open(self.options['location'], 'w') as dst, \
open(self.options['base']) as src:
src = src.read()
i = src.index('[buildout]')
dst.write(src[:i] + self.options['extra'] + '\n' + src[i:])
[template-erp5]
recipe =
target = ${template-erp5-patched:location}
[template-erp5-base]
<= template-erp5
recipe = slapos.recipe.build:download
[template-erp5-patched]
<= patch-template
base = ${template-erp5-base:target}
[template-fluentd]
<= patch-template
extra =
{%- set tags = {} %}
${:tags}
{%- if tags %}
[feeder]
recipe = slapos.cookbook:wrapper
wrapper-path = $${directory:services}/$${:_buildout_section_name_}
command-line = ${feeder:bin-directory}/feeder $${fluentd-agent-conf:rendered}
$${:_buildout_section_name_} .4
{%- for tag in tags %}
[{{ section(tag) }}]
<= feeder
{%- endfor %}
[{{ section('fluentd') }}]
recipe = slapos.cookbook:wrapper
wrapper-path = $${directory:services}/$${:_buildout_section_name_}
command-line = ${fluentd:location}/bin/fluentd
-v -c $${fluentd-agent-conf:rendered}
environment =
GEM_PATH=${fluentd:location}/lib/ruby/gems/1.8
[fluentd-agent-conf]
recipe = slapos.recipe.template:jinja2
rendered = $${directory:etc}/fluentd-agent.conf
template =
inline:{% raw -%}
<match debug.**>
@type stdout
@id stdout_output
</match>
<source>
# in_exec is unusable because it logs dropped events in case of buffer overflow
@type unix
path $${directory:run}/fluentd.sock
</source>
{%- for tag, netloc in tags.iteritems() %}
<match {{ tag }}>
@type wendelin
@id {{ tag }}
streamtool_uri http://{{ netloc }}/erp5/portal_ingestion_policies/scalability_test_${:method}
user {{ user }}
password {{ password }}
buffer_type memory
buffer_queue_limit 4
</match>
{%- endfor %}
{%- endraw %}
context =
key tags :tags
raw user {{ slapparameter_dict['inituser-login'] }}
raw password {{ slapparameter_dict['inituser-password'] }}
tags = {{ dumps(tags) }}
{%- endif %}
[feeder]
recipe = zc.recipe.egg
eggs = fluent-logger
entry-points = ${:_buildout_section_name_}=__main__:main
initialization =
import re, time, urlparse
from urllib import FancyURLopener
from fluent.sender import FluentSender
def main():
conf, tag, interval = sys.argv[1:]
with open(conf) as f:
for k, v in re.compile(r"<(.+?)>\s*(.*?)\s*</.+?>",
re.S).findall(f.read()):
v = dict(v.split() for v in v.splitlines())
if k == 'source':
sock = v['path']
elif v['@type'] == 'wendelin':
if k.split(None, 1)[1] == tag:
conf = v
emit = FluentSender(tag, host='unix://'+sock, timeout=None,
msgpack_kwargs={'use_bin_type': True}).emit
url = urlparse.urlsplit(conf['streamtool_uri'])
url = urlparse.urlunsplit(url._replace(
netloc='%s:%s@%s' % (conf['user'], conf['password'], url.netloc),
path='/erp5/isPortalBeingCreated'))
urlopen = FancyURLopener().open
while True:
try:
f = urlopen(url)
except IOError:
pass
else:
try:
if not f.read():
break
finally:
f.close()
time.sleep(5)
interval = float(interval)
${:feeder}
[versions]
fluent-logger = 0.5.3
software/wendelin-scalability/test-much-deduplication-high-compressibility.cfg 0 → 100644
View file @ 816840c4
[buildout]
extends = test-fluentd-common.cfg
[template-erp5-patched]
extra =
[request-balancer]
config-inituser-login = {{ dumps(inituser_login) }}
config-inituser-password = $${publish-early:inituser-password}
[template-balancer]
recipe =
target = ${template-balancer-patched:location}
[template-balancer-base]
<= template-balancer
recipe = slapos.recipe.build:download
[template-balancer-patched]
<= template-fluentd
base = ${template-balancer-base:target}
method = raw_lf
tags =
{%- set port, backend_list = haproxy_dict['fluentd'] %}
{%- for i in range(len(backend_list)) %}
{%- do tags.__setitem__('wendelin_tag_' ~ i, ipv4 ~ ':' ~ port) %}
{%- endfor %}
[data.gpg]
recipe = slapos.recipe.build:download
url = http://download.shacache.org/0f56a59ab3bc3193e70c1996fe69260e1b225bc53bbb7509cb3ae036006653dec69b3384cd624eefed81c85e692e4b4b7cb3d25c6244c298070569dddf7216b1
md5sum = 374e00dfbd28f27819ccfff31d169c9c
[gpg-decrypt]
recipe = slapos.recipe.build
gpg = ${gnupg:location}/bin/gpg
script =
import subprocess, zc.buildout
options = self.options
args = (options['gpg'], '-d', '--batch', '--passphrase-fd', '0',
'-o', options['location'], options['input'])
p = subprocess.Popen(args, stdin=subprocess.PIPE)
p.communicate(options['password'])
retcode = p.poll()
if retcode:
raise subprocess.CalledProcessError(retcode, ' '.join(args))
[data]
<= gpg-decrypt
location = ${data.gpg:location}/data.log
input = ${data.gpg:target}
[feeder]
feeder =
#
import mmap, os, random
data = mmap.mmap(os.open('${data:location}', os.O_RDONLY),
0, prot=mmap.PROT_READ)
size = 1024 * 1024
max_seek = len(data) - size
while True:
a = random.randint(0, max_seek)
a = {'data': data[a:a+size]}
for i in xrange(20):
emit('', a)
time.sleep(interval)
software/wendelin-scalability/test_scalability_fluentd/PathTemplateItem/portal_ingestion_policies/scalability_test_raw_lf.xml 0 → 100644
View file @ 816840c4
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Ingestion Policy" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_Access_contents_information_Permission</string> </key>
<value>
<tuple>
<string>Assignee</string>
<string>Assignor</string>
<string>Associate</string>
<string>Auditor</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Add_portal_content_Permission</string> </key>
<value>
<tuple>
<string>Assignee</string>
<string>Assignor</string>
<string>Associate</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Modify_portal_content_Permission</string> </key>
<value>
<tuple>
<string>Assignee</string>
<string>Assignor</string>
<string>Associate</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_View_Permission</string> </key>
<value>
<tuple>
<string>Assignee</string>
<string>Assignor</string>
<string>Associate</string>
<string>Auditor</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>scalability_test_raw_lf</string> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Ingestion Policy</string> </value>
</item>
<item>
<key> <string>script_id</string> </key>
<value> <string>ERP5Site_handleRawDataFluentdIngestion</string> </value>
</item>
<item>
<key> <string>workflow_history</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="2" aka="AAAAAAAAAAI=">
<pickle>
<global name="PersistentMapping" module="Persistence.mapping"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>data</string> </key>
<value>
<dictionary>
<item>
<key> <string>validation_workflow</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent>
</value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="3" aka="AAAAAAAAAAM=">
<pickle>
<global name="WorkflowHistoryList" module="Products.ERP5Type.patches.WorkflowTool"/>
</pickle>
<pickle>
<tuple>
<none/>
<list>
<dictionary>
<item>
<key> <string>action</string> </key>
<value> <string>validate</string> </value>
</item>
<item>
<key> <string>actor</string> </key>
<value> <string>zope</string> </value>
</item>
<item>
<key> <string>comment</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>error_message</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>time</string> </key>
<value>
<object>
<klass>
<global name="DateTime" module="DateTime.DateTime"/>
</klass>
<tuple>
<none/>
</tuple>
<state>
<tuple>
<float>1507057895.87</float>
<string>UTC</string>
</tuple>
</state>
</object>
</value>
</item>
<item>
<key> <string>validation_state</string> </key>
<value> <string>validated</string> </value>
</item>
</dictionary>
</list>
</tuple>
</pickle>
</record>
</ZopeData>
software/wendelin-scalability/test_scalability_fluentd/PathTemplateItem/portal_ingestion_policies/scalability_test_unpack.xml 0 → 100644
View file @ 816840c4
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Ingestion Policy" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_Access_contents_information_Permission</string> </key>
<value>
<tuple>
<string>Assignee</string>
<string>Assignor</string>
<string>Associate</string>
<string>Auditor</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Add_portal_content_Permission</string> </key>
<value>
<tuple>
<string>Assignee</string>
<string>Assignor</string>
<string>Associate</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Modify_portal_content_Permission</string> </key>
<value>
<tuple>
<string>Assignee</string>
<string>Assignor</string>
<string>Associate</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_View_Permission</string> </key>
<value>
<tuple>
<string>Assignee</string>
<string>Assignor</string>
<string>Associate</string>
<string>Auditor</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>description</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>scalability_test_unpack</string> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Ingestion Policy</string> </value>
</item>
<item>
<key> <string>script_id</string> </key>
<value> <string>ERP5Site_handleRawDataFluentdIngestion</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>workflow_history</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="2" aka="AAAAAAAAAAI=">
<pickle>
<global name="PersistentMapping" module="Persistence.mapping"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>data</string> </key>
<value>
<dictionary>
<item>
<key> <string>validation_workflow</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent>
</value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="3" aka="AAAAAAAAAAM=">
<pickle>
<global name="WorkflowHistoryList" module="Products.ERP5Type.patches.WorkflowTool"/>
</pickle>
<pickle>
<tuple>
<none/>
<list>
<dictionary>
<item>
<key> <string>action</string> </key>
<value> <string>validate</string> </value>
</item>
<item>
<key> <string>actor</string> </key>
<value> <string>zope</string> </value>
</item>
<item>
<key> <string>comment</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>error_message</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>time</string> </key>
<value>
<object>
<klass>
<global name="DateTime" module="DateTime.DateTime"/>
</klass>
<tuple>
<none/>
</tuple>
<state>
<tuple>
<float>1507057913.93</float>
<string>UTC</string>
</tuple>
</state>
</object>
</value>
</item>
<item>
<key> <string>validation_state</string> </key>
<value> <string>validated</string> </value>
</item>
</dictionary>
</list>
</tuple>
</pickle>
</record>
</ZopeData>
software/wendelin-scalability/test_scalability_fluentd/PathTemplateItem/portal_skins/custom/DataStreamModule_getTotalSize.xml 0 → 100644
View file @ 816840c4
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>Script_magic</string> </key>
<value> <int>3</int> </value>
</item>
<item>
<key> <string>_bind_names</string> </key>
<value>
<object>
<klass>
<global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key> <string>_asgns</string> </key>
<value>
<dictionary>
<item>
<key> <string>name_container</string> </key>
<value> <string>container</string> </value>
</item>
<item>
<key> <string>name_context</string> </key>
<value> <string>context</string> </value>
</item>
<item>
<key> <string>name_m_self</string> </key>
<value> <string>script</string> </value>
</item>
<item>
<key> <string>name_subpath</string> </key>
<value> <string>traverse_subpath</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>_params</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>DataStreamModule_getTotalSize</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
software/wendelin-scalability/test_scalability_fluentd/PathTemplateItem/portal_skins/custom/ERP5Site_handleRawDataFluentdIngestion.py 0 → 100644
View file @ 816840c4
portal = context.getPortalObject()
request = portal.REQUEST
reference = request['reference']
data_chunk = request['data_chunk']
module = portal.data_stream_module
try:
data_stream = module[reference]
except KeyError:
data_stream = module.newContent(reference, 'Data Stream')
append_method = context.getId()
if append_method == 'scalability_test_raw_lf':
data_stream.appendData(data_chunk + '\n')
else:
assert append_method == 'scalability_test_unpack', append_method
for time, data_chunk in context.unpack(data_chunk):
data_stream.appendData(data_chunk)
software/wendelin-scalability/test_scalability_fluentd/PathTemplateItem/portal_skins/custom/ERP5Site_handleRawDataFluentdIngestion.xml 0 → 100644
View file @ 816840c4
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>Script_magic</string> </key>
<value> <int>3</int> </value>
</item>
<item>
<key> <string>_bind_names</string> </key>
<value>
<object>
<klass>
<global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key> <string>_asgns</string> </key>
<value>
<dictionary>
<item>
<key> <string>name_container</string> </key>
<value> <string>container</string> </value>
</item>
<item>
<key> <string>name_context</string> </key>
<value> <string>context</string> </value>
</item>
<item>
<key> <string>name_m_self</string> </key>
<value> <string>script</string> </value>
</item>
<item>
<key> <string>name_subpath</string> </key>
<value> <string>traverse_subpath</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>_params</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>ERP5Site_handleRawDataFluentdIngestion</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
software/wendelin-scalability/test_scalability_fluentd/bt/template_path_list 0 → 100644
View file @ 816840c4
portal_ingestion_policies/scalability_test_*
portal_skins/custom/DataStreamModule_getTotalSize
portal_skins/custom/ERP5Site_handleRawDataFluentdIngestion
\ No newline at end of file
software/wendelin-scalability/test_scalability_fluentd/bt/title 0 → 100644
View file @ 816840c4
test_scalability_fluentd
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7