Commit 7fc6aae8 authored by Jérome Perrin's avatar Jérome Perrin

Cloudooo cluster fixes

 - make cloudooo services use same name as in haproxy, this was off by one (service cloudooo-0 was cloudooo_0 in haproxy)
 - test cluster functionality (to catch bug from nexedi/cloudooo!29 )
 - update cloudooo with that fix
 - make sure we set an existant locale ( by using `C.UTF-8` ) and exercice a bit conversions involving characters encoding

See merge request nexedi/slapos!975
parents 2cb1cf57 147a586a
...@@ -14,7 +14,7 @@ recipe = slapos.recipe.build:gitclone ...@@ -14,7 +14,7 @@ recipe = slapos.recipe.build:gitclone
repository = https://lab.nexedi.com/nexedi/cloudooo.git repository = https://lab.nexedi.com/nexedi/cloudooo.git
branch = master branch = master
git-executable = ${git:location}/bin/git git-executable = ${git:location}/bin/git
revision = be6c35c0156e028f31da3ccb205afc6f95728d97 revision = 0b5ff71a2ede76499e81659aed392057ae910917
[cloudooo] [cloudooo]
recipe = zc.recipe.egg recipe = zc.recipe.egg
......
...@@ -18,4 +18,4 @@ md5sum = e986de01a57161b32425f1cd3ccac924 ...@@ -18,4 +18,4 @@ md5sum = e986de01a57161b32425f1cd3ccac924
[template-cloudooo-instance] [template-cloudooo-instance]
filename = instance-cloudooo.cfg.in filename = instance-cloudooo.cfg.in
md5sum = 9e1a66cf18d7c30c14afeb66c20afb46 md5sum = 440f2b82b119cbfa6f8c7d27652c3170
...@@ -162,6 +162,7 @@ environment = ...@@ -162,6 +162,7 @@ environment =
LD_LIBRARY_PATH = {{ parameter_dict['cairo'] }}/lib:{{ parameter_dict['cups'] }}/lib:{{ parameter_dict['cups'] }}/lib64:{{ parameter_dict['dbus'] }}/lib:{{ parameter_dict['dbus-glib'] }}/lib:{{ parameter_dict['file'] }}/lib:{{ parameter_dict['fontconfig'] }}/lib:{{ parameter_dict['freetype'] }}/lib:{{ parameter_dict['gcc'] }}/lib:{{ parameter_dict['gcc'] }}/lib64:{{ parameter_dict['glib'] }}/lib:{{ parameter_dict['glu'] }}/lib:{{ parameter_dict['libICE'] }}/lib:{{ parameter_dict['libSM'] }}/lib:{{ parameter_dict['libX11'] }}/lib:{{ parameter_dict['libXau'] }}/lib:{{ parameter_dict['libXdmcp'] }}/lib:{{ parameter_dict['libXext'] }}/lib:{{ parameter_dict['libXrender'] }}/lib:{{ parameter_dict['libexpat'] }}/lib:{{ parameter_dict['libffi'] }}/lib:{{ parameter_dict['libffi'] }}/lib64:{{ parameter_dict['libpng12'] }}/lib:{{ parameter_dict['libxcb'] }}/lib:{{ parameter_dict['mesa'] }}/lib:{{ parameter_dict['pixman'] }}/lib:{{ parameter_dict['xdamage'] }}/lib:{{ parameter_dict['xfixes'] }}/lib:{{ parameter_dict['zlib'] }}/lib LD_LIBRARY_PATH = {{ parameter_dict['cairo'] }}/lib:{{ parameter_dict['cups'] }}/lib:{{ parameter_dict['cups'] }}/lib64:{{ parameter_dict['dbus'] }}/lib:{{ parameter_dict['dbus-glib'] }}/lib:{{ parameter_dict['file'] }}/lib:{{ parameter_dict['fontconfig'] }}/lib:{{ parameter_dict['freetype'] }}/lib:{{ parameter_dict['gcc'] }}/lib:{{ parameter_dict['gcc'] }}/lib64:{{ parameter_dict['glib'] }}/lib:{{ parameter_dict['glu'] }}/lib:{{ parameter_dict['libICE'] }}/lib:{{ parameter_dict['libSM'] }}/lib:{{ parameter_dict['libX11'] }}/lib:{{ parameter_dict['libXau'] }}/lib:{{ parameter_dict['libXdmcp'] }}/lib:{{ parameter_dict['libXext'] }}/lib:{{ parameter_dict['libXrender'] }}/lib:{{ parameter_dict['libexpat'] }}/lib:{{ parameter_dict['libffi'] }}/lib:{{ parameter_dict['libffi'] }}/lib64:{{ parameter_dict['libpng12'] }}/lib:{{ parameter_dict['libxcb'] }}/lib:{{ parameter_dict['mesa'] }}/lib:{{ parameter_dict['pixman'] }}/lib:{{ parameter_dict['xdamage'] }}/lib:{{ parameter_dict['xfixes'] }}/lib:{{ parameter_dict['zlib'] }}/lib
FONTCONFIG_FILE = ${fontconfig-conf:rendered} FONTCONFIG_FILE = ${fontconfig-conf:rendered}
PATH = ${binary-link:target-directory} PATH = ${binary-link:target-directory}
LANG = C.UTF-8
mimetype_entry_addition = mimetype_entry_addition =
{% for entry in mimetype_entry_addition.splitlines() -%} {% for entry in mimetype_entry_addition.splitlines() -%}
{{ " " ~ entry.strip() }} {{ " " ~ entry.strip() }}
...@@ -172,7 +173,7 @@ ooo-binary-path = {{ parameter_dict['libreoffice-bin'] }}/program ...@@ -172,7 +173,7 @@ ooo-binary-path = {{ parameter_dict['libreoffice-bin'] }}/program
ooo-paster = {{ bin_directory }}/cloudooo_paster ooo-paster = {{ bin_directory }}/cloudooo_paster
ooo-uno-path = {{ parameter_dict['libreoffice-bin'] }}/basis-link/program ooo-uno-path = {{ parameter_dict['libreoffice-bin'] }}/basis-link/program
{% for index in range(backend_count) -%} {% for index in range(1, backend_count + 1) -%}
{% set name = 'cloudooo-' ~ index -%} {% set name = 'cloudooo-' ~ index -%}
[{{ cloudooo(name) }}] [{{ cloudooo(name) }}]
< = cloudooo-base < = cloudooo-base
...@@ -207,8 +208,8 @@ prepend-path = ${buildout:bin-directory} ...@@ -207,8 +208,8 @@ prepend-path = ${buildout:bin-directory}
run-unit-test = ${buildout:bin-directory}/runUnitTest run-unit-test = ${buildout:bin-directory}/runUnitTest
run-test-suite = ${buildout:bin-directory}/runTestSuite run-test-suite = ${buildout:bin-directory}/runTestSuite
ooo-paster = ${cloudooo-0:ooo-paster} ooo-paster = ${cloudooo-1:ooo-paster}
configuration-file = ${cloudooo-0:configuration-file} configuration-file = ${cloudooo-1:configuration-file}
run-unit-test-binary = {{ bin_directory }}/runCloudoooUnitTest run-unit-test-binary = {{ bin_directory }}/runCloudoooUnitTest
run-test-suite-binary = {{ bin_directory }}/runCloudoooTestSuite run-test-suite-binary = {{ bin_directory }}/runCloudoooTestSuite
......
...@@ -44,6 +44,7 @@ setup(name=name, ...@@ -44,6 +44,7 @@ setup(name=name,
'slapos.core', 'slapos.core',
'slapos.cookbook', 'slapos.cookbook',
'slapos.libnetworkcache', 'slapos.libnetworkcache',
'requests',
'six', 'six',
'PyPDF2', 'PyPDF2',
], ],
......
...@@ -26,23 +26,43 @@ ...@@ -26,23 +26,43 @@
# #
############################################################################## ##############################################################################
import codecs
import csv
import multiprocessing
import os import os
import json import json
import six.moves.xmlrpc_client as xmlrpclib import six.moves.xmlrpc_client as xmlrpclib
import six.moves.urllib.parse as urllib_parse
import ssl import ssl
import base64 import base64
import io import io
import requests
import PyPDF2 import PyPDF2
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
setUpModule, CloudOooTestCase = makeModuleSetUpAndTestCaseClass( setUpModule, _CloudOooTestCase = makeModuleSetUpAndTestCaseClass(
os.path.abspath( os.path.abspath(
os.path.join(os.path.dirname(__file__), '..', 'software.cfg'))) os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
# Cloudooo needs a lot of time before being available.
CloudOooTestCase.instance_max_retry = 30 class CloudOooTestCase(_CloudOooTestCase):
# Cloudooo needs a lot of time before being available.
instance_max_retry = 30
def setUp(self):
self.url = json.loads(
self.computer_partition.getConnectionParameterDict()["_"])['cloudooo']
# XXX ignore certificate errors
ssl_context = ssl.create_default_context()
ssl_context.check_hostname = False
ssl_context.verify_mode = ssl.CERT_NONE
self.server = xmlrpclib.ServerProxy(
self.url,
context=ssl_context,
allow_none=True,
)
def normalizeFontName(font_name): def normalizeFontName(font_name):
...@@ -88,19 +108,6 @@ class HTMLtoPDFConversionFontTestMixin: ...@@ -88,19 +108,6 @@ class HTMLtoPDFConversionFontTestMixin:
"""Convert the HTML source to pdf bytes. """Convert the HTML source to pdf bytes.
""" """
def setUp(self):
self.url = json.loads(
self.computer_partition.getConnectionParameterDict()["_"])['cloudooo']
# XXX ignore certificate errors
ssl_context = ssl.create_default_context()
ssl_context.check_hostname = False
ssl_context.verify_mode = ssl.CERT_NONE
self.server = xmlrpclib.ServerProxy(
self.url,
context=ssl_context,
allow_none=True,
)
def test(self): def test(self):
actual_font_mapping_mapping = {} actual_font_mapping_mapping = {}
for font in self.expected_font_mapping: for font in self.expected_font_mapping:
...@@ -237,3 +244,84 @@ class TestLibreoffice(HTMLtoPDFConversionFontTestMixin, CloudOooTestCase): ...@@ -237,3 +244,84 @@ class TestLibreoffice(HTMLtoPDFConversionFontTestMixin, CloudOooTestCase):
'html', 'html',
'pdf', 'pdf',
).encode()) ).encode())
class TestLibreOfficeTextConversion(CloudOooTestCase):
__partition_reference__ = 'txt'
def test_html_to_text(self):
self.assertEqual(
base64.decodestring(
self.server.convertFile(
base64.encodestring(
u'<html>héhé</html>'.encode('utf-8')).decode(),
'html',
'txt',
).encode()),
codecs.BOM_UTF8 + b'h\xc3\xa9h\xc3\xa9\n',
)
class TestLibreOfficeCluster(CloudOooTestCase):
__partition_reference__ = 'lc'
@classmethod
def getInstanceParameterDict(cls):
return {'backend-count': 4}
def test_multiple_conversions(self):
# make this function global so that it can be picked and used by multiprocessing
global _convert_html_to_text
def _convert_html_to_text(src_html):
return base64.decodestring(
self.server.convertFile(
base64.encodestring(src_html.encode()).decode(),
'html',
'txt',
).encode())
pool = multiprocessing.Pool(5)
# TODO py3: use with pool
converted = pool.map(_convert_html_to_text,
['<html><body>hello</body></html>'] * 100)
pool.terminate()
pool.join()
self.assertEqual(converted, [codecs.BOM_UTF8 + b'hello\n'] * 100)
# haproxy stats are exposed
res = requests.get(
urllib_parse.urljoin(self.url, '/haproxy;csv'),
verify=False,
stream=True,
)
reader = csv.DictReader(res.raw)
line_list = list(reader)
# requests have been balanced
total_hrsp_2xx = {
line['svname']: int(line['hrsp_2xx'])
for line in line_list
}
self.assertEqual(total_hrsp_2xx['FRONTEND'], 100)
self.assertEqual(total_hrsp_2xx['BACKEND'], 100)
for backend in 'cloudooo_1', 'cloudooo_2', 'cloudooo_3', 'cloudooo_4':
# ideally there should be 25% of requests on each backend, because we use
# round robin scheduling, but it can happen that some backend take longer
# to start, so we are tolerant here and just check that each backend
# process at least 15% of requests.
self.assertGreater(total_hrsp_2xx[backend], 15)
# no errors
total_eresp = {
line['svname']: int(line['eresp'] or 0)
for line in line_list
}
self.assertEqual(
total_eresp, {
'FRONTEND': 0,
'cloudooo_1': 0,
'cloudooo_2': 0,
'cloudooo_3': 0,
'cloudooo_4': 0,
'BACKEND': 0,
})
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment