Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
slapos
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
Carlos Ramos Carreño
slapos
Commits
b696fd3d
Commit
b696fd3d
authored
Sep 08, 2022
by
Jérome Perrin
Browse files
Options
Browse Files
Download
Plain Diff
Use python3 only for software release tests
See merge request
!1247
parents
405f2b7e
d3aed928
Changes
33
Hide whitespace changes
Inline
Side-by-side
Showing
33 changed files
with
468 additions
and
544 deletions
+468
-544
software/backupserver/test/test.py
software/backupserver/test/test.py
+1
-1
software/beremiz-ide/test/test.py
software/beremiz-ide/test/test.py
+0
-1
software/caddy-frontend/test/setup.py
software/caddy-frontend/test/setup.py
+0
-1
software/caddy-frontend/test/test.py
software/caddy-frontend/test/test.py
+1
-6
software/cloudooo/test/setup.py
software/cloudooo/test/setup.py
+0
-1
software/cloudooo/test/test.py
software/cloudooo/test/test.py
+23
-25
software/dream/test/test.py
software/dream/test/test.py
+0
-1
software/erp5/test/setup.py
software/erp5/test/setup.py
+0
-2
software/erp5/test/test/__init__.py
software/erp5/test/test/__init__.py
+0
-1
software/erp5/test/test/test_balancer.py
software/erp5/test/test/test_balancer.py
+40
-48
software/erp5/test/test/test_erp5.py
software/erp5/test/test/test_erp5.py
+35
-41
software/erp5/test/test/test_mariadb.py
software/erp5/test/test/test_mariadb.py
+7
-9
software/erp5/test/test/test_wcfs.py
software/erp5/test/test/test_wcfs.py
+1
-2
software/erp5/upgrade_test/setup.py
software/erp5/upgrade_test/setup.py
+0
-1
software/erp5/upgrade_test/test.py
software/erp5/upgrade_test/test.py
+8
-12
software/fluentd/software-py2.cfg
software/fluentd/software-py2.cfg
+0
-6
software/fluentd/test/setup.py
software/fluentd/test/setup.py
+0
-1
software/fluentd/test/test.py
software/fluentd/test/test.py
+22
-24
software/gitlab/test/test.py
software/gitlab/test/test.py
+1
-1
software/kvm/test/setup.py
software/kvm/test/setup.py
+0
-1
software/kvm/test/test.py
software/kvm/test/test.py
+81
-84
software/metabase/test/setup.py
software/metabase/test/setup.py
+0
-1
software/metabase/test/test.py
software/metabase/test/test.py
+1
-2
software/slapos-master/test/test/test_balancer.py
software/slapos-master/test/test/test_balancer.py
+74
-65
software/slapos-master/test/test/test_erp5.py
software/slapos-master/test/test/test_erp5.py
+65
-64
software/slapos-master/test/test/test_mariadb.py
software/slapos-master/test/test/test_mariadb.py
+11
-10
software/slapos-master/test/test/test_wcfs.py
software/slapos-master/test/test/test_wcfs.py
+1
-1
software/slapos-sr-testing/README.md
software/slapos-sr-testing/README.md
+0
-1
software/slapos-sr-testing/software-py3.cfg
software/slapos-sr-testing/software-py3.cfg
+0
-44
software/slapos-sr-testing/software.cfg
software/slapos-sr-testing/software.cfg
+34
-20
software/slaprunner/test/setup.py
software/slaprunner/test/setup.py
+0
-1
software/slaprunner/test/test.py
software/slaprunner/test/test.py
+19
-20
software/theia/test/test.py
software/theia/test/test.py
+43
-46
No files found.
software/backupserver/test/test.py
View file @
b696fd3d
...
...
@@ -45,7 +45,7 @@ class TestBackupServer(InstanceTestCase):
# Check that there is a RSS feed
self
.
assertTrue
(
'rss'
in
parameter_dict
)
self
.
assertTrue
(
parameter_dict
[
'rss'
].
startswith
(
'https://[%s]:9443/'
%
(
self
.
_ipv6_address
,
)
f'https://[
{
self
.
_ipv6_address
}
]:9443/'
))
result
=
requests
.
get
(
...
...
software/beremiz-ide/test/test.py
View file @
b696fd3d
...
...
@@ -24,7 +24,6 @@
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from
__future__
import
unicode_literals
import
os
import
requests
...
...
software/caddy-frontend/test/setup.py
View file @
b696fd3d
...
...
@@ -51,7 +51,6 @@ setup(name=name,
# caucase needed to connect to the KeDiFa caucase
'caucase'
,
'cryptography'
,
'backports.lzma'
,
],
zip_safe
=
True
,
test_suite
=
'test'
,
...
...
software/caddy-frontend/test/test.py
View file @
b696fd3d
...
...
@@ -51,18 +51,13 @@ import urllib.parse
import
socket
import
sys
import
logging
import
lzma
import
random
import
string
from
slapos.slap.standalone
import
SlapOSNodeInstanceError
import
caucase.client
import
caucase.utils
try
:
import
lzma
except
ImportError
:
from
backports
import
lzma
import
datetime
from
cryptography
import
x509
...
...
software/cloudooo/test/setup.py
View file @
b696fd3d
...
...
@@ -45,7 +45,6 @@ setup(name=name,
'slapos.cookbook'
,
'slapos.libnetworkcache'
,
'requests'
,
'six'
,
'PyPDF2'
,
],
zip_safe
=
True
,
...
...
software/cloudooo/test/test.py
View file @
b696fd3d
##############################################################################
# coding: utf-8
#
# Copyright (c) 2020 Nexedi SA and Contributors. All Rights Reserved.
#
...
...
@@ -31,8 +30,8 @@ import csv
import
multiprocessing
import
os
import
json
import
six.moves.xmlrpc_
client
as
xmlrpclib
import
six.moves.
urllib.parse
as
urllib_parse
import
xmlrpc.
client
as
xmlrpclib
import
urllib.parse
as
urllib_parse
import
ssl
import
base64
import
io
...
...
@@ -63,6 +62,7 @@ class CloudOooTestCase(_CloudOooTestCase):
context
=
ssl_context
,
allow_none
=
True
,
)
self
.
addCleanup
(
self
.
server
(
'close'
))
def
normalizeFontName
(
font_name
):
...
...
@@ -111,16 +111,16 @@ class HTMLtoPDFConversionFontTestMixin:
def
test
(
self
):
actual_font_mapping_mapping
=
{}
for
font
in
self
.
expected_font_mapping
:
src_html
=
'''
src_html
=
f
'''
<style>
p {{ font-family: "
{
font
}
"; font-size: 20pt; }}
</style>
<p>the quick brown fox jumps over the lazy dog.</p>
<p>THE QUICK BROWN FOX JUMPS OVER THE LAZY DOG.</p>
'''
.
format
(
**
locals
())
'''
pdf_data
=
self
.
_convert_html_to_pdf
(
src_html
)
pdf_reader
=
PyPDF2
.
PdfFileReader
(
io
.
BytesIO
(
(
pdf_data
)
))
pdf_reader
=
PyPDF2
.
PdfFileReader
(
io
.
BytesIO
(
pdf_data
))
self
.
assertEqual
(
self
.
pdf_producer
,
pdf_reader
.
getDocumentInfo
()[
'/Producer'
])
...
...
@@ -165,7 +165,7 @@ class TestWkhtmlToPDF(HTMLtoPDFConversionFontTestMixin, CloudOooTestCase):
'Liberation Sans Narrow'
:
'LiberationSansNarrow'
,
'Liberation Serif'
:
'LiberationSerif'
,
'Linux LibertineG'
:
'LiberationSans'
,
'OpenSymbol'
:
set
([
'DejaVuSans'
,
'OpenSymbol'
])
,
'OpenSymbol'
:
{
'DejaVuSans'
,
'OpenSymbol'
}
,
'Palatino'
:
'LiberationSans'
,
'Roboto Black'
:
'LiberationSans'
,
'Roboto Condensed Light'
:
'LiberationSans'
,
...
...
@@ -180,9 +180,9 @@ class TestWkhtmlToPDF(HTMLtoPDFConversionFontTestMixin, CloudOooTestCase):
}
def
_convert_html_to_pdf
(
self
,
src_html
):
return
base64
.
decode
string
(
return
base64
.
decode
bytes
(
self
.
server
.
convertFile
(
base64
.
encode
string
(
src_html
.
encode
()).
decode
(),
base64
.
encode
bytes
(
src_html
.
encode
()).
decode
(),
'html'
,
'pdf'
,
False
,
...
...
@@ -238,9 +238,9 @@ class TestLibreoffice(HTMLtoPDFConversionFontTestMixin, CloudOooTestCase):
}
def
_convert_html_to_pdf
(
self
,
src_html
):
return
base64
.
decode
string
(
return
base64
.
decode
bytes
(
self
.
server
.
convertFile
(
base64
.
encode
string
(
src_html
.
encode
()).
decode
(),
base64
.
encode
bytes
(
src_html
.
encode
()).
decode
(),
'html'
,
'pdf'
,
).
encode
())
...
...
@@ -251,10 +251,10 @@ class TestLibreOfficeTextConversion(CloudOooTestCase):
def
test_html_to_text
(
self
):
self
.
assertEqual
(
base64
.
decode
string
(
base64
.
decode
bytes
(
self
.
server
.
convertFile
(
base64
.
encode
string
(
u'<html>héhé</html>'
.
encode
(
'utf-8'
)).
decode
(),
base64
.
encode
bytes
(
'<html>héhé</html>'
.
encode
(
)).
decode
(),
'html'
,
'txt'
,
).
encode
()),
...
...
@@ -274,19 +274,18 @@ class TestLibreOfficeCluster(CloudOooTestCase):
global
_convert_html_to_text
def
_convert_html_to_text
(
src_html
):
return
base64
.
decode
string
(
return
base64
.
decode
bytes
(
self
.
server
.
convertFile
(
base64
.
encode
string
(
src_html
.
encode
()).
decode
(),
base64
.
encode
bytes
(
src_html
.
encode
()).
decode
(),
'html'
,
'txt'
,
).
encode
())
pool
=
multiprocessing
.
Pool
(
5
)
# TODO py3: use with pool
converted
=
pool
.
map
(
_convert_html_to_text
,
[
'<html><body>hello</body></html>'
]
*
100
)
pool
.
terminate
()
pool
.
join
()
with
pool
:
converted
=
pool
.
map
(
_convert_html_to_text
,
[
'<html><body>hello</body></html>'
]
*
100
)
self
.
assertEqual
(
converted
,
[
codecs
.
BOM_UTF8
+
b'hello
\
n
'
]
*
100
)
...
...
@@ -294,9 +293,8 @@ class TestLibreOfficeCluster(CloudOooTestCase):
res
=
requests
.
get
(
urllib_parse
.
urljoin
(
self
.
url
,
'/haproxy;csv'
),
verify
=
False
,
stream
=
True
,
)
reader
=
csv
.
DictReader
(
res
.
raw
)
reader
=
csv
.
DictReader
(
io
.
StringIO
(
res
.
text
)
)
line_list
=
list
(
reader
)
# requests have been balanced
total_hrsp_2xx
=
{
...
...
@@ -309,8 +307,8 @@ class TestLibreOfficeCluster(CloudOooTestCase):
# ideally there should be 25% of requests on each backend, because we use
# round robin scheduling, but it can happen that some backend take longer
# to start, so we are tolerant here and just check that each backend
# process at least
15% of requests.
self
.
assertGreater
(
total_hrsp_2xx
[
backend
],
15
)
# process at least
one request.
self
.
assertGreater
(
total_hrsp_2xx
[
backend
],
0
)
# no errors
total_eresp
=
{
line
[
'svname'
]:
int
(
line
[
'eresp'
]
or
0
)
...
...
software/dream/test/test.py
View file @
b696fd3d
##############################################################################
# coding: utf-8
#
# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
#
...
...
software/erp5/test/setup.py
View file @
b696fd3d
...
...
@@ -24,7 +24,6 @@
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from
__future__
import
absolute_import
from
setuptools
import
setup
,
find_packages
version
=
'0.0.1.dev0'
...
...
@@ -49,7 +48,6 @@ setup(name=name,
'psutil'
,
'requests'
,
'mysqlclient'
,
'backports.lzma'
,
'cryptography'
,
'pexpect'
,
'pyOpenSSL'
,
...
...
software/erp5/test/test/__init__.py
View file @
b696fd3d
...
...
@@ -25,7 +25,6 @@
#
##############################################################################
from
__future__
import
absolute_import
import
json
import
os
...
...
software/erp5/test/test/test_balancer.py
View file @
b696fd3d
from
__future__
import
absolute_import
import
glob
import
hashlib
import
json
...
...
@@ -9,10 +8,10 @@ import shutil
import
subprocess
import
tempfile
import
time
import
six.moves.urllib.request
,
six
.
moves
.
urllib
.
parse
,
six
.
moves
.
urllib
.
error
from
six.moves.BaseHTTPS
erver
import
BaseHTTPRequestHandler
import
urllib.parse
from
http.s
erver
import
BaseHTTPRequestHandler
import
mock
from
unittest
import
mock
import
OpenSSL.SSL
import
pexpect
import
psutil
...
...
@@ -27,7 +26,6 @@ from slapos.testing.utils import (CrontabMixin, ManagedHTTPServer,
findFreeTCPPort
)
from
.
import
ERP5InstanceTestCase
,
setUpModule
from
six.moves
import
range
setUpModule
# pyflakes
...
...
@@ -100,7 +98,7 @@ class CaucaseService(ManagedResource):
os
.
mkdir
(
os
.
path
.
join
(
caucased_dir
,
'user'
))
os
.
mkdir
(
os
.
path
.
join
(
caucased_dir
,
'service'
))
backend_caucased_netloc
=
'%s:%s'
%
(
self
.
_cls
.
_ipv4_address
,
findFreeTCPPort
(
self
.
_cls
.
_ipv4_address
))
backend_caucased_netloc
=
f'
{
self
.
_cls
.
_ipv4_address
}
:
{
findFreeTCPPort
(
self
.
_cls
.
_ipv4_address
)
}
'
self
.
url
=
'http://'
+
backend_caucased_netloc
self
.
_caucased_process
=
subprocess
.
Popen
(
[
...
...
@@ -213,7 +211,7 @@ class TestTimeout(BalancerTestCase, CrontabMixin):
@
classmethod
def
_getInstanceParameterDict
(
cls
):
# type: () -> dict
parameter_dict
=
super
(
TestTimeout
,
cls
).
_getInstanceParameterDict
()
parameter_dict
=
super
().
_getInstanceParameterDict
()
# use a slow server instead
parameter_dict
[
'dummy_http_server'
]
=
[[
cls
.
getManagedResource
(
"slow_web_server"
,
SlowHTTPServer
).
netloc
,
1
,
False
]]
# and set timeout of 1 second
...
...
@@ -224,12 +222,12 @@ class TestTimeout(BalancerTestCase, CrontabMixin):
# type: () -> None
self
.
assertEqual
(
requests
.
get
(
six
.
moves
.
urllib
.
parse
.
urljoin
(
self
.
default_balancer_url
,
'/1'
),
urllib
.
parse
.
urljoin
(
self
.
default_balancer_url
,
'/1'
),
verify
=
False
).
status_code
,
requests
.
codes
.
ok
)
self
.
assertEqual
(
requests
.
get
(
six
.
moves
.
urllib
.
parse
.
urljoin
(
self
.
default_balancer_url
,
'/5'
),
urllib
.
parse
.
urljoin
(
self
.
default_balancer_url
,
'/5'
),
verify
=
False
).
status_code
,
requests
.
codes
.
gateway_timeout
)
...
...
@@ -241,7 +239,7 @@ class TestLog(BalancerTestCase, CrontabMixin):
@
classmethod
def
_getInstanceParameterDict
(
cls
):
# type: () -> dict
parameter_dict
=
super
(
TestLog
,
cls
).
_getInstanceParameterDict
()
parameter_dict
=
super
().
_getInstanceParameterDict
()
# use a slow server instead
parameter_dict
[
'dummy_http_server'
]
=
[[
cls
.
getManagedResource
(
"slow_web_server"
,
SlowHTTPServer
).
netloc
,
1
,
False
]]
return
parameter_dict
...
...
@@ -249,7 +247,7 @@ class TestLog(BalancerTestCase, CrontabMixin):
def
test_access_log_format
(
self
):
# type: () -> None
requests
.
get
(
six
.
moves
.
urllib
.
parse
.
urljoin
(
self
.
default_balancer_url
,
'/url_path'
),
urllib
.
parse
.
urljoin
(
self
.
default_balancer_url
,
'/url_path'
),
verify
=
False
,
)
time
.
sleep
(.
5
)
# wait a bit more until access is logged
...
...
@@ -288,7 +286,7 @@ class TestLog(BalancerTestCase, CrontabMixin):
'
apachedex
',
'
ApacheDex
-*
.
html
',
))
with open(apachedex_report
, 'r'
) as f:
with open(apachedex_report) as f:
report_text = f.read()
self.assertIn('
APacheDEX
', report_text)
# having this table means that apachedex could parse some lines.
...
...
@@ -336,7 +334,7 @@ class TestLog(BalancerTestCase, CrontabMixin):
error_line = error_log_file.read().splitlines()[-1]
self.assertIn('
proxy
family_default
has
no
server
available
!
', error_line)
# this log also include a timestamp
self.assertRegex
pMatches
(error_line, r'
\
d
{
4
}
-
\
d
{
2
}
-
\
d
{
2
}
T
\
d
{
2
}:
\
d
{
2
}:
\
d
{
2
}
')
self.assertRegex(error_line, r'
\
d
{
4
}
-
\
d
{
2
}
-
\
d
{
2
}
T
\
d
{
2
}:
\
d
{
2
}:
\
d
{
2
}
')
class BalancerCookieHTTPServer(ManagedHTTPServer):
...
...
@@ -377,7 +375,7 @@ class TestBalancer(BalancerTestCase):
@
classmethod
def
_getInstanceParameterDict
(
cls
):
# type: () -> dict
parameter_dict
=
super
(
TestBalancer
,
cls
).
_getInstanceParameterDict
()
parameter_dict
=
super
().
_getInstanceParameterDict
()
# use two backend servers
parameter_dict
[
'dummy_http_server'
]
=
[
...
...
@@ -409,7 +407,7 @@ class TestBalancer(BalancerTestCase):
# if backend provides a "SERVERID" cookie, balancer will overwrite it with the
# backend selected by balancing algorithm
self
.
assertIn
(
requests
.
get
(
six
.
moves
.
urllib
.
parse
.
urljoin
(
self
.
default_balancer_url
,
'/set_cookie'
),
verify
=
False
).
cookies
[
'SERVERID'
],
requests
.
get
(
urllib
.
parse
.
urljoin
(
self
.
default_balancer_url
,
'/set_cookie'
),
verify
=
False
).
cookies
[
'SERVERID'
],
(
'default-0'
,
'default-1'
),
)
...
...
@@ -457,10 +455,7 @@ class TestTestRunnerEntryPoints(BalancerTestCase):
@
classmethod
def
_getInstanceParameterDict
(
cls
):
# type: () -> dict
parameter_dict
=
super
(
TestTestRunnerEntryPoints
,
cls
,
).
_getInstanceParameterDict
()
parameter_dict
=
super
().
_getInstanceParameterDict
()
parameter_dict
[
'dummy_http_server-test-runner-address-list'
]
=
[
[
...
...
@@ -485,18 +480,18 @@ class TestTestRunnerEntryPoints(BalancerTestCase):
)[
'default-test-runner-url-list'
]
url_0
,
url_1
,
url_2
=
test_runner_url_list
self
.
assertEqual
(
six
.
moves
.
urllib
.
parse
.
urlparse
(
url_0
).
netloc
,
six
.
moves
.
urllib
.
parse
.
urlparse
(
url_1
).
netloc
)
urllib
.
parse
.
urlparse
(
url_0
).
netloc
,
urllib
.
parse
.
urlparse
(
url_1
).
netloc
)
self
.
assertEqual
(
six
.
moves
.
urllib
.
parse
.
urlparse
(
url_0
).
netloc
,
six
.
moves
.
urllib
.
parse
.
urlparse
(
url_2
).
netloc
)
urllib
.
parse
.
urlparse
(
url_0
).
netloc
,
urllib
.
parse
.
urlparse
(
url_2
).
netloc
)
path_0
=
'/VirtualHostBase/https/{netloc}/VirtualHostRoot/_vh_unit_test_0/something'
.
format
(
netloc
=
six
.
moves
.
urllib
.
parse
.
urlparse
(
url_0
).
netloc
)
netloc
=
urllib
.
parse
.
urlparse
(
url_0
).
netloc
)
path_1
=
'/VirtualHostBase/https/{netloc}/VirtualHostRoot/_vh_unit_test_1/something'
.
format
(
netloc
=
six
.
moves
.
urllib
.
parse
.
urlparse
(
url_0
).
netloc
)
netloc
=
urllib
.
parse
.
urlparse
(
url_0
).
netloc
)
path_2
=
'/VirtualHostBase/https/{netloc}/VirtualHostRoot/_vh_unit_test_2/something'
.
format
(
netloc
=
six
.
moves
.
urllib
.
parse
.
urlparse
(
url_0
).
netloc
)
netloc
=
urllib
.
parse
.
urlparse
(
url_0
).
netloc
)
self
.
assertEqual
(
{
...
...
@@ -534,7 +529,7 @@ class TestHTTP(BalancerTestCase):
@
classmethod
def
_getInstanceParameterDict
(
cls
):
# type: () -> dict
parameter_dict
=
super
(
TestHTTP
,
cls
).
_getInstanceParameterDict
()
parameter_dict
=
super
().
_getInstanceParameterDict
()
# use a HTTP/1.1 server instead
parameter_dict
[
'dummy_http_server'
]
=
[[
cls
.
getManagedResource
(
"HTTP/1.1 Server"
,
EchoHTTP11Server
).
netloc
,
1
,
False
]]
return
parameter_dict
...
...
@@ -574,7 +569,7 @@ class TestHTTP(BalancerTestCase):
session
.
get
(
self
.
default_balancer_url
).
raise_for_status
()
new_conn
.
assert_not_called
()
parsed_url
=
six
.
moves
.
urllib
.
parse
.
urlparse
(
self
.
default_balancer_url
)
parsed_url
=
urllib
.
parse
.
urlparse
(
self
.
default_balancer_url
)
# check that we have an open file for the ip connection
self
.
assertTrue
([
...
...
@@ -617,7 +612,7 @@ class TestContentEncoding(BalancerTestCase):
@
classmethod
def
_getInstanceParameterDict
(
cls
):
# type: () -> dict
parameter_dict
=
super
(
TestContentEncoding
,
cls
).
_getInstanceParameterDict
()
parameter_dict
=
super
().
_getInstanceParameterDict
()
parameter_dict
[
'dummy_http_server'
]
=
[
[
cls
.
getManagedResource
(
"content_type_server"
,
ContentTypeHTTPServer
).
netloc
,
1
,
False
],
]
...
...
@@ -644,17 +639,17 @@ class TestContentEncoding(BalancerTestCase):
'application/font-woff2'
,
'application/x-font-opentype'
,
'application/wasm'
,):
resp
=
requests
.
get
(
six
.
moves
.
urllib
.
parse
.
urljoin
(
self
.
default_balancer_url
,
content_type
),
verify
=
False
)
resp
=
requests
.
get
(
urllib
.
parse
.
urljoin
(
self
.
default_balancer_url
,
content_type
),
verify
=
False
)
self
.
assertEqual
(
resp
.
headers
[
'Content-Type'
],
content_type
)
self
.
assertEqual
(
resp
.
headers
.
get
(
'Content-Encoding'
),
'gzip'
,
'
%s uses wrong encoding: %s'
%
(
content_type
,
resp
.
headers
.
get
(
'Content-Encoding'
)))
'
{} uses wrong encoding: {}'
.
format
(
content_type
,
resp
.
headers
.
get
(
'Content-Encoding'
)))
self
.
assertEqual
(
resp
.
text
,
'OK'
)
def
test_no_gzip_encoding
(
self
):
# type: () -> None
resp
=
requests
.
get
(
six
.
moves
.
urllib
.
parse
.
urljoin
(
self
.
default_balancer_url
,
'/image/png'
),
verify
=
False
)
resp
=
requests
.
get
(
urllib
.
parse
.
urljoin
(
self
.
default_balancer_url
,
'/image/png'
),
verify
=
False
)
self
.
assertNotIn
(
'Content-Encoding'
,
resp
.
headers
)
self
.
assertEqual
(
resp
.
text
,
'OK'
)
...
...
@@ -777,9 +772,9 @@ class TestFrontendXForwardedFor(BalancerTestCase):
# type: () -> dict
frontend_caucase
=
cls
.
getManagedResource
(
'frontend_caucase'
,
CaucaseService
)
certificate
=
cls
.
getManagedResource
(
'client_certificate'
,
CaucaseCertificate
)
certificate
.
request
(
u
'shared frontend'
,
frontend_caucase
)
certificate
.
request
(
'shared frontend'
,
frontend_caucase
)
parameter_dict
=
super
(
TestFrontendXForwardedFor
,
cls
).
_getInstanceParameterDict
()
parameter_dict
=
super
().
_getInstanceParameterDict
()
# add another "-auth" backend, that will have ssl-authentication enabled
parameter_dict
[
'zope-family-dict'
][
'default-auth'
]
=
[
'dummy_http_server'
]
parameter_dict
[
'backend-path-dict'
][
'default-auth'
]
=
'/'
...
...
@@ -815,7 +810,7 @@ class TestFrontendXForwardedFor(BalancerTestCase):
).
json
()
self
.
assertNotEqual
(
result
[
'Incoming Headers'
].
get
(
'x-forwarded-for'
,
''
).
split
(
', '
)[
0
],
'1.2.3.4'
)
balancer_url
=
json
.
loads
(
self
.
computer_partition
.
getConnectionParameterDict
()[
'_'
])[
'default-auth'
]
with
self
.
assertRaisesRegex
p
(
Exception
,
"certificate required"
):
with
self
.
assertRaisesRegex
(
Exception
,
"certificate required"
):
requests
.
get
(
balancer_url
,
headers
=
{
'X-Forwarded-For'
:
'1.2.3.4'
},
...
...
@@ -833,8 +828,8 @@ class TestServerTLSProvidedCertificate(BalancerTestCase):
# type: () -> dict
server_caucase
=
cls
.
getManagedResource
(
'server_caucase'
,
CaucaseService
)
server_certificate
=
cls
.
getManagedResource
(
'server_certificate'
,
CaucaseCertificate
)
server_certificate
.
request
(
six
.
ensure_text
(
cls
.
_ipv4_address
)
,
server_caucase
)
parameter_dict
=
super
(
TestServerTLSProvidedCertificate
,
cls
).
_getInstanceParameterDict
()
server_certificate
.
request
(
cls
.
_ipv4_address
,
server_caucase
)
parameter_dict
=
super
().
_getInstanceParameterDict
()
with
open
(
server_certificate
.
cert_file
)
as
f
:
parameter_dict
[
'ssl'
][
'cert'
]
=
f
.
read
()
with
open
(
server_certificate
.
key_file
)
as
f
:
...
...
@@ -855,13 +850,13 @@ class TestClientTLS(BalancerTestCase):
# type: () -> dict
frontend_caucase1
=
cls
.
getManagedResource
(
'frontend_caucase1'
,
CaucaseService
)
certificate1
=
cls
.
getManagedResource
(
'client_certificate1'
,
CaucaseCertificate
)
certificate1
.
request
(
u
'client_certificate1'
,
frontend_caucase1
)
certificate1
.
request
(
'client_certificate1'
,
frontend_caucase1
)
frontend_caucase2
=
cls
.
getManagedResource
(
'frontend_caucase2'
,
CaucaseService
)
certificate2
=
cls
.
getManagedResource
(
'client_certificate2'
,
CaucaseCertificate
)
certificate2
.
request
(
u
'client_certificate2'
,
frontend_caucase2
)
certificate2
.
request
(
'client_certificate2'
,
frontend_caucase2
)
parameter_dict
=
super
(
TestClientTLS
,
cls
).
_getInstanceParameterDict
()
parameter_dict
=
super
().
_getInstanceParameterDict
()
parameter_dict
[
'ssl-authentication-dict'
]
=
{
'default'
:
True
,
}
...
...
@@ -936,11 +931,11 @@ class TestClientTLS(BalancerTestCase):
process
=
pexpect
.
spawnu
(
"faketime +1day %s"
%
caucase_updater
)
process
.
logfile
=
DebugLogFile
()
process
.
expect
(
u
"Got new CRL.*Next wake-up at.*"
)
process
.
expect
(
"Got new CRL.*Next wake-up at.*"
)
process
.
terminate
()
process
.
wait
()
with
self
.
assertRaisesRegex
p
(
Exception
,
'certificate revoked'
):
with
self
.
assertRaisesRegex
(
Exception
,
'certificate revoked'
):
_make_request
()
...
...
@@ -952,10 +947,7 @@ class TestPathBasedRouting(BalancerTestCase):
@
classmethod
def
_getInstanceParameterDict
(
cls
):
# type: () -> dict
parameter_dict
=
super
(
TestPathBasedRouting
,
cls
,
).
_getInstanceParameterDict
()
parameter_dict
=
super
().
_getInstanceParameterDict
()
parameter_dict
[
'zope-family-dict'
][
'second'
]
=
parameter_dict
[
'zope-family-dict'
][
...
...
@@ -985,7 +977,7 @@ class TestPathBasedRouting(BalancerTestCase):
published_dict
=
json
.
loads
(
self
.
computer_partition
.
getConnectionParameterDict
()[
'_'
])
scheme
=
'scheme'
netloc
=
'example.com:8080'
prefix
=
'/VirtualHostBase/'
+
scheme
+
'//'
+
six
.
moves
.
urllib
.
parse
.
quote
(
prefix
=
'/VirtualHostBase/'
+
scheme
+
'//'
+
urllib
.
parse
.
quote
(
netloc
,
safe
=
''
,
)
...
...
@@ -1009,7 +1001,7 @@ class TestPathBasedRouting(BalancerTestCase):
# test will need to be updated accordingly.
self
.
assertEqual
(
requests
.
get
(
six
.
moves
.
urllib
.
parse
.
urljoin
(
published_dict
[
family
],
prefix
+
vhr
+
path
),
urllib
.
parse
.
urljoin
(
published_dict
[
family
],
prefix
+
vhr
+
path
),
verify
=
False
,
).
json
()[
'Path'
],
expected_path
,
...
...
software/erp5/test/test/test_erp5.py
View file @
b696fd3d
...
...
@@ -25,7 +25,6 @@
#
##############################################################################
from
__future__
import
absolute_import
import
contextlib
import
glob
...
...
@@ -42,9 +41,8 @@ import unittest
import
psutil
import
requests
import
six
import
six.moves.urllib.parse
import
six.moves.xmlrpc_client
import
urllib.parse
import
xmlrpc.client
import
urllib3
from
slapos.testing.utils
import
CrontabMixin
...
...
@@ -53,7 +51,7 @@ from . import ERP5InstanceTestCase, setUpModule
setUpModule
# pyflakes
class
TestPublishedURLIsReachableMixin
(
object
)
:
class
TestPublishedURLIsReachableMixin
:
"""Mixin that checks that default page of ERP5 is reachable.
"""
...
...
@@ -61,7 +59,7 @@ class TestPublishedURLIsReachableMixin(object):
# We access ERP5 trough a "virtual host", which should make
# ERP5 produce URLs using https://virtual-host-name:1234/virtual_host_root
# as base.
virtual_host_url
=
six
.
moves
.
urllib
.
parse
.
urljoin
(
virtual_host_url
=
urllib
.
parse
.
urljoin
(
base_url
,
'/VirtualHostBase/https/virtual-host-name:1234/{}/VirtualHostRoot/_vh_virtual_host_root/'
.
format
(
site_id
))
...
...
@@ -89,7 +87,7 @@ class TestPublishedURLIsReachableMixin(object):
# login page can be rendered and contain the text "ERP5"
r
=
session
.
get
(
six
.
moves
.
urllib
.
parse
.
urljoin
(
base_url
,
'{}/login_form'
.
format
(
site_id
)
),
urllib
.
parse
.
urljoin
(
base_url
,
f'
{
site_id
}
/login_form'
),
verify
=
verify
,
allow_redirects
=
False
,
)
...
...
@@ -184,7 +182,7 @@ class TestBalancerPorts(ERP5InstanceTestCase):
}
def
checkValidHTTPSURL
(
self
,
url
):
parsed
=
six
.
moves
.
urllib
.
parse
.
urlparse
(
url
)
parsed
=
urllib
.
parse
.
urlparse
(
url
)
self
.
assertEqual
(
parsed
.
scheme
,
'https'
)
self
.
assertTrue
(
parsed
.
hostname
)
self
.
assertTrue
(
parsed
.
port
)
...
...
@@ -194,16 +192,16 @@ class TestBalancerPorts(ERP5InstanceTestCase):
param_dict
=
self
.
getRootPartitionConnectionParameterDict
()
for
family_name
in
(
'family1'
,
'family2'
):
self
.
checkValidHTTPSURL
(
param_dict
[
'family-{family_name}'
.
format
(
family_name
=
family_name
)
])
param_dict
[
f'family-
{
family_name
}
'
])
self
.
checkValidHTTPSURL
(
param_dict
[
'family-{family_name}-v6'
.
format
(
family_name
=
family_name
)
])
param_dict
[
f'family-
{
family_name
}
-v6'
])
def
test_published_test_runner_url
(
self
):
# each family's also a list of test test runner URLs, by default 3 per family
param_dict
=
self
.
getRootPartitionConnectionParameterDict
()
for
family_name
in
(
'family1'
,
'family2'
):
family_test_runner_url_list
=
param_dict
[
'{family_name}-test-runner-url-list'
.
format
(
family_name
=
family_name
)
]
f'
{
family_name
}
-test-runner-url-list'
]
self
.
assertEqual
(
3
,
len
(
family_test_runner_url_list
))
for
url
in
family_test_runner_url_list
:
self
.
checkValidHTTPSURL
(
url
)
...
...
@@ -221,16 +219,16 @@ class TestBalancerPorts(ERP5InstanceTestCase):
# normal access on ipv4 and ipv6 and test runner access on ipv4 only
with
self
.
slap
.
instance_supervisor_rpc
as
supervisor
:
all_process_info
=
supervisor
.
getAllProcessInfo
()
process_info
,
=
[
p
for
p
in
all_process_info
if
p
[
'name'
].
startswith
(
'haproxy-'
)]
process_info
,
=
(
p
for
p
in
all_process_info
if
p
[
'name'
].
startswith
(
'haproxy-'
))
haproxy_master_process
=
psutil
.
Process
(
process_info
[
'pid'
])
haproxy_worker_process
,
=
haproxy_master_process
.
children
()
self
.
assertEqual
(
sorted
([
socket
.
AF_INET
]
*
4
+
[
socket
.
AF_INET6
]
*
2
),
sorted
(
[
sorted
(
c
.
family
for
c
in
haproxy_worker_process
.
connections
()
if
c
.
status
==
'LISTEN'
]
))
))
class
TestSeleniumTestRunner
(
ERP5InstanceTestCase
,
TestPublishedURLIsReachableMixin
):
...
...
@@ -296,7 +294,7 @@ class TestDisableTestRunner(ERP5InstanceTestCase, TestPublishedURLIsReachableMix
# Haproxy only listen on two ports, there is no haproxy ports allocated for test runner
with
self
.
slap
.
instance_supervisor_rpc
as
supervisor
:
all_process_info
=
supervisor
.
getAllProcessInfo
()
process_info
,
=
[
p
for
p
in
all_process_info
if
p
[
'name'
].
startswith
(
'haproxy'
)]
process_info
,
=
(
p
for
p
in
all_process_info
if
p
[
'name'
].
startswith
(
'haproxy'
))
haproxy_master_process
=
psutil
.
Process
(
process_info
[
'pid'
])
haproxy_worker_process
,
=
haproxy_master_process
.
children
()
self
.
assertEqual
(
...
...
@@ -366,7 +364,7 @@ class TestZopeNodeParameterOverride(ERP5InstanceTestCase, TestPublishedURLIsReac
zodb
[
"pool-timeout"
]
=
"10m"
storage
[
"storage"
]
=
"root"
storage
[
"server"
]
=
zeo_addr
with
open
(
'%s/etc/zope-%s.conf'
%
(
partition
,
zope
)
)
as
f
:
with
open
(
f'
{
partition
}
/etc/zope-
{
zope
}
.conf'
)
as
f
:
conf
=
list
(
map
(
str
.
strip
,
f
.
readlines
()))
i
=
conf
.
index
(
"<zodb_db root>"
)
+
1
conf
=
iter
(
conf
[
i
:
conf
.
index
(
"</zodb_db>"
,
i
)])
...
...
@@ -376,12 +374,12 @@ class TestZopeNodeParameterOverride(ERP5InstanceTestCase, TestPublishedURLIsReac
if
line
==
'</zeoclient>'
:
break
checkParameter
(
line
,
storage
)
for
k
,
v
in
s
ix
.
iteritems
(
storage
):
for
k
,
v
in
s
torage
.
items
(
):
self
.
assertIsNone
(
v
,
k
)
del
storage
else
:
checkParameter
(
line
,
zodb
)
for
k
,
v
in
six
.
iteritems
(
zodb
):
for
k
,
v
in
zodb
.
items
(
):
self
.
assertIsNone
(
v
,
k
)
partition
=
self
.
getComputerPartitionPath
(
'zope-a'
)
...
...
@@ -436,19 +434,19 @@ class TestWatchActivities(ERP5InstanceTestCase):
env
=
dict
(
os
.
environ
,
PATH
=
os
.
pathsep
.
join
([
tmpdir
,
os
.
environ
[
'PATH'
]])),
stderr
=
subprocess
.
STDOUT
,
universal_newlines
=
True
,
text
=
True
,
)
except
subprocess
.
CalledProcessError
as
e
:
self
.
fail
(
e
.
output
)
self
.
assertIn
(
' dict '
,
output
)
class
ZopeSkinsMixin
(
object
)
:
class
ZopeSkinsMixin
:
"""Mixins with utility methods to test zope behaviors.
"""
@
classmethod
def
_setUpClass
(
cls
):
super
(
ZopeSkinsMixin
,
cls
).
_setUpClass
()
super
().
_setUpClass
()
param_dict
=
cls
.
getRootPartitionConnectionParameterDict
()
with
cls
.
getXMLRPCClient
()
as
erp5_xmlrpc_client
:
# wait for ERP5 to be ready (TODO: this should probably be a promise)
...
...
@@ -456,8 +454,8 @@ class ZopeSkinsMixin(object):
time
.
sleep
(
1
)
try
:
erp5_xmlrpc_client
.
getTitle
()
except
(
six
.
moves
.
xmlrpc_
client
.
ProtocolError
,
six
.
moves
.
xmlrpc_
client
.
Fault
):
except
(
xmlrpc
.
client
.
ProtocolError
,
xmlrpc
.
client
.
Fault
):
pass
else
:
break
...
...
@@ -470,7 +468,7 @@ class ZopeSkinsMixin(object):
path is joined with urllib.parse.urljoin to the URL of the portal.
"""
param_dict
=
cls
.
getRootPartitionConnectionParameterDict
()
parsed
=
six
.
moves
.
urllib
.
parse
.
urlparse
(
param_dict
[
'family-'
+
family_name
])
parsed
=
urllib
.
parse
.
urlparse
(
param_dict
[
'family-'
+
family_name
])
base_url
=
parsed
.
_replace
(
netloc
=
'{}:{}@{}:{}'
.
format
(
param_dict
[
'inituser-login'
],
...
...
@@ -480,7 +478,7 @@ class ZopeSkinsMixin(object):
),
path
=
param_dict
[
'site-id'
]
+
'/'
,
).
geturl
()
return
six
.
moves
.
urllib_
parse
.
urljoin
(
base_url
,
path
)
return
urllib
.
parse
.
urljoin
(
base_url
,
path
)
@
classmethod
@
contextlib
.
contextmanager
...
...
@@ -489,16 +487,12 @@ class ZopeSkinsMixin(object):
ssl_context
=
ssl
.
create_default_context
()
ssl_context
.
check_hostname
=
False
ssl_context
.
verify_mode
=
ssl
.
CERT_NONE
erp5_xmlrpc_client
=
six
.
moves
.
xmlrpc_
client
.
ServerProxy
(
erp5_xmlrpc_client
=
xmlrpc
.
client
.
ServerProxy
(
cls
.
_getAuthenticatedZopeUrl
(
''
),
context
=
ssl_context
,
)
# BBB use as a context manager only on python3
if
sys
.
version_info
<
(
3
,
):
with
erp5_xmlrpc_client
:
yield
erp5_xmlrpc_client
else
:
with
erp5_xmlrpc_client
:
yield
erp5_xmlrpc_client
@
classmethod
def
_addPythonScript
(
cls
,
script_id
,
params
,
body
):
...
...
@@ -507,7 +501,7 @@ class ZopeSkinsMixin(object):
try
:
custom
.
manage_addProduct
.
PythonScripts
.
manage_addPythonScript
(
script_id
)
except
six
.
moves
.
xmlrpc_
client
.
ProtocolError
as
e
:
except
xmlrpc
.
client
.
ProtocolError
as
e
:
if
e
.
errcode
!=
302
:
raise
getattr
(
custom
,
script_id
).
ZPythonScriptHTML_editAction
(
...
...
@@ -542,7 +536,7 @@ class ZopeTestMixin(ZopeSkinsMixin, CrontabMixin):
@
classmethod
def
_setUpClass
(
cls
):
super
(
ZopeTestMixin
,
cls
).
_setUpClass
()
super
().
_setUpClass
()
cls
.
zope_base_url
=
cls
.
_getAuthenticatedZopeUrl
(
''
)
param_dict
=
cls
.
getRootPartitionConnectionParameterDict
()
cls
.
zope_deadlock_debugger_url
=
cls
.
_getAuthenticatedZopeUrl
(
...
...
@@ -565,7 +559,7 @@ class ZopeTestMixin(ZopeSkinsMixin, CrontabMixin):
raise ValueError("Unknown mode: %s" % mode)
'''
,
)
cls
.
zope_verify_activity_processing_url
=
six
.
moves
.
urllib_
parse
.
urljoin
(
cls
.
zope_verify_activity_processing_url
=
urllib
.
parse
.
urljoin
(
cls
.
zope_base_url
,
'ERP5Site_verifyActivityProcessing'
,
)
...
...
@@ -578,7 +572,7 @@ class ZopeTestMixin(ZopeSkinsMixin, CrontabMixin):
return log("hello %s" % name)
'''
,
)
cls
.
zope_log_message_url
=
six
.
moves
.
urllib_
parse
.
urljoin
(
cls
.
zope_log_message_url
=
urllib
.
parse
.
urljoin
(
cls
.
zope_base_url
,
'ERP5Site_logMessage'
,
)
...
...
@@ -593,18 +587,18 @@ class ZopeTestMixin(ZopeSkinsMixin, CrontabMixin):
return "done"
'''
,
)
cls
.
zope_long_request_url
=
six
.
moves
.
urllib_
parse
.
urljoin
(
cls
.
zope_long_request_url
=
urllib
.
parse
.
urljoin
(
cls
.
zope_base_url
,
'ERP5Site_executeLongRequest'
,
)
def
setUp
(
self
):
super
(
ZopeTestMixin
,
self
).
setUp
()
super
().
setUp
()
# run logrotate a first time so that it create state files
self
.
_executeCrontabAtDate
(
'logrotate'
,
'2000-01-01'
)
def
tearDown
(
self
):
super
(
ZopeTestMixin
,
self
).
tearDown
()
super
().
tearDown
()
# reset logrotate status
logrotate_status
=
os
.
path
.
join
(
self
.
getComputerPartitionPath
(
'zope-default'
),
...
...
@@ -859,11 +853,11 @@ class TestZopeWSGI(ZopeTestMixin, ERP5InstanceTestCase):
@
unittest
.
expectedFailure
def
test_long_request_log_rotation
(
self
):
super
(
TestZopeWSGI
,
self
).
test_long_request_log_rotation
(
self
)
super
(
).
test_long_request_log_rotation
(
)
@
unittest
.
expectedFailure
def
test_basic_authentication_user_in_access_log
(
self
):
super
(
TestZopeWSGI
,
self
).
test_basic_authentication_user_in_access_log
(
self
)
super
(
).
test_basic_authentication_user_in_access_log
(
)
class
TestZopePublisherTimeout
(
ZopeSkinsMixin
,
ERP5InstanceTestCase
):
...
...
@@ -902,7 +896,7 @@ class TestZopePublisherTimeout(ZopeSkinsMixin, ERP5InstanceTestCase):
@
classmethod
def
_setUpClass
(
cls
):
super
(
TestZopePublisherTimeout
,
cls
).
_setUpClass
()
super
().
_setUpClass
()
cls
.
_addPythonScript
(
'ERP5Site_doSlowRequest'
,
''
,
...
...
software/erp5/test/test/test_mariadb.py
View file @
b696fd3d
##############################################################################
# coding: utf-8
#
# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
#
...
...
@@ -26,11 +25,10 @@
#
##############################################################################
from
__future__
import
absolute_import
import
os
import
json
import
glob
import
six.moves.
urllib.parse
import
urllib.parse
import
socket
import
sys
import
time
...
...
@@ -39,7 +37,7 @@ import datetime
import
subprocess
import
gzip
from
backports
import
lzma
import
lzma
import
MySQLdb
from
slapos.testing.utils
import
CrontabMixin
...
...
@@ -84,7 +82,7 @@ class MariaDBTestCase(ERP5InstanceTestCase):
# type: () -> MySQLdb.connections.Connection
connection_parameter_dict
=
json
.
loads
(
self
.
computer_partition
.
getConnectionParameterDict
()[
'_'
])
db_url
=
six
.
moves
.
urllib
.
parse
.
urlparse
(
connection_parameter_dict
[
'database-list'
][
0
])
db_url
=
urllib
.
parse
.
urlparse
(
connection_parameter_dict
[
'database-list'
][
0
])
self
.
assertEqual
(
'mysql'
,
db_url
.
scheme
)
self
.
assertTrue
(
db_url
.
path
.
startswith
(
'/'
))
...
...
@@ -208,7 +206,7 @@ class TestMariaDB(MariaDBTestCase):
"""
select * from test_utf8_collation where col1 = "a"
"""
)
self
.
assertEqual
(((
u
'à'
,),),
cnx
.
store_result
().
fetch_row
(
maxrows
=
2
))
self
.
assertEqual
(((
'à'
,),),
cnx
.
store_result
().
fetch_row
(
maxrows
=
2
))
class
TestMroonga
(
MariaDBTestCase
):
...
...
@@ -232,7 +230,7 @@ class TestMroonga(MariaDBTestCase):
SELECT mroonga_normalize("ABCDあぃうぇ㍑")
"""
)
# XXX this is returned as bytes by mroonga/mariadb (this might be a bug)
self
.
assertEqual
(((
u'abcdあぃうぇリットル'
.
encode
(
'utf-8'
),),),
self
.
assertEqual
(((
'abcdあぃうぇリットル'
.
encode
(
),),),
cnx
.
store_result
().
fetch_row
(
maxrows
=
2
))
if
0
:
...
...
@@ -245,7 +243,7 @@ class TestMroonga(MariaDBTestCase):
"""
SELECT mroonga_normalize("aBcDあぃウェ㍑", "NormalizerMySQLUnicodeCIExceptKanaCIKanaWithVoicedSoundMark")
"""
)
self
.
assertEqual
(((
u'ABCDあぃうぇ㍑'
.
encode
(
'utf-8'
),),),
self
.
assertEqual
(((
'ABCDあぃうぇ㍑'
.
encode
(
),),),
cnx
.
store_result
().
fetch_row
(
maxrows
=
2
))
def
test_mroonga_full_text_normalizer
(
self
):
...
...
@@ -282,7 +280,7 @@ class TestMroonga(MariaDBTestCase):
WHERE MATCH (content) AGAINST ("+ブラック" IN BOOLEAN MODE)
"""
)
self
.
assertEqual
(
((
datetime
.
date
(
2013
,
4
,
23
),
u
'ブラックコーヒーを飲んだ。'
),),
((
datetime
.
date
(
2013
,
4
,
23
),
'ブラックコーヒーを飲んだ。'
),),
cnx
.
store_result
().
fetch_row
(
maxrows
=
2
),
)
...
...
software/erp5/test/test/test_wcfs.py
View file @
b696fd3d
...
...
@@ -16,7 +16,6 @@
# See COPYING file for full licensing terms.
# See https://www.nexedi.com/licensing for rationale and options.
from
__future__
import
absolute_import
import
json
import
os.path
import
unittest
...
...
@@ -76,5 +75,5 @@ def lookupMount(zurl):
# readfile returns content of file @path.
def
readfile
(
path
):
with
open
(
path
,
'r'
)
as
f
:
with
open
(
path
)
as
f
:
return
f
.
read
()
software/erp5/upgrade_test/setup.py
View file @
b696fd3d
...
...
@@ -44,7 +44,6 @@ setup(name=name,
'slapos.core'
,
'supervisor'
,
'slapos.libnetworkcache'
,
'typing; python_version<"3"'
,
],
test_suite
=
'test'
,
)
software/erp5/upgrade_test/test.py
View file @
b696fd3d
...
...
@@ -35,8 +35,8 @@ import tempfile
import
time
import
requests
import
six.moves.urllib
as
urllib
import
six.moves.xmlrpc_
client
import
urllib.parse
import
xmlrpc.
client
import
urllib3
from
slapos.grid.utils
import
md5digest
...
...
@@ -83,8 +83,8 @@ class ERP5UpgradeTestCase(SlapOSInstanceTestCase):
@
classmethod
def
setUpClass
(
cls
):
# request and instan
c
iate with old software url
super
(
ERP5UpgradeTestCase
,
cls
).
setUpClass
()
# request and instan
t
iate with old software url
super
().
setUpClass
()
cls
.
setUpOldInstance
()
...
...
@@ -155,7 +155,7 @@ class TestERP5Upgrade(ERP5UpgradeTestCase):
# wait for old site creation
cls
.
session
.
get
(
'{zope_base_url}/person_module'
.
format
(
zope_base_url
=
cls
.
zope_base_url
)
,
f'
{
cls
.
zope_base_url
}
/person_module'
,
auth
=
requests
.
auth
.
HTTPBasicAuth
(
username
=
param_dict
[
'inituser-login'
],
password
=
param_dict
[
'inituser-password'
],
...
...
@@ -171,16 +171,12 @@ class TestERP5Upgrade(ERP5UpgradeTestCase):
ssl_context
=
ssl
.
create_default_context
()
ssl_context
.
check_hostname
=
False
ssl_context
.
verify_mode
=
ssl
.
CERT_NONE
erp5_xmlrpc_client
=
six
.
moves
.
xmlrpc_
client
.
ServerProxy
(
erp5_xmlrpc_client
=
xmlrpc
.
client
.
ServerProxy
(
cls
.
authenticated_zope_base_url
,
context
=
ssl_context
,
)
# BBB use as a context manager only on python3
if
sys
.
version_info
<
(
3
,
):
with
erp5_xmlrpc_client
:
yield
erp5_xmlrpc_client
else
:
with
erp5_xmlrpc_client
:
yield
erp5_xmlrpc_client
def
addPythonScript
(
script_id
,
params
,
body
):
with
getXMLRPCClient
()
as
erp5_xmlrpc_client
:
...
...
@@ -188,7 +184,7 @@ class TestERP5Upgrade(ERP5UpgradeTestCase):
try
:
custom
.
manage_addProduct
.
PythonScripts
.
manage_addPythonScript
(
script_id
)
except
six
.
moves
.
xmlrpc_
client
.
ProtocolError
as
e
:
except
xmlrpc
.
client
.
ProtocolError
as
e
:
if
e
.
errcode
!=
302
:
raise
getattr
(
custom
,
script_id
).
ZPythonScriptHTML_editAction
(
...
...
software/fluentd/software-py2.cfg
deleted
100644 → 0
View file @
405f2b7e
[buildout]
extends =
software.cfg
[python]
part = python2.7
software/fluentd/test/setup.py
View file @
b696fd3d
...
...
@@ -46,7 +46,6 @@ setup(name=name,
'slapos.libnetworkcache'
,
'erp5.util'
,
'supervisor'
,
'six'
,
],
zip_safe
=
True
,
test_suite
=
'test'
,
...
...
software/fluentd/test/test.py
View file @
b696fd3d
...
...
@@ -34,11 +34,10 @@ import struct
import
subprocess
import
tempfile
import
time
import
six
import
sys
from
six.moves.SimpleHTTPS
erver
import
SimpleHTTPRequestHandler
from
s
ix.moves.s
ocketserver
import
StreamRequestHandler
,
TCPServer
from
http.s
erver
import
SimpleHTTPRequestHandler
from
socketserver
import
StreamRequestHandler
,
TCPServer
from
slapos.testing.testcase
import
makeModuleSetUpAndTestCaseClass
from
slapos.testing.utils
import
findFreeTCPPort
...
...
@@ -49,8 +48,8 @@ FLUSH_INTERVAL = 1
setUpModule
,
SlapOSInstanceTestCase
=
makeModuleSetUpAndTestCaseClass
(
os
.
path
.
abspath
(
os
.
path
.
join
(
os
.
path
.
dirname
(
__file__
),
'..'
,
'software%s.cfg'
%
(
"-py2"
if
six
.
PY2
else
""
))))
os
.
path
.
join
(
os
.
path
.
dirname
(
__file__
),
'..'
,
'software.cfg'
)))
class
FluentdTestCase
(
SlapOSInstanceTestCase
):
__partition_reference__
=
'fluentd'
...
...
@@ -135,11 +134,11 @@ class WendelinTutorialTestCase(FluentdTestCase):
return
subprocess
.
check_output
(
[
self
.
_fluentd_bin
,
'-c'
,
conf_path
,
'--dry-run'
],
env
=
{
'GEM_PATH'
:
self
.
_gem_path
},
universal_newlines
=
True
,
text
=
True
,
)
def
_test_configuration
(
self
,
expected_str
):
self
.
assertRegex
pMatches
(
self
.
assertRegex
(
self
.
read_fluentd_conf
(
self
.
_conf
),
expected_str
,
)
...
...
@@ -168,12 +167,12 @@ class SensorConfTestCase(WendelinTutorialTestCase):
@
classmethod
def
sensor_conf
(
cls
,
script_path
):
return
'''
\
return
f
'''
\
<source>
@type exec
tag tag.name
command
%s %s
run_interval
%s
s
command
{
sys
.
executable
}
{
script_path
}
run_interval
{
FLUSH_INTERVAL
}
s
<parse>
keys pressure, humidity, temperature
</parse>
...
...
@@ -182,25 +181,25 @@ class SensorConfTestCase(WendelinTutorialTestCase):
@type forward
<server>
name myserver1
host
%s
host
{
cls
.
_ipv6_address
}
</server>
<buffer>
flush_mode immediate
</buffer>
</match>'''
%
(
sys
.
executable
,
script_path
,
FLUSH_INTERVAL
,
cls
.
_ipv6_address
)
</match>'''
@
classmethod
def
sensor_script
(
cls
,
measurementList
):
return
'''
\
#!/usr/bin/python
measurement_text
=
"
\
t
"
.
join
(
measurementList
)
return
f'''
\
#!
{
sys
.
executable
}
# -*- coding: utf-8 -*-
print("
%s")'''
%
"
\
t
"
.
join
(
measurementList
)
print("
{
measurement_text
}
")'''
def
test_configuration
(
self
):
self
.
_test_configuration
(
r'adding forwarding server \'myserver1\' host="%s" port=%s weight=60'
%
(
self
.
_ipv6_address
,
FLUENTD_PORT
)
fr'adding forwarding server \'myserver1\' host="
{
self
.
_ipv6_address
}
" port=
{
FLUENTD_PORT
}
weight=60'
)
def
test_send_data
(
self
):
...
...
@@ -229,25 +228,24 @@ class GatewayConfTestCase(WendelinTutorialTestCase):
@
classmethod
def
gateway_conf
(
cls
,
fluentd_port
,
wendelin_port
):
return
'''
\
return
f
'''
\
<source>
@type forward
port
%s
bind
%s
port
{
fluentd_port
}
bind
{
cls
.
_ipv6_address
}
</source>
<match tag.name>
@type wendelin
streamtool_uri http://[
%s]:%s
/erp5/portal_ingestion_policies/default
streamtool_uri http://[
{
cls
.
_ipv6_address
}
]:
{
wendelin_port
}
/erp5/portal_ingestion_policies/default
user foo
password bar
<buffer>
flush_mode interval
@type file
path fluentd-buffer-file/
flush_interval
%s
s
flush_interval
{
FLUSH_INTERVAL
}
s
</buffer>
</match>'''
%
(
fluentd_port
,
cls
.
_ipv6_address
,
cls
.
_ipv6_address
,
wendelin_port
,
FLUSH_INTERVAL
)
</match>'''
@
classmethod
def
get_configuration
(
cls
):
...
...
software/gitlab/test/test.py
View file @
b696fd3d
...
...
@@ -27,7 +27,7 @@
import
os
import
logging
from
six.moves.
urllib.parse
import
urlparse
from
urllib.parse
import
urlparse
import
requests
...
...
software/kvm/test/setup.py
View file @
b696fd3d
...
...
@@ -48,7 +48,6 @@ setup(name=name,
'erp5.util'
,
'supervisor'
,
'psutil'
,
'six'
,
],
zip_safe
=
True
,
test_suite
=
'test'
,
...
...
software/kvm/test/test.py
View file @
b696fd3d
...
...
@@ -25,7 +25,8 @@
#
##############################################################################
import
six.moves.http_client
as
httplib
import
http.client
as
httplib
import
http.server
import
json
import
os
import
glob
...
...
@@ -33,15 +34,13 @@ import hashlib
import
psutil
import
re
import
requests
import
six
import
slapos.util
import
sqlite3
from
six.moves.
urllib.parse
import
parse_qs
,
urlparse
from
urllib.parse
import
parse_qs
,
urlparse
import
unittest
import
subprocess
import
tempfile
import
six.moves.socketserver
as
SocketServer
from
six.moves
import
SimpleHTTPServer
import
socketserver
as
SocketServer
import
multiprocessing
import
time
import
shutil
...
...
@@ -117,7 +116,7 @@ bootstrap_machine_param_dict = {
}
class
KvmMixin
(
object
)
:
class
KvmMixin
:
def
getConnectionParameterDictJson
(
self
):
return
json
.
loads
(
self
.
computer_partition
.
getConnectionParameterDict
()[
'_'
])
...
...
@@ -142,10 +141,10 @@ class KvmMixin(object):
for
hash_file
in
hash_file_list
])
with
self
.
slap
.
instance_supervisor_rpc
as
supervisor
:
running_process_info
=
'
\
n
'
.
join
(
sorted
(
[
running_process_info
=
'
\
n
'
.
join
(
sorted
(
'%(group)s:%(name)s %(statename)s'
%
q
for
q
in
supervisor
.
getAllProcessInfo
()
if
q
[
'name'
]
!=
'watchdog'
and
q
[
'group'
]
!=
'watchdog'
]
))
if
q
[
'name'
]
!=
'watchdog'
and
q
[
'group'
]
!=
'watchdog'
))
return
running_process_info
.
replace
(
hash_value
,
'{hash}'
).
replace
(
kvm_hash_value
,
'{kvm-hash-value}'
)
...
...
@@ -164,14 +163,14 @@ class KvmMixin(object):
state
=
state
)
class
KvmMixinJson
(
object
)
:
class
KvmMixinJson
:
@
classmethod
def
getInstanceParameterDict
(
cls
):
return
{
'_'
:
json
.
dumps
(
super
(
KvmMixinJson
,
cls
).
getInstanceParameterDict
())}
'_'
:
json
.
dumps
(
super
().
getInstanceParameterDict
())}
def
rerequestInstance
(
self
,
parameter_dict
,
*
args
,
**
kwargs
):
return
super
(
KvmMixinJson
,
self
).
rerequestInstance
(
return
super
().
rerequestInstance
(
parameter_dict
=
{
'_'
:
json
.
dumps
(
parameter_dict
)},
*
args
,
**
kwargs
)
...
...
@@ -196,10 +195,10 @@ class TestInstance(InstanceTestCase, KvmMixin):
{
'ipv6'
:
self
.
_ipv6_address
,
'maximum-extra-disk-amount'
:
'0'
,
'monitor-base-url'
:
'https://[%s]:8026'
%
(
self
.
_ipv6_address
,)
,
'nat-rule-port-tcp-22'
:
'%s : 10022'
%
(
self
.
_ipv6_address
,)
,
'nat-rule-port-tcp-443'
:
'%s : 10443'
%
(
self
.
_ipv6_address
,)
,
'nat-rule-port-tcp-80'
:
'%s : 10080'
%
(
self
.
_ipv6_address
,)
,
'monitor-base-url'
:
f'https://[
{
self
.
_ipv6_address
}
]:8026'
,
'nat-rule-port-tcp-22'
:
f'
{
self
.
_ipv6_address
}
: 10022'
,
'nat-rule-port-tcp-443'
:
f'
{
self
.
_ipv6_address
}
: 10443'
,
'nat-rule-port-tcp-80'
:
f'
{
self
.
_ipv6_address
}
: 10080'
,
}
)
self
.
assertEqual
(
set
(
present_key_list
),
set
(
assert_key_list
))
...
...
@@ -275,8 +274,8 @@ class TestMemoryManagement(InstanceTestCase, KvmMixin):
qemu_wrapper
=
QemuQMPWrapper
(
os
.
path
.
join
(
self
.
computer_partition_root_path
,
'var'
,
'qmp_socket'
))
ram_mb
=
sum
(
[
q
[
'size'
]
for
q
in
qemu_wrapper
.
getMemoryInfo
()[
'hotplugged'
]
]
)
/
1024
/
1024
q
[
'size'
]
for
q
in
qemu_wrapper
.
getMemoryInfo
()[
'hotplugged'
])
/
1024
/
1024
cpu_count
=
len
(
[
q
[
'CPU'
]
for
q
in
qemu_wrapper
.
getCPUInfo
()[
'hotplugged'
]])
return
{
'cpu_count'
:
cpu_count
,
'ram_mb'
:
ram_mb
}
...
...
@@ -374,8 +373,7 @@ class MonitorAccessMixin(KvmMixin):
connection_xml
=
partition_information
.
get
(
'connection_xml'
)
if
not
connection_xml
:
continue
connection_dict
=
json
.
loads
(
slapos
.
util
.
xml2dict
(
connection_xml
if
six
.
PY3
else
connection_xml
.
encode
(
'utf-8'
))[
'_'
])
connection_dict
=
json
.
loads
(
slapos
.
util
.
xml2dict
(
connection_xml
)[
'_'
])
monitor_base_url
=
connection_dict
.
get
(
'monitor-base-url'
)
if
not
monitor_base_url
:
continue
...
...
@@ -660,22 +658,22 @@ class TestInstanceResilient(InstanceTestCase, KvmMixin):
connection_parameter_dict
.
pop
(
k
)
self
.
assertIn
(
'feed-url-kvm-1-pull'
,
connection_parameter_dict
)
feed_pull
=
connection_parameter_dict
.
pop
(
'feed-url-kvm-1-pull'
)
self
.
assertRegex
pMatches
(
self
.
assertRegex
(
feed_pull
,
'http://
\
\
[
%s
\
\
]:[0-9][0-9][0-9][0-9]/get/local-ir0-kvm-1-pull'
%
(
self
.
_ipv6_address
,
))
'http://
\
\
[
{}
\
\
]:[0-9][0-9][0-9][0-9]/get/local-ir0-kvm-1-pull'
.
format
(
self
.
_ipv6_address
))
feed_push
=
connection_parameter_dict
.
pop
(
'feed-url-kvm-1-push'
)
self
.
assertRegex
pMatches
(
self
.
assertRegex
(
feed_push
,
'http://
\
\
[
%s
\
\
]:[0-9][0-9][0-9][0-9]/get/local-ir0-kvm-1-push'
%
(
self
.
_ipv6_address
,
))
'http://
\
\
[
{}
\
\
]:[0-9][0-9][0-9][0-9]/get/local-ir0-kvm-1-push'
.
format
(
self
.
_ipv6_address
))
self
.
assertEqual
(
connection_parameter_dict
,
{
'ipv6'
:
self
.
_ipv6_address
,
'monitor-base-url'
:
'https://[%s]:8160'
%
(
self
.
_ipv6_address
,)
,
'monitor-base-url'
:
f'https://[
{
self
.
_ipv6_address
}
]:8160'
,
'monitor-user'
:
'admin'
,
'takeover-kvm-1-url'
:
'http://[%s]:9263/'
%
(
self
.
_ipv6_address
,)
,
'takeover-kvm-1-url'
:
f'http://[
{
self
.
_ipv6_address
}
]:9263/'
,
}
)
self
.
assertEqual
(
set
(
present_key_list
),
set
(
assert_key_list
))
...
...
@@ -823,10 +821,10 @@ class TestInstanceNbdServerJson(
pass
class
HttpHandler
(
SimpleHTTPS
erver
.
SimpleHTTPRequestHandler
):
class
HttpHandler
(
http
.
s
erver
.
SimpleHTTPRequestHandler
):
def
log_message
(
self
,
*
args
):
if
os
.
environ
.
get
(
'SLAPOS_TEST_DEBUG'
):
return
SimpleHTTPS
erver
.
SimpleHTTPRequestHandler
.
log_message
(
self
,
*
args
)
return
http
.
s
erver
.
SimpleHTTPRequestHandler
.
log_message
(
self
,
*
args
)
else
:
return
...
...
@@ -873,18 +871,19 @@ class FakeImageServerMixin(KvmMixin):
cls
.
server_process
=
multiprocessing
.
Process
(
target
=
server
.
serve_forever
,
name
=
'FakeImageHttpServer'
)
cls
.
server_process
.
start
()
server
.
socket
.
close
()
finally
:
os
.
chdir
(
old_dir
)
@
classmethod
def
stopImageHttpServer
(
cls
):
cls
.
logger
.
debug
(
'Stopping process %s'
%
(
cls
.
server_process
,)
)
cls
.
logger
.
debug
(
'Stopping process %s'
,
cls
.
server_process
)
cls
.
server_process
.
join
(
10
)
cls
.
server_process
.
terminate
()
time
.
sleep
(
0.1
)
if
cls
.
server_process
.
is_alive
():
cls
.
logger
.
warning
(
'Process %s still alive'
%
(
cls
.
server_process
,
)
)
'Process %s still alive'
,
cls
.
server_process
)
shutil
.
rmtree
(
cls
.
image_source_directory
)
...
...
@@ -932,11 +931,11 @@ class TestBootImageUrlList(InstanceTestCase, FakeImageServerMixin):
@
classmethod
def
setUpClass
(
cls
):
cls
.
startImageHttpServer
()
super
(
TestBootImageUrlList
,
cls
).
setUpClass
()
super
().
setUpClass
()
@
classmethod
def
tearDownClass
(
cls
):
super
(
TestBootImageUrlList
,
cls
).
tearDownClass
()
super
().
tearDownClass
()
cls
.
stopImageHttpServer
()
def
tearDown
(
self
):
...
...
@@ -947,7 +946,7 @@ class TestBootImageUrlList(InstanceTestCase, FakeImageServerMixin):
# 2nd ...move instance to "default" state
self
.
rerequestInstance
({})
self
.
slap
.
waitForInstance
(
max_retry
=
10
)
super
(
TestBootImageUrlList
,
self
).
tearDown
()
super
().
tearDown
()
def
getRunningImageList
(
self
,
kvm_instance_partition
,
...
...
@@ -993,8 +992,8 @@ class TestBootImageUrlList(InstanceTestCase, FakeImageServerMixin):
self.assertEqual(
[
'
$
{
inst
}
/
srv
/%
s
/%
s
' % (self.image_directory, self.fake_image_md5sum)
,
'
$
{
inst
}
/
srv
/%
s
/%
s
' % (self.image_directory, self.fake_image2_md5sum)
,
f'
$
{{
inst
}}
/
srv
/
{
self
.
image_directory
}
/
{
self
.
fake_image_md5sum
}
'
,
f'
$
{{
inst
}}
/
srv
/
{
self
.
image_directory
}
/
{
self
.
fake_image2_md5sum
}
'
,
'
$
{
shared
}
/
debian
-
$
{
ver
}
-
amd64
-
netinst
.
iso
',
],
self.getRunningImageList(kvm_instance_partition)
...
...
@@ -1014,8 +1013,8 @@ class TestBootImageUrlList(InstanceTestCase, FakeImageServerMixin):
self.assertEqual(
[
'
$
{
inst
}
/
srv
/%
s
/%
s
' % (self.image_directory, self.fake_image3_md5sum)
,
'
$
{
inst
}
/
srv
/%
s
/%
s
' % (self.image_directory, self.fake_image2_md5sum)
,
f'
$
{{
inst
}}
/
srv
/
{
self
.
image_directory
}
/
{
self
.
fake_image3_md5sum
}
'
,
f'
$
{{
inst
}}
/
srv
/
{
self
.
image_directory
}
/
{
self
.
fake_image2_md5sum
}
'
,
'
$
{
shared
}
/
debian
-
$
{
ver
}
-
amd64
-
netinst
.
iso
',
],
self.getRunningImageList(kvm_instance_partition)
...
...
@@ -1159,9 +1158,9 @@ class TestBootImageUrlSelect(TestBootImageUrlList):
def
test_together
(
self
):
partition_parameter_kw
=
{
'boot-image-url-list'
:
"
%s#%s"
%
(
'boot-image-url-list'
:
"
{}#{}"
.
format
(
self
.
fake_image
,
self
.
fake_image_md5sum
),
'boot-image-url-select'
:
'["
%s#%s"]'
%
(
'boot-image-url-select'
:
'["
{}#{}"]'
.
format
(
self
.
fake_image
,
self
.
fake_image_md5sum
)
}
self
.
rerequestInstance
(
partition_parameter_kw
)
...
...
@@ -1183,10 +1182,10 @@ class TestBootImageUrlSelect(TestBootImageUrlList):
self
.
assertEqual
(
[
'${
inst}/srv/boot-image-url-select-repository/%s'
%
(
self
.
fake_image_md5sum
,
),
'${
inst}/srv/boot-image-url-list-repository/%s'
%
(
self
.
fake_image_md5sum
,
),
'${
{inst}}/srv/boot-image-url-select-repository/{}'
.
format
(
self
.
fake_image_md5sum
),
'${
{inst}}/srv/boot-image-url-list-repository/{}'
.
format
(
self
.
fake_image_md5sum
),
'${shared}/debian-${ver}-amd64-netinst.iso'
,
],
self
.
getRunningImageList
(
kvm_instance_partition
)
...
...
@@ -1258,12 +1257,12 @@ class TestBootImageUrlListKvmCluster(InstanceTestCase, FakeImageServerMixin):
config_file_name
=
'boot-image-url-list.conf'
def
setUp
(
self
):
super
(
TestBootImageUrlListKvmCluster
,
self
).
setUp
()
super
().
setUp
()
self
.
startImageHttpServer
()
def
tearDown
(
self
):
self
.
stopImageHttpServer
()
super
(
TestBootImageUrlListKvmCluster
,
self
).
tearDown
()
super
().
tearDown
()
@
classmethod
def
getInstanceParameterDict
(
cls
):
...
...
@@ -1302,12 +1301,12 @@ class TestBootImageUrlListKvmCluster(InstanceTestCase, FakeImageServerMixin):
KVM1_config
=
os
.
path
.
join
(
self
.
slap
.
instance_directory
,
self
.
__partition_reference__
+
'2'
,
'etc'
,
self
.
config_file_name
)
with
open
(
KVM0_config
,
'r'
)
as
fh
:
with
open
(
KVM0_config
)
as
fh
:
self
.
assertEqual
(
self
.
input_value
%
(
self
.
fake_image
,
self
.
fake_image_md5sum
),
fh
.
read
().
strip
()
)
with
open
(
KVM1_config
,
'r'
)
as
fh
:
with
open
(
KVM1_config
)
as
fh
:
self
.
assertEqual
(
self
.
input_value
%
(
self
.
fake_image2
,
self
.
fake_image2_md5sum
),
fh
.
read
().
strip
()
...
...
@@ -1340,11 +1339,11 @@ class TestNatRules(KvmMixin, InstanceTestCase):
self
.
assertIn
(
'nat-rule-port-tcp-200'
,
connection_parameter_dict
)
self
.
assertEqual
(
'%s : 10100'
%
(
self
.
_ipv6_address
,)
,
f'
{
self
.
_ipv6_address
}
: 10100'
,
connection_parameter_dict
[
'nat-rule-port-tcp-100'
]
)
self
.
assertEqual
(
'%s : 10200'
%
(
self
.
_ipv6_address
,)
,
f'
{
self
.
_ipv6_address
}
: 10200'
,
connection_parameter_dict
[
'nat-rule-port-tcp-200'
]
)
...
...
@@ -1388,13 +1387,13 @@ class TestNatRulesKvmCluster(InstanceTestCase):
def
test
(
self
):
host_fwd_entry
=
self
.
getRunningHostFwd
()
self
.
assertIn
(
'hostfwd=tcp:%s:10100-:100'
%
(
self
.
_ipv4_address
,)
,
f'hostfwd=tcp:
{
self
.
_ipv4_address
}
:10100-:100'
,
host_fwd_entry
)
self
.
assertIn
(
'hostfwd=tcp:%s:10200-:200'
%
(
self
.
_ipv4_address
,)
,
f'hostfwd=tcp:
{
self
.
_ipv4_address
}
:10200-:200'
,
host_fwd_entry
)
self
.
assertIn
(
'hostfwd=tcp:%s:10300-:300'
%
(
self
.
_ipv4_address
,)
,
f'hostfwd=tcp:
{
self
.
_ipv4_address
}
:10300-:300'
,
host_fwd_entry
)
...
...
@@ -1419,10 +1418,10 @@ class TestWhitelistFirewall(InstanceTestCase):
try
:
self
.
content_json
=
json
.
loads
(
content
)
except
ValueError
:
self
.
fail
(
'Failed to parse json of %r'
%
(
content
,)
)
self
.
fail
(
f'Failed to parse json of
{
content
!
r
}
'
)
self
.
assertTrue
(
isinstance
(
self
.
content_json
,
list
))
# check /etc/resolv.conf
with
open
(
'/etc/resolv.conf'
,
'r'
)
as
f
:
with
open
(
'/etc/resolv.conf'
)
as
f
:
resolv_conf_ip_list
=
[]
for
line
in
f
.
readlines
():
if
line
.
startswith
(
'nameserver'
):
...
...
@@ -1451,7 +1450,7 @@ class TestWhitelistFirewallRequest(TestWhitelistFirewall):
}
def
test
(
self
):
super
(
TestWhitelistFirewallRequest
,
self
).
test
()
super
().
test
()
self
.
assertIn
(
'2.2.2.2'
,
self
.
content_json
)
self
.
assertIn
(
'3.3.3.3'
,
self
.
content_json
)
self
.
assertIn
(
'4.4.4.4'
,
self
.
content_json
)
...
...
@@ -1573,7 +1572,7 @@ class TestImageDownloadController(InstanceTestCase, FakeImageServerMixin):
maxDiff
=
None
def
setUp
(
self
):
super
(
TestImageDownloadController
,
self
).
setUp
()
super
().
setUp
()
self
.
working_directory
=
tempfile
.
mkdtemp
()
self
.
destination_directory
=
os
.
path
.
join
(
self
.
working_directory
,
'destination'
)
...
...
@@ -1595,7 +1594,7 @@ class TestImageDownloadController(InstanceTestCase, FakeImageServerMixin):
def
tearDown
(
self
):
self
.
stopImageHttpServer
()
shutil
.
rmtree
(
self
.
working_directory
)
super
(
TestImageDownloadController
,
self
).
tearDown
()
super
().
tearDown
()
def
callImageDownloadController
(
self
,
*
args
):
call_list
=
[
sys
.
executable
,
self
.
image_download_controller
]
+
list
(
args
)
...
...
@@ -1618,7 +1617,7 @@ class TestImageDownloadController(InstanceTestCase, FakeImageServerMixin):
def
assertFileContent
(
self
,
path
,
content
):
self
.
assertTrue
(
os
.
path
.
exists
,
path
)
with
open
(
path
,
'r'
)
as
fh
:
with
open
(
path
)
as
fh
:
self
.
assertEqual
(
fh
.
read
(),
content
)
...
...
@@ -1788,7 +1787,7 @@ class TestParameterDefault(InstanceTestCase, KvmMixin):
self
.
slap
.
instance_directory
,
'*'
,
'bin'
,
'kvm_raw'
))
self
.
assertEqual
(
len
(
kvm_raw
),
1
)
kvm_raw
=
kvm_raw
[
0
]
with
open
(
kvm_raw
,
'r'
)
as
fh
:
with
open
(
kvm_raw
)
as
fh
:
kvm_raw
=
fh
.
read
()
self
.
assertIn
(
expected
,
kvm_raw
)
...
...
@@ -1929,7 +1928,7 @@ class TestExternalDisk(InstanceTestCase, ExternalDiskMixin):
continue
partition_store_list
=
[]
for
number
in
range
(
10
):
storage
=
os
.
path
.
join
(
external_storage_path
,
'data%s'
%
(
number
,)
)
storage
=
os
.
path
.
join
(
external_storage_path
,
f'data
{
number
}
'
)
if
not
os
.
path
.
exists
(
storage
):
os
.
mkdir
(
storage
)
partition_store
=
os
.
path
.
join
(
storage
,
partition
)
...
...
@@ -1949,7 +1948,7 @@ class TestExternalDisk(InstanceTestCase, ExternalDiskMixin):
if
line
.
strip
()
==
'[slapos]'
:
slapos_config
.
append
(
'[slapos]
\
n
'
)
slapos_config
.
append
(
'instance_storage_home = %s
\
n
'
%
(
external_storage_path
,)
)
f'instance_storage_home =
{
external_storage_path
}\
n
'
)
else
:
slapos_config
.
append
(
line
)
with
open
(
cls
.
slap
.
_slapos_config
,
'w'
)
as
fh
:
...
...
@@ -1968,7 +1967,7 @@ class TestExternalDisk(InstanceTestCase, ExternalDiskMixin):
@
classmethod
def
_setUpClass
(
cls
):
super
(
TestExternalDisk
,
cls
).
_setUpClass
()
super
().
_setUpClass
()
cls
.
working_directory
=
tempfile
.
mkdtemp
()
# setup the external_storage_list, to mimic part of slapformat
cls
.
_prepareExternalStorageList
()
...
...
@@ -1978,7 +1977,7 @@ class TestExternalDisk(InstanceTestCase, ExternalDiskMixin):
@
classmethod
def
tearDownClass
(
cls
):
cls
.
_dropExternalStorageList
()
super
(
TestExternalDisk
,
cls
).
tearDownClass
()
super
().
tearDownClass
()
shutil
.
rmtree
(
cls
.
working_directory
)
def
test
(
self
):
...
...
@@ -2026,7 +2025,7 @@ class TestExternalDiskModern(InstanceTestCase, ExternalDiskMixin):
@
classmethod
def
setUpClass
(
cls
):
super
(
TestExternalDiskModern
,
cls
).
setUpClass
()
super
().
setUpClass
()
def
getExternalDiskInstanceParameterDict
(
self
,
first
,
second
,
third
,
update_dict
=
None
):
...
...
@@ -2090,11 +2089,11 @@ class TestExternalDiskModern(InstanceTestCase, ExternalDiskMixin):
[
'file=${partition}/srv/virtual.qcow2,if=virtio,discard=on,'
'format=qcow2'
,
'file=
%s/first_disk,if=virtio,cache=writeback,format=qcow'
%
(
self
.
working_directory
,
),
'file=
{}/first_disk,if=virtio,cache=writeback,format=qcow'
.
format
(
self
.
working_directory
),
'file=${partition}/second_disk,if=virtio,cache=writeback'
,
'file=
%s/third_disk,if=virtio,cache=none'
%
(
self
.
working_directory
,
)
'file=
{}/third_disk,if=virtio,cache=none'
.
format
(
self
.
working_directory
)
]
)
update_dict
=
{
...
...
@@ -2128,9 +2127,7 @@ class TestExternalDiskModernCluster(TestExternalDiskModern):
return
'kvm-cluster'
def
getExternalDiskInstanceParameterDict
(
self
,
*
args
,
**
kwargs
):
partition_dict
=
super
(
TestExternalDiskModernCluster
,
self
).
getExternalDiskInstanceParameterDict
(
*
args
,
**
kwargs
)
partition_dict
=
super
().
getExternalDiskInstanceParameterDict
(
*
args
,
**
kwargs
)
partition_dict
.
update
({
"disable-ansible-promise"
:
True
})
return
{
"kvm-partition-dict"
:
{
...
...
@@ -2150,7 +2147,7 @@ class TestExternalDiskModernIndexRequired(InstanceTestCase, ExternalDiskMixin):
@
classmethod
def
setUpClass
(
cls
):
super
(
TestExternalDiskModernIndexRequired
,
cls
).
setUpClass
()
super
().
setUpClass
()
def
getExternalDiskInstanceParameterDict
(
self
,
first
,
second
,
third
):
return
{
...
...
@@ -2234,31 +2231,31 @@ class TestInstanceHttpServer(InstanceTestCase, KvmMixin):
@
classmethod
def
stopHttpServer
(
cls
):
cls
.
logger
.
debug
(
'Stopping process %s'
%
(
cls
.
server_process
,)
)
cls
.
logger
.
debug
(
f'Stopping process
{
cls
.
server_process
}
'
)
cls
.
server_process
.
join
(
10
)
cls
.
server_process
.
terminate
()
time
.
sleep
(
0.1
)
if
cls
.
server_process
.
is_alive
():
cls
.
logger
.
warning
(
'Process %s still alive'
%
(
cls
.
server_process
,
)
)
f'Process
{
cls
.
server_process
}
still alive'
)
shutil
.
rmtree
(
cls
.
http_directory
)
@
classmethod
def
setUpClass
(
cls
):
cls
.
startHttpServer
()
super
(
TestInstanceHttpServer
,
cls
).
setUpClass
()
super
().
setUpClass
()
@
classmethod
def
tearDownClass
(
cls
):
super
(
TestInstanceHttpServer
,
cls
).
tearDownClass
()
super
().
tearDownClass
()
cls
.
stopHttpServer
()
@
classmethod
def
getInstanceParameterDict
(
cls
):
return
{
'enable-http-server'
:
True
,
'bootstrap-script-url'
:
'
%s#%s'
%
(
'bootstrap-script-url'
:
'
{}#{}'
.
format
(
cls
.
bootstrap_script_url
,
cls
.
bootstrap_script_md5sum
),
'data-to-vm'
:
"""data
to
...
...
@@ -2280,10 +2277,10 @@ vm""",
{
'ipv6'
:
self
.
_ipv6_address
,
'maximum-extra-disk-amount'
:
'0'
,
'monitor-base-url'
:
'https://[%s]:8026'
%
(
self
.
_ipv6_address
,)
,
'nat-rule-port-tcp-22'
:
'%s : 10022'
%
(
self
.
_ipv6_address
,)
,
'nat-rule-port-tcp-443'
:
'%s : 10443'
%
(
self
.
_ipv6_address
,)
,
'nat-rule-port-tcp-80'
:
'%s : 10080'
%
(
self
.
_ipv6_address
,)
,
'monitor-base-url'
:
f'https://[
{
self
.
_ipv6_address
}
]:8026'
,
'nat-rule-port-tcp-22'
:
f'
{
self
.
_ipv6_address
}
: 10022'
,
'nat-rule-port-tcp-443'
:
f'
{
self
.
_ipv6_address
}
: 10443'
,
'nat-rule-port-tcp-80'
:
f'
{
self
.
_ipv6_address
}
: 10080'
,
}
)
self
.
assertEqual
(
set
(
present_key_list
),
set
(
assert_key_list
))
...
...
@@ -2311,11 +2308,11 @@ ihs0:whitelist-firewall-{hash} RUNNING""",
'ipv6_config.sh'
,
'netmask'
,
'network'
,
'vm-bootstrap'
],
sorted
(
os
.
listdir
(
public_dir
))
)
with
open
(
os
.
path
.
join
(
public_dir
,
'data'
)
,
'r'
)
as
fh
:
with
open
(
os
.
path
.
join
(
public_dir
,
'data'
))
as
fh
:
self
.
assertEqual
(
"""data
to
vm"""
,
fh
.
read
())
with
open
(
os
.
path
.
join
(
public_dir
,
'vm-bootstrap'
)
,
'r'
)
as
fh
:
with
open
(
os
.
path
.
join
(
public_dir
,
'vm-bootstrap'
))
as
fh
:
self
.
assertEqual
(
'bootstrap_script'
,
fh
.
read
())
...
...
software/metabase/test/setup.py
View file @
b696fd3d
...
...
@@ -45,7 +45,6 @@ setup(name=name,
'slapos.cookbook'
,
'slapos.libnetworkcache'
,
'supervisor'
,
'six'
,
'requests'
],
zip_safe
=
True
,
...
...
software/metabase/test/test.py
View file @
b696fd3d
##############################################################################
# coding: utf-8
#
# Copyright (c) 2020 Nexedi SA and Contributors. All Rights Reserved.
#
...
...
@@ -28,7 +27,7 @@
import
os
import
json
from
six.moves.
urllib
import
parse
from
urllib
import
parse
import
requests
from
slapos.testing.testcase
import
makeModuleSetUpAndTestCaseClass
...
...
software/slapos-master/test/test/test_balancer.py
View file @
b696fd3d
...
...
@@ -8,12 +8,11 @@ import shutil
import
subprocess
import
tempfile
import
time
import
urllib
import
urlparse
from
BaseHTTPServer
import
BaseHTTPRequestHandler
import
urllib.parse
from
http.server
import
BaseHTTPRequestHandler
from
typing
import
Dict
import
mock
from
unittest
import
mock
import
OpenSSL.SSL
import
pexpect
import
psutil
...
...
@@ -44,10 +43,10 @@ class EchoHTTPServer(ManagedHTTPServer):
response
=
json
.
dumps
(
{
'Path'
:
self
.
path
,
'Incoming Headers'
:
self
.
headers
.
dict
'Incoming Headers'
:
dict
(
self
.
headers
.
items
()),
},
indent
=
2
,
)
)
.
encode
(
'utf-8'
)
self
.
end_headers
()
self
.
wfile
.
write
(
response
)
...
...
@@ -67,11 +66,11 @@ class EchoHTTP11Server(ManagedHTTPServer):
response
=
json
.
dumps
(
{
'Path'
:
self
.
path
,
'Incoming Headers'
:
self
.
headers
.
dict
'Incoming Headers'
:
dict
(
self
.
headers
.
items
()),
},
indent
=
2
,
)
self
.
send_header
(
"Content-Length"
,
len
(
response
))
)
.
encode
(
'utf-8'
)
self
.
send_header
(
"Content-Length"
,
str
(
len
(
response
)
))
self
.
end_headers
()
self
.
wfile
.
write
(
response
)
...
...
@@ -100,7 +99,7 @@ class CaucaseService(ManagedResource):
os
.
mkdir
(
os
.
path
.
join
(
caucased_dir
,
'user'
))
os
.
mkdir
(
os
.
path
.
join
(
caucased_dir
,
'service'
))
backend_caucased_netloc
=
'%s:%s'
%
(
self
.
_cls
.
_ipv4_address
,
findFreeTCPPort
(
self
.
_cls
.
_ipv4_address
))
backend_caucased_netloc
=
f'
{
self
.
_cls
.
_ipv4_address
}
:
{
findFreeTCPPort
(
self
.
_cls
.
_ipv4_address
)
}
'
self
.
url
=
'http://'
+
backend_caucased_netloc
self
.
_caucased_process
=
subprocess
.
Popen
(
[
...
...
@@ -110,6 +109,7 @@ class CaucaseService(ManagedResource):
'--netloc'
,
backend_caucased_netloc
,
'--service-auto-approve-count'
,
'1'
,
],
# capture subprocess output not to pollute test's own stdout
stdout
=
subprocess
.
PIPE
,
stderr
=
subprocess
.
STDOUT
,
)
...
...
@@ -127,6 +127,7 @@ class CaucaseService(ManagedResource):
# type: () -> None
self
.
_caucased_process
.
terminate
()
self
.
_caucased_process
.
wait
()
self
.
_caucased_process
.
stdout
.
close
()
shutil
.
rmtree
(
self
.
directory
)
...
...
@@ -166,10 +167,11 @@ class BalancerTestCase(ERP5InstanceTestCase):
'backend-path-dict'
:
{
'default'
:
''
,
},
'ssl-authentication-dict'
:
{},
'ssl-authentication-dict'
:
{
'default'
:
False
},
'ssl'
:
{
'caucase-url'
:
cls
.
getManagedResource
(
"caucase"
,
CaucaseService
).
url
,
},
'timeout-dict'
:
{
'default'
:
None
},
'family-path-routing-dict'
:
{},
'path-routing-list'
:
[],
}
...
...
@@ -185,18 +187,27 @@ class BalancerTestCase(ERP5InstanceTestCase):
class
SlowHTTPServer
(
ManagedHTTPServer
):
"""An HTTP Server which reply after 2 seconds.
"""An HTTP Server which reply after a timeout.
Timeout is 2 seconds by default, and can be specified in the path of the URL
"""
class
RequestHandler
(
BaseHTTPRequestHandler
):
def
do_GET
(
self
):
# type: () -> None
self
.
send_response
(
200
)
self
.
send_header
(
"Content-Type"
,
"text/plain"
)
time
.
sleep
(
2
)
timeout
=
2
try
:
timeout
=
int
(
self
.
path
[
1
:])
except
ValueError
:
pass
time
.
sleep
(
timeout
)
self
.
end_headers
()
self
.
wfile
.
write
(
"OK
\
n
"
)
self
.
wfile
.
write
(
b"OK
\
n
"
)
log_message
=
logging
.
getLogger
(
__name__
+
'.SlowHTTPServer'
).
info
log_message
=
logging
.
getLogger
(
__name__
+
'.SlowHandler'
).
info
class
TestLog
(
BalancerTestCase
,
CrontabMixin
):
...
...
@@ -206,7 +217,7 @@ class TestLog(BalancerTestCase, CrontabMixin):
@
classmethod
def
_getInstanceParameterDict
(
cls
):
# type: () -> Dict
parameter_dict
=
super
(
TestLog
,
cls
).
_getInstanceParameterDict
()
parameter_dict
=
super
().
_getInstanceParameterDict
()
# use a slow server instead
parameter_dict
[
'dummy_http_server'
]
=
[[
cls
.
getManagedResource
(
"slow_web_server"
,
SlowHTTPServer
).
netloc
,
1
,
False
]]
return
parameter_dict
...
...
@@ -214,7 +225,7 @@ class TestLog(BalancerTestCase, CrontabMixin):
def
test_access_log_format
(
self
):
# type: () -> None
requests
.
get
(
urlparse
.
urljoin
(
self
.
default_balancer_url
,
'/url_path'
),
url
lib
.
parse
.
urljoin
(
self
.
default_balancer_url
,
'/url_path'
),
verify
=
False
,
)
time
.
sleep
(.
5
)
# wait a bit more until access is logged
...
...
@@ -254,7 +265,7 @@ class TestLog(BalancerTestCase, CrontabMixin):
'
apachedex
',
'
ApacheDex
-*
.
html
',
))
with open(apachedex_report
, 'r'
) as f:
with open(apachedex_report) as f:
report_text = f.read()
self.assertIn('
APacheDEX
', report_text)
# having this table means that apachedex could parse some lines.
...
...
@@ -301,7 +312,7 @@ class TestLog(BalancerTestCase, CrontabMixin):
error_line = error_log_file.read().splitlines()[-1]
self.assertIn('
apache
.
conf
-
D
FOREGROUND
', error_line)
# this log also include a timestamp
# This regex is for haproxy mostly, so keep it commented for now, until we can
# This regex is for haproxy mostly, so keep it commented for now, until we can
# Merge the slapos-master setup and erp5.
# self.assertRegexpMatches(error_line, r'
\
d
{
4
}
-
\
d
{
2
}
-
\
d
{
2
}
T
\
d
{
2
}:
\
d
{
2
}:
\
d
{
2
}
')
...
...
@@ -331,7 +342,7 @@ class BalancerCookieHTTPServer(ManagedHTTPServer):
# The name of this cookie is SERVERID
assert
self
.
headers
[
'X-Balancer-Current-Cookie'
]
==
'SERVERID'
self
.
end_headers
()
self
.
wfile
.
write
(
server
.
_name
)
self
.
wfile
.
write
(
server
.
_name
.
encode
(
'utf-8'
)
)
log_message
=
logging
.
getLogger
(
__name__
+
'.BalancerCookieHTTPServer'
).
info
return
RequestHandler
...
...
@@ -344,7 +355,7 @@ class TestBalancer(BalancerTestCase):
@
classmethod
def
_getInstanceParameterDict
(
cls
):
# type: () -> Dict
parameter_dict
=
super
(
TestBalancer
,
cls
).
_getInstanceParameterDict
()
parameter_dict
=
super
().
_getInstanceParameterDict
()
# use two backend servers
parameter_dict
[
'dummy_http_server'
]
=
[
...
...
@@ -373,7 +384,7 @@ class TestBalancer(BalancerTestCase):
# if backend provides a "SERVERID" cookie, balancer will overwrite it with the
# backend selected by balancing algorithm
self
.
assertIn
(
requests
.
get
(
urlparse
.
urljoin
(
self
.
default_balancer_url
,
'/set_cookie'
),
verify
=
False
).
cookies
[
'SERVERID'
],
requests
.
get
(
url
lib
.
parse
.
urljoin
(
self
.
default_balancer_url
,
'/set_cookie'
),
verify
=
False
).
cookies
[
'SERVERID'
],
(
'default-0'
,
'default-1'
),
)
...
...
@@ -400,10 +411,7 @@ class TestTestRunnerEntryPoints(BalancerTestCase):
@
classmethod
def
_getInstanceParameterDict
(
cls
):
# type: () -> Dict
parameter_dict
=
super
(
TestTestRunnerEntryPoints
,
cls
,
).
_getInstanceParameterDict
()
parameter_dict
=
super
().
_getInstanceParameterDict
()
parameter_dict
[
'dummy_http_server-test-runner-address-list'
]
=
[
[
...
...
@@ -427,18 +435,18 @@ class TestTestRunnerEntryPoints(BalancerTestCase):
)[
'default-test-runner-url-list'
]
url_0
,
url_1
,
url_2
=
test_runner_url_list
self
.
assertEqual
(
urlparse
.
urlparse
(
url_0
).
netloc
,
urlparse
.
urlparse
(
url_1
).
netloc
)
url
lib
.
parse
.
urlparse
(
url_0
).
netloc
,
url
lib
.
parse
.
urlparse
(
url_1
).
netloc
)
self
.
assertEqual
(
urlparse
.
urlparse
(
url_0
).
netloc
,
urlparse
.
urlparse
(
url_2
).
netloc
)
url
lib
.
parse
.
urlparse
(
url_0
).
netloc
,
url
lib
.
parse
.
urlparse
(
url_2
).
netloc
)
path_0
=
'/VirtualHostBase/https/{netloc}/VirtualHostRoot/_vh_unit_test_0/something'
.
format
(
netloc
=
urlparse
.
urlparse
(
url_0
).
netloc
)
netloc
=
url
lib
.
parse
.
urlparse
(
url_0
).
netloc
)
path_1
=
'/VirtualHostBase/https/{netloc}/VirtualHostRoot/_vh_unit_test_1/something'
.
format
(
netloc
=
urlparse
.
urlparse
(
url_0
).
netloc
)
netloc
=
url
lib
.
parse
.
urlparse
(
url_0
).
netloc
)
path_2
=
'/VirtualHostBase/https/{netloc}/VirtualHostRoot/_vh_unit_test_2/something'
.
format
(
netloc
=
urlparse
.
urlparse
(
url_0
).
netloc
)
netloc
=
url
lib
.
parse
.
urlparse
(
url_0
).
netloc
)
self
.
assertEqual
(
{
...
...
@@ -476,7 +484,7 @@ class TestHTTP(BalancerTestCase):
@
classmethod
def
_getInstanceParameterDict
(
cls
):
# type: () -> Dict
parameter_dict
=
super
(
TestHTTP
,
cls
).
_getInstanceParameterDict
()
parameter_dict
=
super
().
_getInstanceParameterDict
()
# use a HTTP/1.1 server instead
parameter_dict
[
'dummy_http_server'
]
=
[[
cls
.
getManagedResource
(
"HTTP/1.1 Server"
,
EchoHTTP11Server
).
netloc
,
1
,
False
]]
return
parameter_dict
...
...
@@ -497,32 +505,33 @@ class TestHTTP(BalancerTestCase):
'%{http_version}'
,
self
.
default_balancer_url
,
]),
'1.1'
,
b
'1.1'
,
)
def
test_keep_alive
(
self
):
# type: () -> None
# when doing two requests, connection is established only once
session
=
requests
.
Session
()
session
.
verify
=
False
with
requests
.
Session
()
as
session
:
session
.
verify
=
False
# do a first request, which establish a first connection
session
.
get
(
self
.
default_balancer_url
).
raise_for_status
()
# "break" new connection method and check we can make another request
with
mock
.
patch
(
"requests.packages.urllib3.connectionpool.HTTPSConnectionPool._new_conn"
,
)
as
new_conn
:
# do a first request, which establish a first connection
session
.
get
(
self
.
default_balancer_url
).
raise_for_status
()
new_conn
.
assert_not_called
()
parsed_url
=
urlparse
.
urlparse
(
self
.
default_balancer_url
)
# check that we have an open file for the ip connection
self
.
assertTrue
([
c
for
c
in
psutil
.
Process
(
os
.
getpid
()).
connections
()
if
c
.
status
==
'ESTABLISHED'
and
c
.
raddr
.
ip
==
parsed_url
.
hostname
and
c
.
raddr
.
port
==
parsed_url
.
port
])
# "break" new connection method and check we can make another request
with
mock
.
patch
(
"requests.packages.urllib3.connectionpool.HTTPSConnectionPool._new_conn"
,
)
as
new_conn
:
session
.
get
(
self
.
default_balancer_url
).
raise_for_status
()
new_conn
.
assert_not_called
()
parsed_url
=
urllib
.
parse
.
urlparse
(
self
.
default_balancer_url
)
# check that we have an open file for the ip connection
self
.
assertTrue
([
c
for
c
in
psutil
.
Process
(
os
.
getpid
()).
connections
()
if
c
.
status
==
'ESTABLISHED'
and
c
.
raddr
.
ip
==
parsed_url
.
hostname
and
c
.
raddr
.
port
==
parsed_url
.
port
])
class
ContentTypeHTTPServer
(
ManagedHTTPServer
):
...
...
@@ -539,12 +548,12 @@ class ContentTypeHTTPServer(ManagedHTTPServer):
# type: () -> None
self
.
send_response
(
200
)
if
self
.
path
==
'/'
:
self
.
send_header
(
"Content-Length"
,
0
)
self
.
send_header
(
"Content-Length"
,
'0'
)
return
self
.
end_headers
()
content_type
=
self
.
path
[
1
:]
body
=
"OK"
body
=
b
"OK"
self
.
send_header
(
"Content-Type"
,
content_type
)
self
.
send_header
(
"Content-Length"
,
len
(
body
))
self
.
send_header
(
"Content-Length"
,
str
(
len
(
body
)
))
self
.
end_headers
()
self
.
wfile
.
write
(
body
)
...
...
@@ -558,7 +567,7 @@ class TestContentEncoding(BalancerTestCase):
@
classmethod
def
_getInstanceParameterDict
(
cls
):
# type: () -> Dict
parameter_dict
=
super
(
TestContentEncoding
,
cls
).
_getInstanceParameterDict
()
parameter_dict
=
super
().
_getInstanceParameterDict
()
parameter_dict
[
'dummy_http_server'
]
=
[
[
cls
.
getManagedResource
(
"content_type_server"
,
ContentTypeHTTPServer
).
netloc
,
1
,
False
],
]
...
...
@@ -588,19 +597,19 @@ class TestContentEncoding(BalancerTestCase):
'application/x-font-opentype'
,
'application/wasm'
,):
resp
=
requests
.
get
(
urlparse
.
urljoin
(
self
.
default_balancer_url
,
content_type
),
url
lib
.
parse
.
urljoin
(
self
.
default_balancer_url
,
content_type
),
verify
=
False
,
headers
=
{
"Accept-Encoding"
:
"gzip, deflate"
,})
self
.
assertEqual
(
resp
.
headers
[
'Content-Type'
],
content_type
)
self
.
assertEqual
(
resp
.
headers
.
get
(
'Content-Encoding'
),
'gzip'
,
'
%s uses wrong encoding: %s'
%
(
content_type
,
resp
.
headers
.
get
(
'Content-Encoding'
)))
'
{} uses wrong encoding: {}'
.
format
(
content_type
,
resp
.
headers
.
get
(
'Content-Encoding'
)))
self
.
assertEqual
(
resp
.
text
,
'OK'
)
def
test_no_gzip_encoding
(
self
):
# type: () -> None
resp
=
requests
.
get
(
urlparse
.
urljoin
(
self
.
default_balancer_url
,
'/image/png'
),
verify
=
False
)
resp
=
requests
.
get
(
url
lib
.
parse
.
urljoin
(
self
.
default_balancer_url
,
'/image/png'
),
verify
=
False
)
self
.
assertNotIn
(
'Content-Encoding'
,
resp
.
headers
)
self
.
assertEqual
(
resp
.
text
,
'OK'
)
...
...
@@ -683,7 +692,7 @@ class CaucaseCertificate(ManagedResource):
cas_args
+
[
'--send-csr'
,
self
.
csr_file
,
],
).
split
()[
0
]
).
split
()[
0
]
.
decode
()
assert
csr_id
for
_
in
range
(
30
):
...
...
@@ -699,8 +708,8 @@ class CaucaseCertificate(ManagedResource):
time
.
sleep
(
1
)
else
:
raise
RuntimeError
(
'getting service certificate failed.'
)
with
open
(
self
.
cert_file
)
as
f
:
assert
'BEGIN CERTIFICATE'
in
f
.
read
()
with
open
(
self
.
cert_file
)
as
cert_file
:
assert
'BEGIN CERTIFICATE'
in
cert_file
.
read
()
def
revoke
(
self
,
caucase
):
# type: (str, CaucaseService) -> None
...
...
@@ -724,8 +733,8 @@ class TestServerTLSProvidedCertificate(BalancerTestCase):
# type: () -> Dict
server_caucase
=
cls
.
getManagedResource
(
'server_caucase'
,
CaucaseService
)
server_certificate
=
cls
.
getManagedResource
(
'server_certificate'
,
CaucaseCertificate
)
server_certificate
.
request
(
cls
.
_ipv4_address
.
decode
()
,
server_caucase
)
parameter_dict
=
super
(
TestServerTLSProvidedCertificate
,
cls
).
_getInstanceParameterDict
()
server_certificate
.
request
(
cls
.
_ipv4_address
,
server_caucase
)
parameter_dict
=
super
().
_getInstanceParameterDict
()
with
open
(
server_certificate
.
cert_file
)
as
f
:
parameter_dict
[
'ssl'
][
'cert'
]
=
f
.
read
()
with
open
(
server_certificate
.
key_file
)
as
f
:
...
...
software/slapos-master/test/test/test_erp5.py
View file @
b696fd3d
...
...
@@ -28,11 +28,11 @@
import
os
import
json
import
glob
import
urlparse
import
url
lib.
parse
import
socket
import
time
import
re
import
BaseHTTPS
erver
import
http.s
erver
import
multiprocessing
import
subprocess
...
...
@@ -44,7 +44,7 @@ from . import setUpModule
setUpModule
# pyflakes
class
TestPublishedURLIsReachableMixin
(
object
)
:
class
TestPublishedURLIsReachableMixin
:
"""Mixin that checks that default page of ERP5 is reachable.
"""
...
...
@@ -52,7 +52,7 @@ class TestPublishedURLIsReachableMixin(object):
# We access ERP5 trough a "virtual host", which should make
# ERP5 produce URLs using https://virtual-host-name:1234/virtual_host_root
# as base.
virtual_host_url
=
urlparse
.
urljoin
(
virtual_host_url
=
url
lib
.
parse
.
urljoin
(
base_url
,
'/VirtualHostBase/https/virtual-host-name:1234/{}/VirtualHostRoot/_vh_virtual_host_root/'
.
format
(
site_id
))
...
...
@@ -72,20 +72,20 @@ class TestPublishedURLIsReachableMixin(object):
total
=
60
,
backoff_factor
=
.
5
,
status_forcelist
=
(
404
,
500
,
503
))))
r
=
session
.
get
(
virtual_host_url
,
verify
=
verify
,
allow_redirects
=
False
)
self
.
assertEqual
(
r
.
status_code
,
requests
.
codes
.
found
)
# access on / are redirected to login form, with virtual host preserved
self
.
assertEqual
(
r
.
headers
.
get
(
'location'
),
'https://virtual-host-name:1234/virtual_host_root/login_form'
)
# login page can be rendered and contain the text "ERP5"
r
=
session
.
get
(
urlparse
.
urljoin
(
base_url
,
'{}/login_form'
.
format
(
site_id
)
),
verify
=
verify
,
allow_redirects
=
False
,
)
self
.
assertEqual
(
r
.
status_code
,
requests
.
codes
.
ok
)
self
.
assertIn
(
"ERP5"
,
r
.
text
)
with
session
:
r
=
session
.
get
(
virtual_host_url
,
verify
=
verify
,
allow_redirects
=
False
)
self
.
assertEqual
(
r
.
status_code
,
requests
.
codes
.
found
)
# access on / are redirected to login form, with virtual host preserved
self
.
assertEqual
(
r
.
headers
.
get
(
'location'
),
'https://virtual-host-name:1234/virtual_host_root/login_form'
)
# login page can be rendered and contain the text "ERP5"
r
=
session
.
get
(
urllib
.
parse
.
urljoin
(
base_url
,
f'
{
site_id
}
/login_form'
),
verify
=
verify
,
allow_redirects
=
False
,
)
self
.
assertEqual
(
r
.
status_code
,
requests
.
codes
.
ok
)
self
.
assertIn
(
"ERP5"
,
r
.
text
)
def
test_published_family_default_v6_is_reachable
(
self
):
"""Tests the IPv6 URL published by the root partition is reachable.
...
...
@@ -134,7 +134,7 @@ class TestJupyter(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
param_dict
=
self
.
getRootPartitionConnectionParameterDict
()
self
.
assertEqual
(
'https://[%s]:8888/tree'
%
self
.
_ipv6_address
,
f'https://[
{
self
.
_ipv6_address
}
]:8888/tree'
,
param_dict
[
'jupyter-url'
]
)
...
...
@@ -172,7 +172,7 @@ class TestApacheBalancerPorts(ERP5InstanceTestCase):
}
def
checkValidHTTPSURL
(
self
,
url
):
parsed
=
urlparse
.
urlparse
(
url
)
parsed
=
url
lib
.
parse
.
urlparse
(
url
)
self
.
assertEqual
(
parsed
.
scheme
,
'https'
)
self
.
assertTrue
(
parsed
.
hostname
)
self
.
assertTrue
(
parsed
.
port
)
...
...
@@ -182,16 +182,16 @@ class TestApacheBalancerPorts(ERP5InstanceTestCase):
param_dict
=
self
.
getRootPartitionConnectionParameterDict
()
for
family_name
in
(
'family1'
,
'family2'
):
self
.
checkValidHTTPSURL
(
param_dict
[
'family-{family_name}'
.
format
(
family_name
=
family_name
)
])
param_dict
[
f'family-
{
family_name
}
'
])
self
.
checkValidHTTPSURL
(
param_dict
[
'family-{family_name}-v6'
.
format
(
family_name
=
family_name
)
])
param_dict
[
f'family-
{
family_name
}
-v6'
])
def
test_published_test_runner_url
(
self
):
# each family's also a list of test test runner URLs, by default 3 per family
param_dict
=
self
.
getRootPartitionConnectionParameterDict
()
for
family_name
in
(
'family1'
,
'family2'
):
family_test_runner_url_list
=
param_dict
[
'{family_name}-test-runner-url-list'
.
format
(
family_name
=
family_name
)
]
f'
{
family_name
}
-test-runner-url-list'
]
self
.
assertEqual
(
3
,
len
(
family_test_runner_url_list
))
for
url
in
family_test_runner_url_list
:
self
.
checkValidHTTPSURL
(
url
)
...
...
@@ -209,23 +209,23 @@ class TestApacheBalancerPorts(ERP5InstanceTestCase):
# normal access on ipv4 and ipv6 and test runner access on ipv4 only
with
self
.
slap
.
instance_supervisor_rpc
as
supervisor
:
all_process_info
=
supervisor
.
getAllProcessInfo
()
process_info
,
=
[
p
for
p
in
all_process_info
if
p
[
'name'
]
==
'apache'
]
process_info
,
=
(
p
for
p
in
all_process_info
if
p
[
'name'
]
==
'apache'
)
apache_process
=
psutil
.
Process
(
process_info
[
'pid'
])
self
.
assertEqual
(
sorted
([
socket
.
AF_INET
]
*
4
+
[
socket
.
AF_INET6
]
*
2
),
sorted
(
[
sorted
(
c
.
family
for
c
in
apache_process
.
connections
()
if
c
.
status
==
'LISTEN'
]
))
))
def
test_haproxy_listen
(
self
):
# There is one haproxy per family
with
self
.
slap
.
instance_supervisor_rpc
as
supervisor
:
all_process_info
=
supervisor
.
getAllProcessInfo
()
process_info
,
=
[
process_info
,
=
(
p
for
p
in
all_process_info
if
p
[
'name'
].
startswith
(
'haproxy-'
)
]
)
haproxy_process
=
psutil
.
Process
(
process_info
[
'pid'
])
self
.
assertEqual
([
socket
.
AF_INET
,
socket
.
AF_INET
],
[
c
.
family
for
c
in
haproxy_process
.
connections
()
if
c
.
status
==
'LISTEN'
...
...
@@ -290,8 +290,8 @@ class TestZopeNodeParameterOverride(ERP5InstanceTestCase, TestPublishedURLIsReac
zodb
[
"pool-timeout"
]
=
"10m"
storage
[
"storage"
]
=
"root"
storage
[
"server"
]
=
zeo_addr
with
open
(
'%s/etc/zope-%s.conf'
%
(
partition
,
zope
)
)
as
f
:
conf
=
map
(
str
.
strip
,
f
.
readlines
(
))
with
open
(
f'
{
partition
}
/etc/zope-
{
zope
}
.conf'
)
as
f
:
conf
=
list
(
map
(
str
.
strip
,
f
.
readlines
()
))
i
=
conf
.
index
(
"<zodb_db root>"
)
+
1
conf
=
iter
(
conf
[
i
:
conf
.
index
(
"</zodb_db>"
,
i
)])
for
line
in
conf
:
...
...
@@ -300,23 +300,23 @@ class TestZopeNodeParameterOverride(ERP5InstanceTestCase, TestPublishedURLIsReac
if
line
==
'</zeoclient>'
:
break
checkParameter
(
line
,
storage
)
for
k
,
v
in
storage
.
ite
rite
ms
():
for
k
,
v
in
storage
.
items
():
self
.
assertIsNone
(
v
,
k
)
del
storage
else
:
checkParameter
(
line
,
zodb
)
for
k
,
v
in
zodb
.
ite
rite
ms
():
for
k
,
v
in
zodb
.
items
():
self
.
assertIsNone
(
v
,
k
)
partition
=
self
.
getComputerPartitionPath
(
'zope-a'
)
for
zope
in
x
range
(
3
):
for
zope
in
range
(
3
):
checkConf
({
"cache-size-bytes"
:
"20MB"
,
},
{
"cache-size"
:
"50MB"
,
})
partition
=
self
.
getComputerPartitionPath
(
'zope-bb'
)
for
zope
in
x
range
(
5
):
for
zope
in
range
(
5
):
checkConf
({
"cache-size-bytes"
:
"500MB"
if
zope
else
1
<<
20
,
},
{
...
...
@@ -332,19 +332,20 @@ def popenCommunicate(command_list, input_=None, **kwargs):
popen
.
kill
()
if
popen
.
returncode
!=
0
:
raise
ValueError
(
'Issue during calling %r, result was:
\
n
%s'
%
(
command_list
,
result
)
)
f'Issue during calling
{
command_list
!
r
}
, result was:
\
n
{
result
}
'
)
return
result
class
TestHandler
(
BaseHTTPS
erver
.
BaseHTTPRequestHandler
):
class
TestHandler
(
http
.
s
erver
.
BaseHTTPRequestHandler
):
def
do_GET
(
self
):
self
.
send_response
(
200
)
response
=
{
'Path'
:
self
.
path
,
'Incoming Headers'
:
self
.
headers
.
dict
}
response
=
json
.
dumps
(
response
,
indent
=
2
)
response
=
json
.
dumps
(
{
'Path'
:
self
.
path
,
'Incoming Headers'
:
{
k
.
lower
():
v
for
k
,
v
in
self
.
headers
.
items
()},
},
indent
=
2
,
).
encode
(
'utf-8'
)
self
.
end_headers
()
self
.
wfile
.
write
(
response
)
...
...
@@ -352,7 +353,7 @@ class TestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
class
TestDeploymentScriptInstantiation
(
ERP5InstanceTestCase
):
"""This check deployment script like instantiation
Low level assertions are done here in
ro
der to assure that
Low level assertions are done here in
or
der to assure that
https://lab.nexedi.com/nexedi/slapos.package/blob/master/playbook/
slapos-master-standalone.yml
works correctly
...
...
@@ -426,8 +427,8 @@ class TestDeploymentScriptInstantiation(ERP5InstanceTestCase):
1
,
len
(
backend_apache_configuration_list
)
)
backend_apache_configuration
=
open
(
backend_apache_configuration
_list
[
0
])
.
read
()
with
open
(
backend_apache_configuration_list
[
0
])
as
f
:
backend_apache_configuration
=
f
.
read
()
self
.
assertIn
(
'SSLVerifyClient require'
,
backend_apache_configuration
...
...
@@ -452,7 +453,7 @@ class TestDeploymentScriptInstantiation(ERP5InstanceTestCase):
common_name
=
'TEST-SSL-AUTH'
popenCommunicate
([
'openssl'
,
'req'
,
'-utf8'
,
'-nodes'
,
'-config'
,
openssl_config
,
'-new'
,
'-keyout'
,
key
,
'-out'
,
csr
,
'-days'
,
'3650'
],
'%s
\
n
'
%
(
common_name
,
),
'-keyout'
,
key
,
'-out'
,
csr
,
'-days'
,
'3650'
],
f'
{
common_name
}\
n
'
.
encode
(
),
stdin
=
subprocess
.
PIPE
)
popenCommunicate
([
'openssl'
,
'ca'
,
'-utf8'
,
'-days'
,
'3650'
,
'-batch'
,
'-config'
,
...
...
@@ -464,24 +465,24 @@ class TestDeploymentScriptInstantiation(ERP5InstanceTestCase):
ip
,
port
=
re
.
search
(
r'.*http:\
/
\/(.*):(\
d*)
\/.*'
,
portal_slap_line
).
groups
()
port
=
int
(
port
)
server
=
BaseHTTPS
erver
.
HTTPServer
((
ip
,
port
),
TestHandler
)
server
=
http
.
s
erver
.
HTTPServer
((
ip
,
port
),
TestHandler
)
server_process
=
multiprocessing
.
Process
(
target
=
server
.
serve_forever
,
name
=
'HTTPServer'
)
server_process
.
start
()
try
:
# assert that accessing the service endpoint results with certificate
# authentication and proper information extraction
result_json
=
requests
.
get
(
self
.
getRootPartitionConnectionParameterDict
()[
'family-service'
],
verify
=
False
,
cert
=
(
cert
,
key
)).
json
()
self
.
assertEqual
(
common_name
,
result_json
[
'Incoming Headers'
][
'remote-user'
]
)
self
.
assertEqual
(
'/erp5/portal_slap/'
,
result_json
[
'Path'
]
)
finally
:
server_process
.
join
(
10
)
server_process
.
terminate
(
)
self
.
addCleanup
(
server_process
.
terminate
)
self
.
addCleanup
(
server_process
.
join
,
10
)
server
.
socket
.
close
()
# assert that accessing the service endpoint results with certificate
# authentication and proper information extraction
result_json
=
requests
.
get
(
self
.
getRootPartitionConnectionParameterDict
()[
'family-service'
]
,
verify
=
False
,
cert
=
(
cert
,
key
)).
json
()
self
.
assertEqual
(
common_name
,
result_json
[
'Incoming Headers'
][
'remote-user'
]
)
self
.
assertEqual
(
'/erp5/portal_slap/'
,
result_json
[
'Path'
]
)
software/slapos-master/test/test/test_mariadb.py
View file @
b696fd3d
##############################################################################
# coding: utf-8
#
# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
#
...
...
@@ -29,7 +28,7 @@
import
os
import
json
import
glob
import
urlparse
import
url
lib.
parse
import
socket
import
sys
import
time
...
...
@@ -38,7 +37,7 @@ import datetime
import
subprocess
import
gzip
from
backports
import
lzma
import
lzma
import
MySQLdb
from
slapos.testing.utils
import
CrontabMixin
...
...
@@ -80,7 +79,7 @@ class MariaDBTestCase(ERP5InstanceTestCase):
def
getDatabaseConnection
(
self
):
connection_parameter_dict
=
json
.
loads
(
self
.
computer_partition
.
getConnectionParameterDict
()[
'_'
])
db_url
=
urlparse
.
urlparse
(
connection_parameter_dict
[
'database-list'
][
0
])
db_url
=
url
lib
.
parse
.
urlparse
(
connection_parameter_dict
[
'database-list'
][
0
])
self
.
assertEqual
(
'mysql'
,
db_url
.
scheme
)
self
.
assertTrue
(
db_url
.
path
.
startswith
(
'/'
))
...
...
@@ -91,6 +90,8 @@ class MariaDBTestCase(ERP5InstanceTestCase):
host
=
db_url
.
hostname
,
port
=
db_url
.
port
,
db
=
database_name
,
use_unicode
=
True
,
charset
=
'utf8mb4'
)
...
...
@@ -106,7 +107,7 @@ class TestCrontabs(MariaDBTestCase, CrontabMixin):
'mariadb-full'
,
'20500101000000.sql.gz'
,
),
'r'
)
as
dump
:
'r
t
'
)
as
dump
:
self
.
assertIn
(
'CREATE TABLE'
,
dump
.
read
())
def
test_logrotate_and_slow_query_digest
(
self
):
...
...
@@ -148,7 +149,7 @@ class TestCrontabs(MariaDBTestCase, CrontabMixin):
'slowquery_digest'
,
'slowquery_digest.txt-2050-01-01.xz'
,
)
with
lzma
.
open
(
slow_query_report
,
'r'
)
as
f
:
with
lzma
.
open
(
slow_query_report
,
'r
t
'
)
as
f
:
# this is the hash for our "select sleep(n)" slow query
self
.
assertIn
(
"ID 0xF9A57DD5A41825CA"
,
f
.
read
())
...
...
@@ -170,7 +171,7 @@ class TestCrontabs(MariaDBTestCase, CrontabMixin):
subprocess
.
check_output
(
'faketime 2050-01-01 %s'
%
check_slow_query_promise_plugin
[
'command'
],
shell
=
True
)
self
.
assertEqual
(
error_context
.
exception
.
output
,
"""
\
b
"""
\
Threshold is lower than expected:
Expected total queries : 1.0 and current is: 2
Expected slowest query : 0.1 and current is: 3
...
...
@@ -220,7 +221,7 @@ class TestMroonga(MariaDBTestCase):
"""
SELECT mroonga_normalize("ABCDあぃうぇ㍑")
"""
)
self
.
assertEqual
(((
'abcdあぃうぇリットル'
,),),
self
.
assertEqual
(((
'abcdあぃうぇリットル'
.
encode
()
,),),
cnx
.
store_result
().
fetch_row
(
maxrows
=
2
))
if
0
:
...
...
@@ -233,7 +234,7 @@ class TestMroonga(MariaDBTestCase):
"""
SELECT mroonga_normalize("aBcDあぃウェ㍑", "NormalizerMySQLUnicodeCIExceptKanaCIKanaWithVoicedSoundMark")
"""
)
self
.
assertEqual
(((
'ABCDあぃうぇ㍑'
,),),
self
.
assertEqual
(((
'ABCDあぃうぇ㍑'
.
encode
()
,),),
cnx
.
store_result
().
fetch_row
(
maxrows
=
2
))
def
test_mroonga_full_text_normalizer
(
self
):
...
...
@@ -321,7 +322,7 @@ class TestMroonga(MariaDBTestCase):
cnx
=
self
.
getDatabaseConnection
()
with
contextlib
.
closing
(
cnx
):
cnx
.
query
(
"SELECT mroonga_command('register token_filters/stem')"
)
self
.
assertEqual
(((
'true'
,),),
cnx
.
store_result
().
fetch_row
(
maxrows
=
2
))
self
.
assertEqual
(((
b
'true'
,),),
cnx
.
store_result
().
fetch_row
(
maxrows
=
2
))
cnx
.
query
(
"""
CREATE TABLE memos (
...
...
software/slapos-master/test/test/test_wcfs.py
View file @
b696fd3d
...
...
@@ -75,5 +75,5 @@ def lookupMount(zurl):
# readfile returns content of file @path.
def
readfile
(
path
):
with
open
(
path
,
'r'
)
as
f
:
with
open
(
path
)
as
f
:
return
f
.
read
()
software/slapos-sr-testing/README.md
View file @
b696fd3d
...
...
@@ -17,7 +17,6 @@ changes to the code, run tests and publish changes.
```
bash
# install this software release and request an instance
# use software-py3.cfg instead of software.cfg if the SR you want to test is written in Python 3
SR
=
https://lab.nexedi.com/nexedi/slapos/raw/1.0/software/slapos-sr-testing/software.cfg
COMP
=
slaprunner
INSTANCE_NAME
=
$COMP
...
...
software/slapos-sr-testing/software-py3.cfg
deleted
100644 → 0
View file @
405f2b7e
[buildout]
extends =
software.cfg
[python]
part = python3
[python-interpreter]
extra-eggs +=
# plantuml 0.3.0 is only available for Python 3
${slapos.test.plantuml-setup:egg}
[template]
extra =
# The following list is for SR whose buildout runs only with Python 3.
caddy-frontend ${slapos.test.caddy-frontend-setup:setup}
caucase ${slapos.test.caucase-setup:setup}
erp5testnode ${slapos.test.erp5testnode-setup:setup}
galene ${slapos.test.galene-setup:setup}
grafana ${slapos.test.grafana-setup:setup}
headless-chromium ${slapos.test.headless-chromium-setup:setup}
helloworld ${slapos.test.helloworld-setup:setup}
html5as ${slapos.test.html5as-setup:setup}
html5as-base ${slapos.test.html5as-base-setup:setup}
htmlvalidatorserver ${slapos.test.htmlvalidatorserver-setup:setup}
hugo ${slapos.test.hugo-setup:setup}
jscrawler ${slapos.test.jscrawler-setup:setup}
jstestnode ${slapos.test.jstestnode-setup:setup}
jupyter ${slapos.test.jupyter-setup:setup}
kvm ${slapos.test.kvm-setup:setup}
matomo ${slapos.test.matomo-setup:setup}
metabase ${slapos.test.metabase-setup:setup}
monitor ${slapos.test.monitor-setup:setup}
nextcloud ${slapos.test.nextcloud-setup:setup}
nginx-push-stream ${slapos.test.nginx-push-stream-setup:setup}
ors-amarisoft ${slapos.test.ors-amarisoft-setup:setup}
plantuml ${slapos.test.plantuml-setup:setup}
powerdns ${slapos.test.powerdns-setup:setup}
proftpd ${slapos.test.proftpd-setup:setup}
repman ${slapos.test.repman-setup:setup}
restic-rest-server ${slapos.test.restic_rest_server-setup:setup}
seleniumserver ${slapos.test.seleniumserver-setup:setup}
theia ${slapos.test.theia-setup:setup}
turnserver ${slapos.test.turnserver-setup:setup}
software/slapos-sr-testing/software.cfg
View file @
b696fd3d
...
...
@@ -27,9 +27,6 @@ parts =
shared-part-list =
[python]
part = python2.7
[setup-develop-egg]
recipe = zc.recipe.egg:develop
...
...
@@ -250,8 +247,7 @@ egg = slapos.core
setup = ${slapos.core-repository:location}
[python-interpreter]
eggs += ${:extra-eggs}
extra-eggs =
eggs +=
${lxml-python:egg}
${python-PyYAML:egg}
${slapos.core-setup:egg}
...
...
@@ -289,10 +285,11 @@ extra-eggs =
${slapos.test.kvm-setup:egg}
${slapos.test.matomo-setup:egg}
${slapos.test.metabase-setup:egg}
${slapos.test.ors-amarisoft-setup:egg}
${slapos.test.monitor-setup:egg}
${slapos.test.nextcloud-setup:egg}
${slapos.test.nginx-push-stream-setup:egg}
${slapos.test.ors-amarisoft-setup:egg}
${slapos.test.plantuml-setup:egg}
${slapos.test.powerdns-setup:egg}
${slapos.test.proftpd-setup:egg}
${slapos.test.re6stnet-setup:egg}
...
...
@@ -352,27 +349,46 @@ context =
tests =
json-schemas ${slapos.cookbook-setup:setup}
# The following list is for SR that work with either Python 2 and 3
# (as main Python). The test egg must supply a URL which depends on
# the version of Python that is used to run the test.
# Due to a bug in the way promises are run, we may also list some Py3-only SR
# here, to check there's no promise issue when slapos node runs with Python 2.
erp5 ${slapos.test.erp5-setup:setup}
fluentd ${slapos.test.fluentd-setup:setup}
###
${:extra}
extra =
# WARNING: This is for SR that only support Python 2.
# You should not add more lines here.
backupserver ${slapos.test.backupserver-setup:setup}
beremiz-ide ${slapos.test.beremiz-ide-setup:setup}
caddy-frontend ${slapos.test.caddy-frontend-setup:setup}
caucase ${slapos.test.caucase-setup:setup}
cloudooo ${slapos.test.cloudooo-setup:setup}
dream ${slapos.test.dream-setup:setup}
erp5 ${slapos.test.erp5-setup:setup}
erp5testnode ${slapos.test.erp5testnode-setup:setup}
fluentd ${slapos.test.fluentd-setup:setup}
galene ${slapos.test.galene-setup:setup}
gitlab ${slapos.test.gitlab-setup:setup}
grafana ${slapos.test.grafana-setup:setup}
headless-chromium ${slapos.test.headless-chromium-setup:setup}
helloworld ${slapos.test.helloworld-setup:setup}
html5as ${slapos.test.html5as-setup:setup}
html5as-base ${slapos.test.html5as-base-setup:setup}
htmlvalidatorserver ${slapos.test.htmlvalidatorserver-setup:setup}
hugo ${slapos.test.hugo-setup:setup}
jscrawler ${slapos.test.jscrawler-setup:setup}
jstestnode ${slapos.test.jstestnode-setup:setup}
jupyter ${slapos.test.jupyter-setup:setup}
kvm ${slapos.test.kvm-setup:setup}
matomo ${slapos.test.matomo-setup:setup}
metabase ${slapos.test.metabase-setup:setup}
monitor ${slapos.test.monitor-setup:setup}
nextcloud ${slapos.test.nextcloud-setup:setup}
nginx-push-stream ${slapos.test.nginx-push-stream-setup:setup}
ors-amarisoft ${slapos.test.ors-amarisoft-setup:setup}
plantuml ${slapos.test.plantuml-setup:setup}
powerdns ${slapos.test.powerdns-setup:setup}
proftpd ${slapos.test.proftpd-setup:setup}
re6stnet ${slapos.test.re6stnet-setup:setup}
repman ${slapos.test.repman-setup:setup}
restic-rest-server ${slapos.test.restic_rest_server-setup:setup}
seleniumserver ${slapos.test.seleniumserver-setup:setup}
slapos-master ${slapos.test.slapos-master-setup:setup}
slaprunner ${slapos.test.slaprunner-setup:setup}
theia ${slapos.test.theia-setup:setup}
turnserver ${slapos.test.turnserver-setup:setup}
upgrade_erp5 ${slapos.test.upgrade_erp5-setup:setup}
[versions]
...
...
@@ -397,11 +413,9 @@ PyPDF2 = 1.26.0+SlapOSPatched001
# Django 1.11 is python 2 compatible
Django = 1.11
mock = 2.0.0:whl
testfixtures = 6.11.0
funcsigs = 1.0.2
mysqlclient = 1.3.12
pexpect = 4.8.0
ptyprocess = 0.6.0
typing = 3.7.4.3
psycopg2 = 2.8.6
software/slaprunner/test/setup.py
View file @
b696fd3d
...
...
@@ -48,7 +48,6 @@ setup(name=name,
'supervisor'
,
'psutil'
,
'paramiko'
,
'six'
,
'requests'
,
],
zip_safe
=
True
,
...
...
software/slaprunner/test/test.py
View file @
b696fd3d
...
...
@@ -35,12 +35,11 @@ import subprocess
import
json
import
time
from
six.moves.
urllib.parse
import
urlparse
from
six.moves.
urllib.parse
import
quote
from
six.moves.
urllib.parse
import
urljoin
from
six.moves.
configparser
import
ConfigParser
from
urllib.parse
import
urlparse
from
urllib.parse
import
quote
from
urllib.parse
import
urljoin
from
configparser
import
ConfigParser
import
requests
import
six
from
slapos.recipe.librecipe
import
generateHashFromFiles
from
slapos.testing.testcase
import
makeModuleSetUpAndTestCaseClass
...
...
@@ -61,7 +60,7 @@ class SlaprunnerTestCase(SlapOSInstanceTestCase):
data
=
{
"path"
:
"workspace/slapos/software/%s"
%
software_release
,
}
resp
=
self
.
_postToSlaprunner
(
url
,
data
)
resp
=
self
.
_postToSlaprunner
(
url
,
data
)
self
.
assertEqual
(
requests
.
codes
.
ok
,
resp
.
status_code
)
self
.
assertNotEqual
(
json
.
loads
(
resp
.
text
)[
'code'
],
0
,
'Unexpecting result in call to setCurrentProject: %s'
%
resp
.
text
)
...
...
@@ -69,7 +68,7 @@ class SlaprunnerTestCase(SlapOSInstanceTestCase):
def
_buildSoftwareRelease
(
self
):
parameter_dict
=
self
.
computer_partition
.
getConnectionParameterDict
()
url
=
"%s/runSoftwareProfile"
%
parameter_dict
[
'url'
]
resp
=
self
.
_postToSlaprunner
(
url
,
{})
resp
=
self
.
_postToSlaprunner
(
url
,
{})
self
.
assertEqual
(
requests
.
codes
.
ok
,
resp
.
status_code
)
self
.
assertEqual
(
json
.
loads
(
resp
.
text
)[
'result'
],
True
,
'Unexpecting result in call to runSoftwareProfile: %s'
%
resp
.
text
)
...
...
@@ -77,7 +76,7 @@ class SlaprunnerTestCase(SlapOSInstanceTestCase):
def
_deployInstance
(
self
):
parameter_dict
=
self
.
computer_partition
.
getConnectionParameterDict
()
url
=
"%s/runInstanceProfile"
%
parameter_dict
[
'url'
]
resp
=
self
.
_postToSlaprunner
(
url
,
{})
resp
=
self
.
_postToSlaprunner
(
url
,
{})
self
.
assertEqual
(
requests
.
codes
.
ok
,
resp
.
status_code
)
self
.
assertEqual
(
json
.
loads
(
resp
.
text
)[
'result'
],
True
,
'Unexpecting result in call to runSoftwareProfile: %s'
%
resp
.
text
)
...
...
@@ -100,7 +99,7 @@ class SlaprunnerTestCase(SlapOSInstanceTestCase):
def
_isSoftwareReleaseReady
(
self
):
parameter_dict
=
self
.
computer_partition
.
getConnectionParameterDict
()
url
=
"%s/isSRReady"
%
parameter_dict
[
'url'
]
resp
=
self
.
_getFromSlaprunner
(
url
)
resp
=
self
.
_getFromSlaprunner
(
url
)
if
requests
.
codes
.
ok
!=
resp
.
status_code
:
return
-
1
return
resp
.
text
...
...
@@ -125,7 +124,7 @@ class SlaprunnerTestCase(SlapOSInstanceTestCase):
}
while
True
:
time
.
sleep
(
25
)
resp
=
self
.
_postToSlaprunner
(
url
,
data
)
resp
=
self
.
_postToSlaprunner
(
url
,
data
)
if
requests
.
codes
.
ok
!=
resp
.
status_code
:
continue
if
json
.
loads
(
resp
.
text
)[
"instance"
][
"state"
]
is
False
:
...
...
@@ -153,9 +152,9 @@ class SlaprunnerTestCase(SlapOSInstanceTestCase):
url
=
"%s/getFileContent"
%
parameter_dict
[
'url'
]
data
=
{
"file"
:
relative_path
"file"
:
relative_path
}
resp
=
self
.
_postToSlaprunner
(
url
,
data
)
resp
=
self
.
_postToSlaprunner
(
url
,
data
)
self
.
assertEqual
(
requests
.
codes
.
ok
,
resp
.
status_code
)
self
.
assertNotEqual
(
json
.
loads
(
resp
.
text
)[
'code'
],
0
,
'Unexpecting result in call to getFileContent: %s'
%
resp
.
text
)
...
...
@@ -189,9 +188,9 @@ class SlaprunnerTestCase(SlapOSInstanceTestCase):
parameter_dict
=
self
.
computer_partition
.
getConnectionParameterDict
()
takeover_url
=
parameter_dict
[
"takeover-%s-url"
%
scope
]
takeover_password
=
parameter_dict
[
"takeover-%s-password"
%
scope
]
resp
=
requests
.
get
(
"%s?password=%s"
%
(
takeover_url
,
takeover_password
)
,
f"
{
takeover_url
}
?password=
{
takeover_password
}
"
,
verify
=
True
)
self
.
assertEqual
(
requests
.
codes
.
ok
,
resp
.
status_code
)
self
.
assertNotIn
(
"Error"
,
resp
.
text
,
...
...
@@ -236,7 +235,7 @@ class TestWebRunnerAutorun(SlaprunnerTestCase):
@
classmethod
def
getInstanceParameterDict
(
cls
):
return
{
# Auto deploy is required for the isSRReady works.
# Auto deploy is required for the isSRReady works.
'auto-deploy'
:
'true'
,
'autorun'
:
'true'
,
'software-root'
:
os
.
path
.
join
(
cls
.
slap
.
_instance_root
,
".."
,
"soft"
),
...
...
@@ -363,7 +362,7 @@ class TestSSH(SlaprunnerTestCase):
self
.
assertTrue
(
fingerprint_from_url
.
startswith
(
'ssh-rsa-'
),
fingerprint_from_url
)
fingerprint_from_url
=
fingerprint_from_url
[
len
(
'ssh-rsa-'
):]
class
KeyPolicy
(
object
)
:
class
KeyPolicy
:
"""Accept server key and keep it in self.key for inspection
"""
def
missing_host_key
(
self
,
client
,
hostname
,
key
):
...
...
@@ -507,7 +506,7 @@ class TestResilientInstance(SlaprunnerTestCase):
# just check that keys returned on requested partition are for resilient
self
.
assertSetEqual
(
set
(
self
.
computer_partition
.
getConnectionParameterDict
().
keys
()),
set
([
{
'backend-url'
,
'feed-url-runner-1-pull'
,
'feed-url-runner-1-push'
,
...
...
@@ -520,7 +519,7 @@ class TestResilientInstance(SlaprunnerTestCase):
'takeover-runner-1-password'
,
'takeover-runner-1-url'
,
'url'
,
'webdav-url'
])
)
'webdav-url'
}
)
class
TestResilientCustomFrontend
(
TestCustomFrontend
):
instance_max_retry
=
20
...
...
@@ -589,7 +588,7 @@ class TestResilientDummyInstance(SlaprunnerTestCase):
self
.
_waitForCloneToBeReadyForTakeover
()
self
.
_doTakeover
()
self
.
slap
.
waitForInstance
(
20
)
self
.
slap
.
waitForInstance
(
20
)
previous_computer_partition
=
self
.
computer_partition
self
.
computer_partition
=
self
.
requestDefaultInstance
()
...
...
@@ -601,5 +600,5 @@ class TestResilientDummyInstance(SlaprunnerTestCase):
self
.
assertTrue
(
result_after
.
startswith
(
"Hello"
),
result_after
)
self
.
assertIn
(
result
,
result_after
,
"%s not in %s"
%
(
result
,
result_after
)
)
f"
{
result
}
not in
{
result_after
}
"
)
software/theia/test/test.py
View file @
b696fd3d
...
...
@@ -391,52 +391,49 @@ class TestTheiaEnv(TheiaTestCase):
# Start a theia shell that inherits the environment of the theia process
# This simulates the environment of a shell launched from the browser application
theia_shell_process
=
pexpect
.
spawnu
(
'{}/bin/theia-shell'
.
format
(
self
.
getPath
()),
env
=
theia_env
)
try
:
theia_shell_process
.
expect_exact
(
'Standalone SlapOS for computer `slaprunner` activated'
)
# Launch slapos node software from theia shell
theia_shell_process
.
sendline
(
'slapos node software'
)
theia_shell_process
.
expect
(
'Installing software release %s'
%
self
.
dummy_software_path
)
theia_shell_process
.
expect
(
'Finished software releases.'
)
# Get the theia shell environment
with
open
(
env_json_path
)
as
f
:
theia_shell_env
=
json
.
load
(
f
)
# Remove the env.json file to later be sure that a new one has been generated
os
.
remove
(
env_json_path
)
# Launch slapos node software service from the embedded supervisord.
# Note that we have two services, slapos-node-software and slapos-node-software-all
# The later uses --all which is what we want to use here, because the software
# is already installed and we want to install it again, this time from supervisor
embedded_run_path
=
self
.
getPath
(
'srv'
,
'runner'
,
'var'
,
'run'
)
embedded_supervisord_socket_path
=
_getSupervisordSocketPath
(
embedded_run_path
,
self
.
logger
)
with
getSupervisorRPC
(
embedded_supervisord_socket_path
)
as
embedded_supervisor
:
previous_stop_time
=
embedded_supervisor
.
getProcessInfo
(
'slapos-node-software-all'
)[
'stop'
]
embedded_supervisor
.
startProcess
(
'slapos-node-software-all'
)
for
_retries
in
range
(
20
):
time
.
sleep
(
1
)
if
embedded_supervisor
.
getProcessInfo
(
'slapos-node-software-all'
)[
'stop'
]
!=
previous_stop_time
:
break
else
:
self
.
fail
(
"the supervisord service 'slapos-node-software-all' takes too long to finish"
)
# Get the supervisord environment
with
open
(
env_json_path
)
as
f
:
supervisord_env
=
json
.
load
(
f
)
# Compare relevant variables from both environments
self
.
maxDiff
=
None
self
.
assertEqual
(
theia_shell_env
[
'PATH'
].
split
(
':'
),
supervisord_env
[
'PATH'
].
split
(
':'
))
self
.
assertEqual
(
theia_shell_env
[
'SLAPOS_CONFIGURATION'
],
supervisord_env
[
'SLAPOS_CONFIGURATION'
])
self
.
assertEqual
(
theia_shell_env
[
'SLAPOS_CLIENT_CONFIGURATION'
],
supervisord_env
[
'SLAPOS_CLIENT_CONFIGURATION'
])
self
.
assertEqual
(
theia_shell_env
[
'HOME'
],
supervisord_env
[
'HOME'
])
finally
:
# Cleanup the theia shell process
theia_shell_process
.
terminate
()
theia_shell_process
.
wait
()
self
.
addCleanup
(
theia_shell_process
.
wait
)
self
.
addCleanup
(
theia_shell_process
.
terminate
)
theia_shell_process
.
expect_exact
(
'Standalone SlapOS for computer `slaprunner` activated'
)
# Launch slapos node software from theia shell
theia_shell_process
.
sendline
(
'slapos node software'
)
theia_shell_process
.
expect
(
'Installing software release %s'
%
self
.
dummy_software_path
)
theia_shell_process
.
expect
(
'Finished software releases.'
)
# Get the theia shell environment
with
open
(
env_json_path
)
as
f
:
theia_shell_env
=
json
.
load
(
f
)
# Remove the env.json file to later be sure that a new one has been generated
os
.
remove
(
env_json_path
)
# Launch slapos node software service from the embedded supervisord.
# Note that we have two services, slapos-node-software and slapos-node-software-all
# The later uses --all which is what we want to use here, because the software
# is already installed and we want to install it again, this time from supervisor
embedded_run_path
=
self
.
getPath
(
'srv'
,
'runner'
,
'var'
,
'run'
)
embedded_supervisord_socket_path
=
_getSupervisordSocketPath
(
embedded_run_path
,
self
.
logger
)
with
getSupervisorRPC
(
embedded_supervisord_socket_path
)
as
embedded_supervisor
:
previous_stop_time
=
embedded_supervisor
.
getProcessInfo
(
'slapos-node-software-all'
)[
'stop'
]
embedded_supervisor
.
startProcess
(
'slapos-node-software-all'
)
for
_retries
in
range
(
20
):
time
.
sleep
(
1
)
if
embedded_supervisor
.
getProcessInfo
(
'slapos-node-software-all'
)[
'stop'
]
!=
previous_stop_time
:
break
else
:
self
.
fail
(
"the supervisord service 'slapos-node-software-all' takes too long to finish"
)
# Get the supervisord environment
with
open
(
env_json_path
)
as
f
:
supervisord_env
=
json
.
load
(
f
)
# Compare relevant variables from both environments
self
.
maxDiff
=
None
self
.
assertEqual
(
theia_shell_env
[
'PATH'
].
split
(
':'
),
supervisord_env
[
'PATH'
].
split
(
':'
))
self
.
assertEqual
(
theia_shell_env
[
'SLAPOS_CONFIGURATION'
],
supervisord_env
[
'SLAPOS_CONFIGURATION'
])
self
.
assertEqual
(
theia_shell_env
[
'SLAPOS_CLIENT_CONFIGURATION'
],
supervisord_env
[
'SLAPOS_CLIENT_CONFIGURATION'
])
self
.
assertEqual
(
theia_shell_env
[
'HOME'
],
supervisord_env
[
'HOME'
])
class
ResilientTheiaMixin
(
object
):
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment