Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
slapos
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
Léo-Paul Géneau
slapos
Commits
6e7330a5
Commit
6e7330a5
authored
Sep 28, 2020
by
Łukasz Nowak
Browse files
Options
Browse Files
Download
Plain Diff
Fix/slapos master instantiation
See merge request
!814
parents
b6a12b43
d2d2a1b6
Changes
5
Show whitespace changes
Inline
Side-by-side
Showing
5 changed files
with
203 additions
and
21 deletions
+203
-21
component/openssl/buildout.cfg
component/openssl/buildout.cfg
+1
-0
software/slapos-master/test/test/test_erp5.py
software/slapos-master/test/test/test_erp5.py
+198
-19
software/slapos-sr-testing/buildout.hash.cfg
software/slapos-sr-testing/buildout.hash.cfg
+1
-1
software/slapos-sr-testing/instance.cfg
software/slapos-sr-testing/instance.cfg
+1
-1
software/slapos-sr-testing/software.cfg
software/slapos-sr-testing/software.cfg
+2
-0
No files found.
component/openssl/buildout.cfg
View file @
6e7330a5
...
...
@@ -53,6 +53,7 @@ stop-on-error = true
update-command = ${:command}
command = ${coreutils-output:test} -x ${:openssl}
openssl = ${openssl:location}/bin/openssl
bin = ${openssl:location}
[openssl-1.0]
recipe = slapos.recipe.cmmi
...
...
software/slapos-master/test/test/test_erp5.py
View file @
6e7330a5
...
...
@@ -31,6 +31,10 @@ import glob
import
urlparse
import
socket
import
time
import
re
import
BaseHTTPServer
import
multiprocessing
import
subprocess
import
psutil
import
requests
...
...
@@ -46,15 +50,17 @@ class TestPublishedURLIsReachableMixin(object):
def
_checkERP5IsReachable
(
self
,
url
):
# What happens is that instanciation just create the services, but does not
# wait for ERP5 to be initialized. When this test run ERP5 instance is
# instanciated, but zope is still busy creating the site and haproxy
replies
#
with 503 Service Unavailable when zope is not started yet, with 404 when
#
erp5 site is not created, with 500 when mysql is not yet reachable, so we
# retry in a loop until we get a succesful response.
# instanciated, but zope is still busy creating the site and haproxy
#
replies with 503 Service Unavailable when zope is not started yet, with
#
404 when erp5 site is not created, with 500 when mysql is not yet
# re
achable, so we re
try in a loop until we get a succesful response.
for
i
in
range
(
1
,
60
):
r
=
requests
.
get
(
url
,
verify
=
False
)
# XXX can we get CA from caucase already ?
# XXX can we get CA from caucase already ?
r
=
requests
.
get
(
url
,
verify
=
False
)
if
r
.
status_code
!=
requests
.
codes
.
ok
:
delay
=
i
*
2
self
.
logger
.
warn
(
"ERP5 was not available, sleeping for %ds and retrying"
,
delay
)
self
.
logger
.
warn
(
"ERP5 was not available, sleeping for %ds and retrying"
,
delay
)
time
.
sleep
(
delay
)
continue
r
.
raise_for_status
()
...
...
@@ -77,7 +83,8 @@ class TestPublishedURLIsReachableMixin(object):
urlparse
.
urljoin
(
param_dict
[
'family-default'
],
param_dict
[
'site-id'
]))
class
TestDefaultParameters
(
ERP5InstanceTestCase
,
TestPublishedURLIsReachableMixin
):
class
TestDefaultParameters
(
ERP5InstanceTestCase
,
TestPublishedURLIsReachableMixin
):
"""Test ERP5 can be instanciated with no parameters
"""
__partition_reference__
=
'defp'
...
...
@@ -125,16 +132,19 @@ class TestApacheBalancerPorts(ERP5InstanceTestCase):
self
.
assertTrue
(
parsed
.
port
)
def
test_published_family_parameters
(
self
):
# when we request two families, we have two published family-{family_name} URLs
# when we request two families, we have two published family-{family_name}
# URLs
param_dict
=
self
.
getRootPartitionConnectionParameterDict
()
for
family_name
in
(
'family1'
,
'family2'
):
self
.
checkValidHTTPSURL
(
param_dict
[
'family-{family_name}'
.
format
(
family_name
=
family_name
)])
self
.
checkValidHTTPSURL
(
param_dict
[
'family-{family_name}-v6'
.
format
(
family_name
=
family_name
)])
param_dict
[
'family-{family_name}-v6'
.
format
(
family_name
=
family_name
)])
def
test_published_test_runner_url
(
self
):
# each family's also a list of test test runner URLs, by default 3 per family
# each family's also a list of test test runner URLs, by default 3 per
# family
param_dict
=
self
.
getRootPartitionConnectionParameterDict
()
for
family_name
in
(
'family1'
,
'family2'
):
family_test_runner_url_list
=
param_dict
[
...
...
@@ -144,7 +154,8 @@ class TestApacheBalancerPorts(ERP5InstanceTestCase):
self
.
checkValidHTTPSURL
(
url
)
def
test_zope_listen
(
self
):
# we requested 3 zope in family1 and 5 zopes in family2, we should have 8 zope running.
# we requested 3 zope in family1 and 5 zopes in family2, we should have 8
# zope running.
with
self
.
slap
.
instance_supervisor_rpc
as
supervisor
:
all_process_info
=
supervisor
.
getAllProcessInfo
()
self
.
assertEqual
(
...
...
@@ -179,7 +190,8 @@ class TestApacheBalancerPorts(ERP5InstanceTestCase):
])
class
TestDisableTestRunner
(
ERP5InstanceTestCase
,
TestPublishedURLIsReachableMixin
):
class
TestDisableTestRunner
(
ERP5InstanceTestCase
,
TestPublishedURLIsReachableMixin
):
"""Test ERP5 can be instanciated without test runner.
"""
__partition_reference__
=
'distr'
...
...
@@ -192,7 +204,8 @@ class TestDisableTestRunner(ERP5InstanceTestCase, TestPublishedURLIsReachableMix
"""
# self.computer_partition_root_path is the path of root partition.
# we want to assert that no scripts exist in any partition.
bin_programs
=
map
(
os
.
path
.
basename
,
bin_programs
=
map
(
os
.
path
.
basename
,
glob
.
glob
(
self
.
computer_partition_root_path
+
"/../*/bin/*"
))
self
.
assertTrue
(
bin_programs
)
# just to check the glob was correct.
...
...
@@ -200,7 +213,8 @@ class TestDisableTestRunner(ERP5InstanceTestCase, TestPublishedURLIsReachableMix
self
.
assertNotIn
(
'runTestSuite'
,
bin_programs
)
def
test_no_apache_testrunner_port
(
self
):
# Apache only listen on two ports, there is no apache ports allocated for test runner
# Apache only listen on two ports, there is no apache ports allocated for
# test runner
with
self
.
slap
.
instance_supervisor_rpc
as
supervisor
:
all_process_info
=
supervisor
.
getAllProcessInfo
()
process_info
,
=
[
p
for
p
in
all_process_info
if
p
[
'name'
]
==
'apache'
]
...
...
@@ -213,7 +227,9 @@ class TestDisableTestRunner(ERP5InstanceTestCase, TestPublishedURLIsReachableMix
if
c
.
status
==
'LISTEN'
))
class
TestZopeNodeParameterOverride
(
ERP5InstanceTestCase
,
TestPublishedURLIsReachableMixin
):
class
TestZopeNodeParameterOverride
(
ERP5InstanceTestCase
,
TestPublishedURLIsReachableMixin
):
"""Test override zope node parameters
"""
__partition_reference__
=
'override'
...
...
@@ -228,7 +244,7 @@ class TestZopeNodeParameterOverride(ERP5InstanceTestCase, TestPublishedURLIsReac
"server"
:
{},
"cache-size-bytes"
:
"20MB"
,
"cache-size-bytes!"
:
[
(
"bb-0"
,
1
<<
20
),
(
"bb-0"
,
1
<<
20
),
(
"bb-.*"
,
"500MB"
),
],
"pool-timeout"
:
"10m"
,
...
...
@@ -299,7 +315,170 @@ class TestZopeNodeParameterOverride(ERP5InstanceTestCase, TestPublishedURLIsReac
partition
=
self
.
getComputerPartitionPath
(
'zope-bb'
)
for
zope
in
xrange
(
5
):
checkConf
({
"cache-size-bytes"
:
"500MB"
if
zope
else
1
<<
20
,
"cache-size-bytes"
:
"500MB"
if
zope
else
1
<<
20
,
},
{
"cache-size"
:
None
,
})
def
popenCommunicate
(
command_list
,
input_
=
None
,
**
kwargs
):
kwargs
.
update
(
stdout
=
subprocess
.
PIPE
,
stderr
=
subprocess
.
STDOUT
)
popen
=
subprocess
.
Popen
(
command_list
,
**
kwargs
)
result
=
popen
.
communicate
(
input_
)[
0
]
if
popen
.
returncode
is
None
:
popen
.
kill
()
if
popen
.
returncode
!=
0
:
raise
ValueError
(
'Issue during calling %r, result was:
\
n
%s'
%
(
command_list
,
result
))
return
result
class
TestHandler
(
BaseHTTPServer
.
BaseHTTPRequestHandler
):
def
do_GET
(
self
):
self
.
send_response
(
200
)
response
=
{
'Path'
:
self
.
path
,
'Incoming Headers'
:
self
.
headers
.
dict
}
response
=
json
.
dumps
(
response
,
indent
=
2
)
self
.
end_headers
()
self
.
wfile
.
write
(
response
)
class
TestDeploymentScriptInstantiation
(
ERP5InstanceTestCase
):
"""This check deployment script like instantiation
Low level assertions are done here in roder to assure that
https://lab.nexedi.com/nexedi/slapos.package/blob/master/playbook/
slapos-master-standalone.yml
works correctly
"""
__partition_reference__
=
'tdsi'
# a bit more partition is required
partition_count
=
20
@
classmethod
def
getInstanceParameterDict
(
cls
):
# As close as possible configuration to deployment script
parameter_dict
=
{
"timezone"
:
"UTC"
,
"site-id"
:
"erp5"
,
"bt5"
:
"erp5_full_text_myisam_catalog slapos_configurator"
,
"wsgi"
:
False
,
"test-runner"
:
{
"enabled"
:
False
},
# won't work anyway here
"zope-partition-dict"
:
{
"admin"
:
{
"family"
:
"admin"
,
"thread-amount"
:
4
,
"port-base"
:
2220
,
"instance-count"
:
1
},
"activities-node"
:
{
"family"
:
"activities"
,
"thread-amount"
:
4
,
"instance-count"
:
1
,
"timerserver-interval"
:
1
,
"port-base"
:
2230
},
"distribution-node"
:
{
"family"
:
"distribution"
,
"thread-amount"
:
1
,
"instance-count"
:
1
,
"port-base"
:
2210
,
"timerserver-interval"
:
1
},
"web-node"
:
{
"family"
:
"web"
,
"thread-amount"
:
2
,
"instance-count"
:
1
,
"port-base"
:
2240
},
"service-slapos"
:
{
"family"
:
"service"
,
"thread-amount"
:
2
,
"instance-count"
:
1
,
"port-base"
:
2250
,
"ssl-authentication"
:
True
,
"backend-path"
:
"/%(site-id)s/portal_slap"
}
}
}
# put shared-certificate-authority-path in controlled location
cls
.
ca_path
=
os
.
path
.
join
(
cls
.
slap
.
instance_directory
,
'ca_path'
)
parameter_dict
[
"shared-certificate-authority-path"
]
=
cls
.
ca_path
return
{
'_'
:
json
.
dumps
(
parameter_dict
)}
@
classmethod
def
callSupervisorMethod
(
cls
,
method
,
*
args
,
**
kwargs
):
with
cls
.
slap
.
instance_supervisor_rpc
as
instance_supervisor
:
return
getattr
(
instance_supervisor
,
method
)(
*
args
,
**
kwargs
)
def
test_ssl_auth
(
self
):
backend_apache_configuration_list
=
glob
.
glob
(
os
.
path
.
join
(
self
.
slap
.
instance_directory
,
'*'
,
'etc'
,
'apache'
,
'apache.conf'
))
self
.
assertEqual
(
1
,
len
(
backend_apache_configuration_list
)
)
backend_apache_configuration
=
open
(
backend_apache_configuration_list
[
0
]).
read
()
self
.
assertIn
(
'SSLVerifyClient require'
,
backend_apache_configuration
)
self
.
assertIn
(
r'RequestHeader set Remote-User %{SSL_CLIENT_S_DN_CN}s'
,
backend_apache_configuration
)
# stop haproxy, it's going to be hijacked
haproxy_name
=
':'
.
join
([
(
q
[
'group'
],
q
[
'name'
])
for
q
in
self
.
callSupervisorMethod
(
'getAllProcessInfo'
)
if
'haproxy'
in
q
[
'name'
]][
0
])
self
.
callSupervisorMethod
(
'stopProcess'
,
haproxy_name
)
# do similar certificate request like CertificateAuthorityTool
openssl_config
=
os
.
path
.
join
(
self
.
ca_path
,
'openssl.cnf'
)
key
=
os
.
path
.
join
(
self
.
ca_path
,
'private'
,
'test.key'
)
csr
=
os
.
path
.
join
(
self
.
ca_path
,
'text.csr'
)
cert
=
os
.
path
.
join
(
self
.
ca_path
,
'certs'
,
'test.crt'
)
common_name
=
'TEST-SSL-AUTH'
popenCommunicate
([
'openssl'
,
'req'
,
'-utf8'
,
'-nodes'
,
'-config'
,
openssl_config
,
'-new'
,
'-keyout'
,
key
,
'-out'
,
csr
,
'-days'
,
'3650'
],
'%s
\
n
'
%
(
common_name
,),
stdin
=
subprocess
.
PIPE
)
popenCommunicate
([
'openssl'
,
'ca'
,
'-utf8'
,
'-days'
,
'3650'
,
'-batch'
,
'-config'
,
openssl_config
,
'-out'
,
cert
,
'-infiles'
,
csr
])
# find IP and port on which hijacked process shall listen
portal_slap_line
=
[
q
for
q
in
backend_apache_configuration
.
splitlines
()
if
'portal_slap'
in
q
][
0
]
ip
,
port
=
re
.
search
(
r'.*http:\
/
\/(.*):(\
d*)
\/.*'
,
portal_slap_line
).
groups
()
port
=
int
(
port
)
server
=
BaseHTTPServer
.
HTTPServer
((
ip
,
port
),
TestHandler
)
server_process
=
multiprocessing
.
Process
(
target
=
server
.
serve_forever
,
name
=
'HTTPServer'
)
server_process
.
start
()
try
:
# assert that accessing the service endpoint results with certificate
# authentication and proper information extraction
result_json
=
requests
.
get
(
self
.
getRootPartitionConnectionParameterDict
()[
'family-service'
],
verify
=
False
,
cert
=
(
cert
,
key
)).
json
()
self
.
assertEqual
(
common_name
,
result_json
[
'Incoming Headers'
][
'remote-user'
]
)
self
.
assertEqual
(
'/erp5/portal_slap/'
,
result_json
[
'Path'
]
)
finally
:
server_process
.
join
(
10
)
server_process
.
terminate
()
software/slapos-sr-testing/buildout.hash.cfg
View file @
6e7330a5
...
...
@@ -15,4 +15,4 @@
[template]
filename = instance.cfg
md5sum =
14d2f49d20670e44c2a162bcec9e0a8e
md5sum =
4246cde0a27138e057ba1635cc621edf
software/slapos-sr-testing/instance.cfg
View file @
6e7330a5
...
...
@@ -44,7 +44,7 @@ environment =
SLAPOS_TEST_WORKING_DIR=${slapos-test-runner-environment:SLAPOS_TEST_WORKING_DIR}
[slapos-test-runner-environment]
PATH = {{ buildout['bin-directory'] }}:{{ curl_location }}/bin/:{{ faketime_location }}/bin/:/usr/bin/:/bin
PATH = {{ buildout['bin-directory'] }}:{{ curl_location }}/bin/:{{ faketime_location }}/bin/:
{{ openssl_location }}/bin/:
/usr/bin/:/bin
SLAPOS_TEST_IPV4 = ${slap-configuration:ipv4-random}
SLAPOS_TEST_IPV6 = ${slap-configuration:ipv6-random}
SLAPOS_TEST_WORKING_DIR = ${directory:working-dir}
...
...
software/slapos-sr-testing/software.cfg
View file @
6e7330a5
...
...
@@ -3,6 +3,7 @@
extends =
../../component/bcrypt/buildout.cfg
../../component/curl/buildout.cfg
../../component/openssl/buildout.cfg
../../component/git/buildout.cfg
../../component/faketime/buildout.cfg
../../component/pillow/buildout.cfg
...
...
@@ -240,6 +241,7 @@ context =
key slapos_location slapos-repository:location
key interpreter eggs:interpreter
key curl_location curl:location
key openssl_location openssl-output:bin
key faketime_location faketime:location
key tests :tests
tests =
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment