Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
slapos
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
Léo-Paul Géneau
slapos
Commits
6e7330a5
Commit
6e7330a5
authored
Sep 28, 2020
by
Łukasz Nowak
Browse files
Options
Browse Files
Download
Plain Diff
Fix/slapos master instantiation
See merge request
!814
parents
b6a12b43
d2d2a1b6
Changes
5
Hide whitespace changes
Inline
Side-by-side
Showing
5 changed files
with
203 additions
and
21 deletions
+203
-21
component/openssl/buildout.cfg
component/openssl/buildout.cfg
+1
-0
software/slapos-master/test/test/test_erp5.py
software/slapos-master/test/test/test_erp5.py
+198
-19
software/slapos-sr-testing/buildout.hash.cfg
software/slapos-sr-testing/buildout.hash.cfg
+1
-1
software/slapos-sr-testing/instance.cfg
software/slapos-sr-testing/instance.cfg
+1
-1
software/slapos-sr-testing/software.cfg
software/slapos-sr-testing/software.cfg
+2
-0
No files found.
component/openssl/buildout.cfg
View file @
6e7330a5
...
@@ -53,6 +53,7 @@ stop-on-error = true
...
@@ -53,6 +53,7 @@ stop-on-error = true
update-command = ${:command}
update-command = ${:command}
command = ${coreutils-output:test} -x ${:openssl}
command = ${coreutils-output:test} -x ${:openssl}
openssl = ${openssl:location}/bin/openssl
openssl = ${openssl:location}/bin/openssl
bin = ${openssl:location}
[openssl-1.0]
[openssl-1.0]
recipe = slapos.recipe.cmmi
recipe = slapos.recipe.cmmi
...
...
software/slapos-master/test/test/test_erp5.py
View file @
6e7330a5
...
@@ -31,13 +31,17 @@ import glob
...
@@ -31,13 +31,17 @@ import glob
import
urlparse
import
urlparse
import
socket
import
socket
import
time
import
time
import
re
import
BaseHTTPServer
import
multiprocessing
import
subprocess
import
psutil
import
psutil
import
requests
import
requests
from
.
import
ERP5InstanceTestCase
from
.
import
ERP5InstanceTestCase
from
.
import
setUpModule
from
.
import
setUpModule
setUpModule
# pyflakes
setUpModule
# pyflakes
class
TestPublishedURLIsReachableMixin
(
object
):
class
TestPublishedURLIsReachableMixin
(
object
):
...
@@ -46,15 +50,17 @@ class TestPublishedURLIsReachableMixin(object):
...
@@ -46,15 +50,17 @@ class TestPublishedURLIsReachableMixin(object):
def
_checkERP5IsReachable
(
self
,
url
):
def
_checkERP5IsReachable
(
self
,
url
):
# What happens is that instanciation just create the services, but does not
# What happens is that instanciation just create the services, but does not
# wait for ERP5 to be initialized. When this test run ERP5 instance is
# wait for ERP5 to be initialized. When this test run ERP5 instance is
# instanciated, but zope is still busy creating the site and haproxy
replies
# instanciated, but zope is still busy creating the site and haproxy
#
with 503 Service Unavailable when zope is not started yet, with 404 when
#
replies with 503 Service Unavailable when zope is not started yet, with
#
erp5 site is not created, with 500 when mysql is not yet reachable, so we
#
404 when erp5 site is not created, with 500 when mysql is not yet
# retry in a loop until we get a succesful response.
# re
achable, so we re
try in a loop until we get a succesful response.
for
i
in
range
(
1
,
60
):
for
i
in
range
(
1
,
60
):
r
=
requests
.
get
(
url
,
verify
=
False
)
# XXX can we get CA from caucase already ?
# XXX can we get CA from caucase already ?
r
=
requests
.
get
(
url
,
verify
=
False
)
if
r
.
status_code
!=
requests
.
codes
.
ok
:
if
r
.
status_code
!=
requests
.
codes
.
ok
:
delay
=
i
*
2
delay
=
i
*
2
self
.
logger
.
warn
(
"ERP5 was not available, sleeping for %ds and retrying"
,
delay
)
self
.
logger
.
warn
(
"ERP5 was not available, sleeping for %ds and retrying"
,
delay
)
time
.
sleep
(
delay
)
time
.
sleep
(
delay
)
continue
continue
r
.
raise_for_status
()
r
.
raise_for_status
()
...
@@ -77,7 +83,8 @@ class TestPublishedURLIsReachableMixin(object):
...
@@ -77,7 +83,8 @@ class TestPublishedURLIsReachableMixin(object):
urlparse
.
urljoin
(
param_dict
[
'family-default'
],
param_dict
[
'site-id'
]))
urlparse
.
urljoin
(
param_dict
[
'family-default'
],
param_dict
[
'site-id'
]))
class
TestDefaultParameters
(
ERP5InstanceTestCase
,
TestPublishedURLIsReachableMixin
):
class
TestDefaultParameters
(
ERP5InstanceTestCase
,
TestPublishedURLIsReachableMixin
):
"""Test ERP5 can be instanciated with no parameters
"""Test ERP5 can be instanciated with no parameters
"""
"""
__partition_reference__
=
'defp'
__partition_reference__
=
'defp'
...
@@ -125,16 +132,19 @@ class TestApacheBalancerPorts(ERP5InstanceTestCase):
...
@@ -125,16 +132,19 @@ class TestApacheBalancerPorts(ERP5InstanceTestCase):
self
.
assertTrue
(
parsed
.
port
)
self
.
assertTrue
(
parsed
.
port
)
def
test_published_family_parameters
(
self
):
def
test_published_family_parameters
(
self
):
# when we request two families, we have two published family-{family_name} URLs
# when we request two families, we have two published family-{family_name}
# URLs
param_dict
=
self
.
getRootPartitionConnectionParameterDict
()
param_dict
=
self
.
getRootPartitionConnectionParameterDict
()
for
family_name
in
(
'family1'
,
'family2'
):
for
family_name
in
(
'family1'
,
'family2'
):
self
.
checkValidHTTPSURL
(
self
.
checkValidHTTPSURL
(
param_dict
[
'family-{family_name}'
.
format
(
family_name
=
family_name
)])
param_dict
[
'family-{family_name}'
.
format
(
family_name
=
family_name
)])
self
.
checkValidHTTPSURL
(
self
.
checkValidHTTPSURL
(
param_dict
[
'family-{family_name}-v6'
.
format
(
family_name
=
family_name
)])
param_dict
[
'family-{family_name}-v6'
.
format
(
family_name
=
family_name
)])
def
test_published_test_runner_url
(
self
):
def
test_published_test_runner_url
(
self
):
# each family's also a list of test test runner URLs, by default 3 per family
# each family's also a list of test test runner URLs, by default 3 per
# family
param_dict
=
self
.
getRootPartitionConnectionParameterDict
()
param_dict
=
self
.
getRootPartitionConnectionParameterDict
()
for
family_name
in
(
'family1'
,
'family2'
):
for
family_name
in
(
'family1'
,
'family2'
):
family_test_runner_url_list
=
param_dict
[
family_test_runner_url_list
=
param_dict
[
...
@@ -144,7 +154,8 @@ class TestApacheBalancerPorts(ERP5InstanceTestCase):
...
@@ -144,7 +154,8 @@ class TestApacheBalancerPorts(ERP5InstanceTestCase):
self
.
checkValidHTTPSURL
(
url
)
self
.
checkValidHTTPSURL
(
url
)
def
test_zope_listen
(
self
):
def
test_zope_listen
(
self
):
# we requested 3 zope in family1 and 5 zopes in family2, we should have 8 zope running.
# we requested 3 zope in family1 and 5 zopes in family2, we should have 8
# zope running.
with
self
.
slap
.
instance_supervisor_rpc
as
supervisor
:
with
self
.
slap
.
instance_supervisor_rpc
as
supervisor
:
all_process_info
=
supervisor
.
getAllProcessInfo
()
all_process_info
=
supervisor
.
getAllProcessInfo
()
self
.
assertEqual
(
self
.
assertEqual
(
...
@@ -179,7 +190,8 @@ class TestApacheBalancerPorts(ERP5InstanceTestCase):
...
@@ -179,7 +190,8 @@ class TestApacheBalancerPorts(ERP5InstanceTestCase):
])
])
class
TestDisableTestRunner
(
ERP5InstanceTestCase
,
TestPublishedURLIsReachableMixin
):
class
TestDisableTestRunner
(
ERP5InstanceTestCase
,
TestPublishedURLIsReachableMixin
):
"""Test ERP5 can be instanciated without test runner.
"""Test ERP5 can be instanciated without test runner.
"""
"""
__partition_reference__
=
'distr'
__partition_reference__
=
'distr'
...
@@ -192,15 +204,17 @@ class TestDisableTestRunner(ERP5InstanceTestCase, TestPublishedURLIsReachableMix
...
@@ -192,15 +204,17 @@ class TestDisableTestRunner(ERP5InstanceTestCase, TestPublishedURLIsReachableMix
"""
"""
# self.computer_partition_root_path is the path of root partition.
# self.computer_partition_root_path is the path of root partition.
# we want to assert that no scripts exist in any partition.
# we want to assert that no scripts exist in any partition.
bin_programs
=
map
(
os
.
path
.
basename
,
bin_programs
=
map
(
os
.
path
.
basename
,
glob
.
glob
(
self
.
computer_partition_root_path
+
"/../*/bin/*"
))
glob
.
glob
(
self
.
computer_partition_root_path
+
"/../*/bin/*"
))
self
.
assertTrue
(
bin_programs
)
# just to check the glob was correct.
self
.
assertTrue
(
bin_programs
)
# just to check the glob was correct.
self
.
assertNotIn
(
'runUnitTest'
,
bin_programs
)
self
.
assertNotIn
(
'runUnitTest'
,
bin_programs
)
self
.
assertNotIn
(
'runTestSuite'
,
bin_programs
)
self
.
assertNotIn
(
'runTestSuite'
,
bin_programs
)
def
test_no_apache_testrunner_port
(
self
):
def
test_no_apache_testrunner_port
(
self
):
# Apache only listen on two ports, there is no apache ports allocated for test runner
# Apache only listen on two ports, there is no apache ports allocated for
# test runner
with
self
.
slap
.
instance_supervisor_rpc
as
supervisor
:
with
self
.
slap
.
instance_supervisor_rpc
as
supervisor
:
all_process_info
=
supervisor
.
getAllProcessInfo
()
all_process_info
=
supervisor
.
getAllProcessInfo
()
process_info
,
=
[
p
for
p
in
all_process_info
if
p
[
'name'
]
==
'apache'
]
process_info
,
=
[
p
for
p
in
all_process_info
if
p
[
'name'
]
==
'apache'
]
...
@@ -213,7 +227,9 @@ class TestDisableTestRunner(ERP5InstanceTestCase, TestPublishedURLIsReachableMix
...
@@ -213,7 +227,9 @@ class TestDisableTestRunner(ERP5InstanceTestCase, TestPublishedURLIsReachableMix
if
c
.
status
==
'LISTEN'
if
c
.
status
==
'LISTEN'
))
))
class
TestZopeNodeParameterOverride
(
ERP5InstanceTestCase
,
TestPublishedURLIsReachableMixin
):
class
TestZopeNodeParameterOverride
(
ERP5InstanceTestCase
,
TestPublishedURLIsReachableMixin
):
"""Test override zope node parameters
"""Test override zope node parameters
"""
"""
__partition_reference__
=
'override'
__partition_reference__
=
'override'
...
@@ -228,7 +244,7 @@ class TestZopeNodeParameterOverride(ERP5InstanceTestCase, TestPublishedURLIsReac
...
@@ -228,7 +244,7 @@ class TestZopeNodeParameterOverride(ERP5InstanceTestCase, TestPublishedURLIsReac
"server"
:
{},
"server"
:
{},
"cache-size-bytes"
:
"20MB"
,
"cache-size-bytes"
:
"20MB"
,
"cache-size-bytes!"
:
[
"cache-size-bytes!"
:
[
(
"bb-0"
,
1
<<
20
),
(
"bb-0"
,
1
<<
20
),
(
"bb-.*"
,
"500MB"
),
(
"bb-.*"
,
"500MB"
),
],
],
"pool-timeout"
:
"10m"
,
"pool-timeout"
:
"10m"
,
...
@@ -299,7 +315,170 @@ class TestZopeNodeParameterOverride(ERP5InstanceTestCase, TestPublishedURLIsReac
...
@@ -299,7 +315,170 @@ class TestZopeNodeParameterOverride(ERP5InstanceTestCase, TestPublishedURLIsReac
partition
=
self
.
getComputerPartitionPath
(
'zope-bb'
)
partition
=
self
.
getComputerPartitionPath
(
'zope-bb'
)
for
zope
in
xrange
(
5
):
for
zope
in
xrange
(
5
):
checkConf
({
checkConf
({
"cache-size-bytes"
:
"500MB"
if
zope
else
1
<<
20
,
"cache-size-bytes"
:
"500MB"
if
zope
else
1
<<
20
,
},
{
},
{
"cache-size"
:
None
,
"cache-size"
:
None
,
})
})
def
popenCommunicate
(
command_list
,
input_
=
None
,
**
kwargs
):
kwargs
.
update
(
stdout
=
subprocess
.
PIPE
,
stderr
=
subprocess
.
STDOUT
)
popen
=
subprocess
.
Popen
(
command_list
,
**
kwargs
)
result
=
popen
.
communicate
(
input_
)[
0
]
if
popen
.
returncode
is
None
:
popen
.
kill
()
if
popen
.
returncode
!=
0
:
raise
ValueError
(
'Issue during calling %r, result was:
\
n
%s'
%
(
command_list
,
result
))
return
result
class
TestHandler
(
BaseHTTPServer
.
BaseHTTPRequestHandler
):
def
do_GET
(
self
):
self
.
send_response
(
200
)
response
=
{
'Path'
:
self
.
path
,
'Incoming Headers'
:
self
.
headers
.
dict
}
response
=
json
.
dumps
(
response
,
indent
=
2
)
self
.
end_headers
()
self
.
wfile
.
write
(
response
)
class
TestDeploymentScriptInstantiation
(
ERP5InstanceTestCase
):
"""This check deployment script like instantiation
Low level assertions are done here in roder to assure that
https://lab.nexedi.com/nexedi/slapos.package/blob/master/playbook/
slapos-master-standalone.yml
works correctly
"""
__partition_reference__
=
'tdsi'
# a bit more partition is required
partition_count
=
20
@
classmethod
def
getInstanceParameterDict
(
cls
):
# As close as possible configuration to deployment script
parameter_dict
=
{
"timezone"
:
"UTC"
,
"site-id"
:
"erp5"
,
"bt5"
:
"erp5_full_text_myisam_catalog slapos_configurator"
,
"wsgi"
:
False
,
"test-runner"
:
{
"enabled"
:
False
},
# won't work anyway here
"zope-partition-dict"
:
{
"admin"
:
{
"family"
:
"admin"
,
"thread-amount"
:
4
,
"port-base"
:
2220
,
"instance-count"
:
1
},
"activities-node"
:
{
"family"
:
"activities"
,
"thread-amount"
:
4
,
"instance-count"
:
1
,
"timerserver-interval"
:
1
,
"port-base"
:
2230
},
"distribution-node"
:
{
"family"
:
"distribution"
,
"thread-amount"
:
1
,
"instance-count"
:
1
,
"port-base"
:
2210
,
"timerserver-interval"
:
1
},
"web-node"
:
{
"family"
:
"web"
,
"thread-amount"
:
2
,
"instance-count"
:
1
,
"port-base"
:
2240
},
"service-slapos"
:
{
"family"
:
"service"
,
"thread-amount"
:
2
,
"instance-count"
:
1
,
"port-base"
:
2250
,
"ssl-authentication"
:
True
,
"backend-path"
:
"/%(site-id)s/portal_slap"
}
}
}
# put shared-certificate-authority-path in controlled location
cls
.
ca_path
=
os
.
path
.
join
(
cls
.
slap
.
instance_directory
,
'ca_path'
)
parameter_dict
[
"shared-certificate-authority-path"
]
=
cls
.
ca_path
return
{
'_'
:
json
.
dumps
(
parameter_dict
)}
@
classmethod
def
callSupervisorMethod
(
cls
,
method
,
*
args
,
**
kwargs
):
with
cls
.
slap
.
instance_supervisor_rpc
as
instance_supervisor
:
return
getattr
(
instance_supervisor
,
method
)(
*
args
,
**
kwargs
)
def
test_ssl_auth
(
self
):
backend_apache_configuration_list
=
glob
.
glob
(
os
.
path
.
join
(
self
.
slap
.
instance_directory
,
'*'
,
'etc'
,
'apache'
,
'apache.conf'
))
self
.
assertEqual
(
1
,
len
(
backend_apache_configuration_list
)
)
backend_apache_configuration
=
open
(
backend_apache_configuration_list
[
0
]).
read
()
self
.
assertIn
(
'SSLVerifyClient require'
,
backend_apache_configuration
)
self
.
assertIn
(
r'RequestHeader set Remote-User %{SSL_CLIENT_S_DN_CN}s'
,
backend_apache_configuration
)
# stop haproxy, it's going to be hijacked
haproxy_name
=
':'
.
join
([
(
q
[
'group'
],
q
[
'name'
])
for
q
in
self
.
callSupervisorMethod
(
'getAllProcessInfo'
)
if
'haproxy'
in
q
[
'name'
]][
0
])
self
.
callSupervisorMethod
(
'stopProcess'
,
haproxy_name
)
# do similar certificate request like CertificateAuthorityTool
openssl_config
=
os
.
path
.
join
(
self
.
ca_path
,
'openssl.cnf'
)
key
=
os
.
path
.
join
(
self
.
ca_path
,
'private'
,
'test.key'
)
csr
=
os
.
path
.
join
(
self
.
ca_path
,
'text.csr'
)
cert
=
os
.
path
.
join
(
self
.
ca_path
,
'certs'
,
'test.crt'
)
common_name
=
'TEST-SSL-AUTH'
popenCommunicate
([
'openssl'
,
'req'
,
'-utf8'
,
'-nodes'
,
'-config'
,
openssl_config
,
'-new'
,
'-keyout'
,
key
,
'-out'
,
csr
,
'-days'
,
'3650'
],
'%s
\
n
'
%
(
common_name
,),
stdin
=
subprocess
.
PIPE
)
popenCommunicate
([
'openssl'
,
'ca'
,
'-utf8'
,
'-days'
,
'3650'
,
'-batch'
,
'-config'
,
openssl_config
,
'-out'
,
cert
,
'-infiles'
,
csr
])
# find IP and port on which hijacked process shall listen
portal_slap_line
=
[
q
for
q
in
backend_apache_configuration
.
splitlines
()
if
'portal_slap'
in
q
][
0
]
ip
,
port
=
re
.
search
(
r'.*http:\
/
\/(.*):(\
d*)
\/.*'
,
portal_slap_line
).
groups
()
port
=
int
(
port
)
server
=
BaseHTTPServer
.
HTTPServer
((
ip
,
port
),
TestHandler
)
server_process
=
multiprocessing
.
Process
(
target
=
server
.
serve_forever
,
name
=
'HTTPServer'
)
server_process
.
start
()
try
:
# assert that accessing the service endpoint results with certificate
# authentication and proper information extraction
result_json
=
requests
.
get
(
self
.
getRootPartitionConnectionParameterDict
()[
'family-service'
],
verify
=
False
,
cert
=
(
cert
,
key
)).
json
()
self
.
assertEqual
(
common_name
,
result_json
[
'Incoming Headers'
][
'remote-user'
]
)
self
.
assertEqual
(
'/erp5/portal_slap/'
,
result_json
[
'Path'
]
)
finally
:
server_process
.
join
(
10
)
server_process
.
terminate
()
software/slapos-sr-testing/buildout.hash.cfg
View file @
6e7330a5
...
@@ -15,4 +15,4 @@
...
@@ -15,4 +15,4 @@
[template]
[template]
filename = instance.cfg
filename = instance.cfg
md5sum =
14d2f49d20670e44c2a162bcec9e0a8e
md5sum =
4246cde0a27138e057ba1635cc621edf
software/slapos-sr-testing/instance.cfg
View file @
6e7330a5
...
@@ -44,7 +44,7 @@ environment =
...
@@ -44,7 +44,7 @@ environment =
SLAPOS_TEST_WORKING_DIR=${slapos-test-runner-environment:SLAPOS_TEST_WORKING_DIR}
SLAPOS_TEST_WORKING_DIR=${slapos-test-runner-environment:SLAPOS_TEST_WORKING_DIR}
[slapos-test-runner-environment]
[slapos-test-runner-environment]
PATH = {{ buildout['bin-directory'] }}:{{ curl_location }}/bin/:{{ faketime_location }}/bin/:/usr/bin/:/bin
PATH = {{ buildout['bin-directory'] }}:{{ curl_location }}/bin/:{{ faketime_location }}/bin/:
{{ openssl_location }}/bin/:
/usr/bin/:/bin
SLAPOS_TEST_IPV4 = ${slap-configuration:ipv4-random}
SLAPOS_TEST_IPV4 = ${slap-configuration:ipv4-random}
SLAPOS_TEST_IPV6 = ${slap-configuration:ipv6-random}
SLAPOS_TEST_IPV6 = ${slap-configuration:ipv6-random}
SLAPOS_TEST_WORKING_DIR = ${directory:working-dir}
SLAPOS_TEST_WORKING_DIR = ${directory:working-dir}
...
...
software/slapos-sr-testing/software.cfg
View file @
6e7330a5
...
@@ -3,6 +3,7 @@
...
@@ -3,6 +3,7 @@
extends =
extends =
../../component/bcrypt/buildout.cfg
../../component/bcrypt/buildout.cfg
../../component/curl/buildout.cfg
../../component/curl/buildout.cfg
../../component/openssl/buildout.cfg
../../component/git/buildout.cfg
../../component/git/buildout.cfg
../../component/faketime/buildout.cfg
../../component/faketime/buildout.cfg
../../component/pillow/buildout.cfg
../../component/pillow/buildout.cfg
...
@@ -240,6 +241,7 @@ context =
...
@@ -240,6 +241,7 @@ context =
key slapos_location slapos-repository:location
key slapos_location slapos-repository:location
key interpreter eggs:interpreter
key interpreter eggs:interpreter
key curl_location curl:location
key curl_location curl:location
key openssl_location openssl-output:bin
key faketime_location faketime:location
key faketime_location faketime:location
key tests :tests
key tests :tests
tests =
tests =
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment