Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
erp5
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
Alexander Emmerich
erp5
Commits
a01da084
Commit
a01da084
authored
Nov 15, 2017
by
Julien Muchembled
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
testnode: some code clean up
A lot was found with pylint.
parent
8e40fb59
Changes
9
Expand all
Show whitespace changes
Inline
Side-by-side
Showing
9 changed files
with
218 additions
and
336 deletions
+218
-336
erp5/tests/testERP5TestNode.py
erp5/tests/testERP5TestNode.py
+7
-7
erp5/util/testnode/NodeTestSuite.py
erp5/util/testnode/NodeTestSuite.py
+19
-30
erp5/util/testnode/ProcessManager.py
erp5/util/testnode/ProcessManager.py
+12
-9
erp5/util/testnode/SlapOSControler.py
erp5/util/testnode/SlapOSControler.py
+62
-102
erp5/util/testnode/SlapOSMasterCommunicator.py
erp5/util/testnode/SlapOSMasterCommunicator.py
+17
-33
erp5/util/testnode/UnitTestRunner.py
erp5/util/testnode/UnitTestRunner.py
+23
-35
erp5/util/testnode/Updater.py
erp5/util/testnode/Updater.py
+12
-16
erp5/util/testnode/Utils.py
erp5/util/testnode/Utils.py
+15
-25
erp5/util/testnode/testnode.py
erp5/util/testnode/testnode.py
+51
-79
No files found.
erp5/tests/testERP5TestNode.py
View file @
a01da084
...
@@ -237,7 +237,7 @@ class ERP5TestNode(TestCase):
...
@@ -237,7 +237,7 @@ class ERP5TestNode(TestCase):
node_test_suite
.
custom_profile_path
)
node_test_suite
.
custom_profile_path
)
profile
=
open
(
node_test_suite
.
custom_profile_path
,
'r'
)
profile
=
open
(
node_test_suite
.
custom_profile_path
,
'r'
)
if
my_test_type
==
'UnitTest'
:
if
my_test_type
==
'UnitTest'
:
expected_profile
=
"""
expected_profile
=
"""
\
[buildout]
[buildout]
extends = %(temp_dir)s/testnode/foo/rep0/software.cfg
extends = %(temp_dir)s/testnode/foo/rep0/software.cfg
...
@@ -258,7 +258,7 @@ shared = true
...
@@ -258,7 +258,7 @@ shared = true
else
:
else
:
revision1
=
"azerty"
revision1
=
"azerty"
revision2
=
"qwerty"
revision2
=
"qwerty"
expected_profile
=
"""
expected_profile
=
"""
\
[buildout]
[buildout]
extends = %(temp_dir)s/testnode/foo/rep0/software.cfg
extends = %(temp_dir)s/testnode/foo/rep0/software.cfg
...
@@ -802,17 +802,17 @@ shared = true
...
@@ -802,17 +802,17 @@ shared = true
test_node
.
node_test_suite_dict
test_node
.
node_test_suite_dict
rand_part_set
=
set
()
rand_part_set
=
set
()
self
.
assertEquals
(
2
,
len
(
test_node
.
node_test_suite_dict
))
self
.
assertEquals
(
2
,
len
(
test_node
.
node_test_suite_dict
))
assert
(
test_node
.
suite_log
is
not
None
)
self
.
assertIsNot
(
test_node
.
suite_log
,
None
)
assert
(
isinstance
(
test_node
.
suite_log
,
types
.
MethodType
))
self
.
assertTrue
(
isinstance
(
test_node
.
suite_log
,
types
.
MethodType
))
for
ref
,
suite
in
test_node
.
node_test_suite_dict
.
items
():
for
ref
,
suite
in
test_node
.
node_test_suite_dict
.
items
():
self
.
assertTrue
(
'var/log/testnode/%s'
%
suite
.
reference
in
\
self
.
assertTrue
(
'var/log/testnode/%s'
%
suite
.
reference
in
\
suite
.
suite_log_path
,
suite
.
suite_log_path
,
"Incorrect suite log path : %r"
%
suite
.
suite_log_path
)
"Incorrect suite log path : %r"
%
suite
.
suite_log_path
)
assert
(
suite
.
suite_log_path
.
endswith
(
'suite.log'
))
self
.
assertTrue
(
suite
.
suite_log_path
.
endswith
(
'suite.log'
))
m
=
re
.
match
(
'.*
\
-(.*)
\
/suite.log'
,
suite
.
suite_log_path
)
m
=
re
.
match
(
'.*
\
-(.*)
\
/suite.log'
,
suite
.
suite_log_path
)
rand_part
=
m
.
groups
()[
0
]
rand_part
=
m
.
groups
()[
0
]
assert
(
len
(
rand_part
)
==
32
)
self
.
assertEqual
(
len
(
rand_part
),
32
)
assert
(
rand_part
not
in
rand_part_set
)
self
.
assertNotIn
(
rand_part
,
rand_part_set
)
rand_part_set
.
add
(
rand_part
)
rand_part_set
.
add
(
rand_part
)
suite_log
=
open
(
suite
.
suite_log_path
,
'r'
)
suite_log
=
open
(
suite
.
suite_log_path
,
'r'
)
self
.
assertEquals
(
1
,
len
([
x
for
x
in
suite_log
.
readlines
()
\
self
.
assertEquals
(
1
,
len
([
x
for
x
in
suite_log
.
readlines
()
\
...
...
erp5/util/testnode/NodeTestSuite.py
View file @
a01da084
...
@@ -24,23 +24,13 @@
...
@@ -24,23 +24,13 @@
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
#
##############################################################################
##############################################################################
from
datetime
import
datetime
,
timedelta
import
errno
import
os
import
os
import
subprocess
import
sys
import
time
import
glob
import
glob
import
SlapOSControler
import
json
import
time
import
shutil
import
shutil
import
logging
import
string
import
string
import
random
import
random
from
ProcessManager
import
SubprocessError
,
ProcessManager
,
CancellationError
from
.Utils
import
createFolder
from
subprocess
import
CalledProcessError
from
Updater
import
Updater
from
erp5.util
import
taskdistribution
class
SlapOSInstance
(
object
):
class
SlapOSInstance
(
object
):
"""
"""
...
@@ -67,22 +57,19 @@ class NodeTestSuite(SlapOSInstance):
...
@@ -67,22 +57,19 @@ class NodeTestSuite(SlapOSInstance):
self
.
reference
=
reference
self
.
reference
=
reference
self
.
cluster_configuration
=
{}
self
.
cluster_configuration
=
{}
def
edit
(
self
,
**
kw
):
super
(
NodeTestSuite
,
self
).
edit
(
**
kw
)
def
_checkData
(
self
):
def
_checkData
(
self
):
if
getattr
(
self
,
"working_directory"
,
None
)
is
not
None
:
if
getattr
(
self
,
"working_directory"
,
None
)
is
not
None
:
if
not
(
self
.
working_directory
.
endswith
(
os
.
path
.
sep
+
self
.
reference
)):
if
not
(
self
.
working_directory
.
endswith
(
os
.
path
.
sep
+
self
.
reference
)):
self
.
working_directory
=
os
.
path
.
join
(
self
.
working_directory
,
self
.
working_directory
=
os
.
path
.
join
(
self
.
working_directory
,
self
.
reference
)
self
.
reference
)
SlapOSControler
.
createFolder
(
self
.
working_directory
)
createFolder
(
self
.
working_directory
)
self
.
test_suite_directory
=
os
.
path
.
join
(
self
.
test_suite_directory
=
os
.
path
.
join
(
self
.
working_directory
,
"test_suite"
)
self
.
working_directory
,
"test_suite"
)
self
.
custom_profile_path
=
os
.
path
.
join
(
self
.
working_directory
,
self
.
custom_profile_path
=
os
.
path
.
join
(
self
.
working_directory
,
'software.cfg'
)
'software.cfg'
)
if
getattr
(
self
,
"vcs_repository_list"
,
None
)
is
not
None
:
if
getattr
(
self
,
"vcs_repository_list"
,
None
)
is
not
None
:
for
vcs_repository
in
self
.
vcs_repository_list
:
for
vcs_repository
in
self
.
vcs_repository_list
:
buildout_section_id
=
vcs_repository
.
get
(
'buildout_section_id'
,
None
)
buildout_section_id
=
vcs_repository
.
get
(
'buildout_section_id'
)
repository_id
=
buildout_section_id
or
\
repository_id
=
buildout_section_id
or
\
vcs_repository
.
get
(
'url'
).
split
(
'/'
)[
-
1
].
split
(
'.'
)[
0
]
vcs_repository
.
get
(
'url'
).
split
(
'/'
)[
-
1
].
split
(
'.'
)[
0
]
repository_path
=
os
.
path
.
join
(
self
.
working_directory
,
repository_id
)
repository_path
=
os
.
path
.
join
(
self
.
working_directory
,
repository_id
)
...
@@ -92,20 +79,22 @@ class NodeTestSuite(SlapOSInstance):
...
@@ -92,20 +79,22 @@ class NodeTestSuite(SlapOSInstance):
def
createSuiteLog
(
self
):
def
createSuiteLog
(
self
):
# /srv/slapgrid/slappartXX/srv/var/log/testnode/az-mlksjfmlk234Sljssdflkj23KSdfslj/suite.log
# /srv/slapgrid/slappartXX/srv/var/log/testnode/az-mlksjfmlk234Sljssdflkj23KSdfslj/suite.log
alphabets
=
string
.
digits
+
string
.
letters
alphabets
=
string
.
digits
+
string
.
letters
rand_part
=
''
.
join
(
random
.
choice
(
alphabets
)
for
i
in
xrange
(
32
))
while
1
:
random_suite_folder_id
=
'%s-%s'
%
(
self
.
reference
,
rand_part
)
log_folder_name
=
'%s-%s'
%
(
self
.
reference
,
suite_log_directory
=
os
.
path
.
join
(
self
.
log_directory
,
''
.
join
(
random
.
choice
(
alphabets
)
for
i
in
xrange
(
32
)))
random_suite_folder_id
)
log_folder_path
=
os
.
path
.
join
(
self
.
log_directory
,
log_folder_name
)
SlapOSControler
.
createFolders
(
suite_log_directory
)
try
:
os
.
makedirs
(
log_folder_path
)
except
OSError
,
e
:
if
e
.
errno
!=
errno
.
EEXIST
:
raise
else
:
break
# XXX copy the whole content of the log viewer app
# XXX copy the whole content of the log viewer app
for
fname
in
glob
.
glob
(
os
.
path
.
join
(
os
.
path
.
dirname
(
__file__
),
'js-logtail'
,
'*'
)):
for
fname
in
glob
.
glob
(
os
.
path
.
join
(
os
.
path
.
dirname
(
__file__
),
'js-logtail'
,
'*'
)):
shutil
.
copy
(
fname
,
suite_log_directory
)
shutil
.
copy
(
fname
,
log_folder_path
)
self
.
suite_log_path
=
os
.
path
.
join
(
suite_log_directory
,
self
.
suite_log_path
=
os
.
path
.
join
(
log_folder_path
,
'suite.log'
)
'suite.log'
)
return
self
.
suite_log_path
,
log_folder_name
return
self
.
getSuiteLogPath
(),
random_suite_folder_id
def
getSuiteLogPath
(
self
):
return
getattr
(
self
,
"suite_log_path"
,
None
)
@
property
@
property
def
revision
(
self
):
def
revision
(
self
):
...
...
erp5/util/testnode/ProcessManager.py
View file @
a01da084
...
@@ -131,14 +131,14 @@ class ProcessManager(object):
...
@@ -131,14 +131,14 @@ class ProcessManager(object):
stdin
=
file
(
os
.
devnull
)
stdin
=
file
(
os
.
devnull
)
def
__init__
(
self
,
log
,
*
args
,
**
kw
):
def
__init__
(
self
,
log
,
max_timeout
=
MAX_TIMEOUT
):
self
.
log
=
log
self
.
log
=
log
self
.
process_pid_set
=
set
()
self
.
process_pid_set
=
set
()
signal
.
signal
(
signal
.
SIGTERM
,
self
.
sigterm_handler
)
signal
.
signal
(
signal
.
SIGTERM
,
self
.
sigterm_handler
)
self
.
under_cancellation
=
False
self
.
under_cancellation
=
False
self
.
p
=
None
self
.
p
=
None
self
.
result
=
None
self
.
result
=
None
self
.
max_timeout
=
kw
.
get
(
"max_timeout"
)
or
MAX_TIMEOUT
self
.
max_timeout
=
max_timeout
self
.
timer_set
=
set
()
self
.
timer_set
=
set
()
def
spawn
(
self
,
*
args
,
**
kw
):
def
spawn
(
self
,
*
args
,
**
kw
):
...
@@ -188,7 +188,7 @@ class ProcessManager(object):
...
@@ -188,7 +188,7 @@ class ProcessManager(object):
return
result
return
result
def
getSupportedParameterList
(
self
,
program_path
):
def
getSupportedParameterList
(
self
,
program_path
):
return
re
.
findall
(
'^ (--
\
w+)
'
,
return
re
.
findall
(
r
'^ (--\
w+)
',
self.spawn(program_path, '
--
help
')['
stdout
'], re.M)
self.spawn(program_path, '
--
help
')['
stdout
'], re.M)
def killall(self, name):
def killall(self, name):
...
@@ -212,13 +212,15 @@ class ProcessManager(object):
...
@@ -212,13 +212,15 @@ class ProcessManager(object):
continue
continue
except (psutil.AccessDenied, psutil.NoSuchProcess):
except (psutil.AccessDenied, psutil.NoSuchProcess):
continue
continue
self.log('ProcesssManager, killall on %s having pid %s' % (name, process.pid))
self.log('ProcesssManager, killall on %s having pid %s',
name, process.pid)
to_kill_list.append(process.pid)
to_kill_list.append(process.pid)
for pid in to_kill_list:
for pid in to_kill_list:
killCommand(pid, self.log)
killCommand(pid, self.log)
def killPreviousRun(self, cancellation=False):
def killPreviousRun(self, cancellation=False):
self.log('ProcessManager killPreviousRun, going to kill %r' % (self.process_pid_set,))
self.log('ProcessManager killPreviousRun, going to kill %r',
self.process_pid_set)
if cancellation:
if cancellation:
self.under_cancellation = True
self.under_cancellation = True
for timer in self.timer_set:
for timer in self.timer_set:
...
@@ -227,12 +229,13 @@ class ProcessManager(object):
...
@@ -227,12 +229,13 @@ class ProcessManager(object):
killCommand(pgpid, self.log)
killCommand(pgpid, self.log)
try:
try:
if os.path.exists(self.supervisord_pid_file):
if os.path.exists(self.supervisord_pid_file):
supervisor_pid = int(open(self.supervisord_pid_file).read().strip())
with open(self.supervisord_pid_file) as f:
self.log('ProcessManager killPreviousRun, going to kill supervisor with pid %r' % supervisor_pid)
supervisor_pid = int(f.read().strip())
self.log('ProcessManager killPreviousRun, going to kill supervisor with pid %r',
supervisor_pid)
os.kill(supervisor_pid, signal.SIGTERM)
os.kill(supervisor_pid, signal.SIGTERM)
except:
except
Exception
:
self.log('ProcessManager killPreviousRun, exception when killing supervisor')
self.log('ProcessManager killPreviousRun, exception when killing supervisor')
pass
self.process_pid_set.clear()
self.process_pid_set.clear()
def sigterm_handler(self, signal, frame):
def sigterm_handler(self, signal, frame):
...
...
erp5/util/testnode/SlapOSControler.py
View file @
a01da084
This diff is collapsed.
Click to expand it.
erp5/util/testnode/SlapOSMasterCommunicator.py
View file @
a01da084
import
datetime
import
datetime
import
json
import
json
import
sys
import
traceback
import
traceback
import
time
import
time
#import feedparser
#import feedparser
from
functools
import
wraps
from
uritemplate
import
expand
from
uritemplate
import
expand
import
slapos.slap
import
slapos.slap
from
slapos.slap
import
SoftwareProductCollection
from
slapos.slap
import
SoftwareProductCollection
from
slapos.slap.slap
import
ConnectionError
from
requests.exceptions
import
HTTPError
from
requests.exceptions
import
HTTPError
from
erp5.util
.taskdistribution
import
SAFE_RPC_EXCEPTION_LIST
from
.
.taskdistribution
import
SAFE_RPC_EXCEPTION_LIST
# max time to instance changing state: 2 hour
# max time to instance changing state: 2 hour
MAX_INSTANCE_TIME
=
60
*
60
*
2
MAX_INSTANCE_TIME
=
60
*
60
*
2
...
@@ -41,36 +39,29 @@ TESTER_STATE_INSTANCE_UNINSTALLED = "TESTER_STATE_INSTANCE_UNINSTALLED"
...
@@ -41,36 +39,29 @@ TESTER_STATE_INSTANCE_UNINSTALLED = "TESTER_STATE_INSTANCE_UNINSTALLED"
# Simple decorator to prevent raise due small
# Simple decorator to prevent raise due small
# network failures.
# network failures.
def
retryOnNetworkFailure
(
func
):
def
retryOnNetworkFailure
(
func
,
def
wrapper
(
*
args
,
**
kwargs
):
_except_list
=
SAFE_RPC_EXCEPTION_LIST
+
(
HTTPError
,
slapos
.
slap
.
ConnectionError
),
):
def
wrapper
(
*
args
,
**
kw
):
retry_time
=
64
retry_time
=
64
while
True
:
while
True
:
try
:
try
:
return
func
(
*
args
,
**
kwargs
)
return
func
(
*
args
,
**
kw
)
except
SAFE_RPC_EXCEPTION_LIST
,
e
:
except
_except_list
:
print
'Network failure: %s , %s'
%
(
sys
.
exc_info
(),
e
)
traceback
.
print_exc
()
except
HTTPError
,
e
:
print
'Network failure: %s , %s'
%
(
sys
.
exc_info
(),
e
)
print
'Network failure. Retry method %s in %i seconds'
%
(
func
,
retry_time
)
except
ConnectionError
,
e
:
print
'Network failure: %s , %s'
%
(
sys
.
exc_info
(),
e
)
except
slapos
.
slap
.
ConnectionError
,
e
:
print
'Network failure: %s , %s'
%
(
sys
.
exc_info
(),
e
)
print
'Retry method %s in %i seconds'
%
(
func
,
retry_time
)
time
.
sleep
(
retry_time
)
time
.
sleep
(
retry_time
)
retry_time
=
min
(
retry_time
*
1.5
,
640
)
retry_time
=
min
(
retry_time
*
1.5
,
640
)
return
wraps
(
func
)(
wrapper
)
wrapper
.
__name__
=
func
.
__name__
wrapper
.
__doc__
=
func
.
__doc__
return
wrapper
class
SlapOSMasterCommunicator
(
object
):
class
SlapOSMasterCommunicator
(
object
):
latest_state
=
None
latest_state
=
None
def
__init__
(
self
,
slap
,
slap_supply
,
slap_order
,
url
,
logger
):
def
__init__
(
self
,
slap
,
slap_supply
,
slap_order
,
url
,
logger
):
self
.
_logger
=
logger
self
.
_logger
=
logger
self
.
slap
=
slap
self
.
slap
=
slap
self
.
slap_order
=
slap_order
self
.
slap_order
=
slap_order
...
@@ -102,8 +93,7 @@ class SlapOSMasterCommunicator(object):
...
@@ -102,8 +93,7 @@ class SlapOSMasterCommunicator(object):
if
instance_title
is
not
None
:
if
instance_title
is
not
None
:
self
.
name
=
instance_title
self
.
name
=
instance_title
if
request_kw
is
not
None
:
if
request_kw
is
not
None
:
if
isinstance
(
request_kw
,
str
)
or
\
if
isinstance
(
request_kw
,
basestring
):
isinstance
(
request_kw
,
unicode
):
self
.
request_kw
=
json
.
loads
(
request_kw
)
self
.
request_kw
=
json
.
loads
(
request_kw
)
else
:
else
:
self
.
request_kw
=
request_kw
self
.
request_kw
=
request_kw
...
@@ -116,12 +106,11 @@ class SlapOSMasterCommunicator(object):
...
@@ -116,12 +106,11 @@ class SlapOSMasterCommunicator(object):
**
self
.
request_kw
)
**
self
.
request_kw
)
def
isInstanceRequested
(
self
,
instance_title
):
def
isInstanceRequested
(
self
,
instance_title
):
hateoas
=
getattr
(
self
.
slap
,
'_hateoas_navigator'
,
None
)
hateoas
=
self
.
_hateoas_navigator
return
instance_title
in
hateoas
.
getHostingSubscriptionDict
()
return
instance_title
in
hateoas
.
getHostingSubscriptionDict
()
@
retryOnNetworkFailure
@
retryOnNetworkFailure
def
_hateoas_getComputer
(
self
,
reference
):
def
_hateoas_getComputer
(
self
,
reference
):
root_document
=
self
.
hateoas_navigator
.
getRootDocument
()
root_document
=
self
.
hateoas_navigator
.
getRootDocument
()
search_url
=
root_document
[
"_links"
][
'raw_search'
][
'href'
]
search_url
=
root_document
[
"_links"
][
'raw_search'
][
'href'
]
...
@@ -147,7 +136,6 @@ class SlapOSMasterCommunicator(object):
...
@@ -147,7 +136,6 @@ class SlapOSMasterCommunicator(object):
@
retryOnNetworkFailure
@
retryOnNetworkFailure
def
getSoftwareInstallationList
(
self
):
def
getSoftwareInstallationList
(
self
):
# XXX Move me to slap.py API
# XXX Move me to slap.py API
computer
=
self
.
_hateoas_getComputer
(
self
.
computer_guid
)
computer
=
self
.
_hateoas_getComputer
(
self
.
computer_guid
)
# Not a list ?
# Not a list ?
...
@@ -191,7 +179,6 @@ class SlapOSMasterCommunicator(object):
...
@@ -191,7 +179,6 @@ class SlapOSMasterCommunicator(object):
@
retryOnNetworkFailure
@
retryOnNetworkFailure
def
getInstanceUrlList
(
self
):
def
getInstanceUrlList
(
self
):
if
self
.
hosting_subscription_url
is
None
:
if
self
.
hosting_subscription_url
is
None
:
hosting_subscription_dict
=
self
.
hateoas_navigator
.
_hateoas_getHostingSubscriptionDict
()
hosting_subscription_dict
=
self
.
hateoas_navigator
.
_hateoas_getHostingSubscriptionDict
()
for
hs
in
hosting_subscription_dict
:
for
hs
in
hosting_subscription_dict
:
...
@@ -207,7 +194,6 @@ class SlapOSMasterCommunicator(object):
...
@@ -207,7 +194,6 @@ class SlapOSMasterCommunicator(object):
@
retryOnNetworkFailure
@
retryOnNetworkFailure
def
getNewsFromInstance
(
self
,
url
):
def
getNewsFromInstance
(
self
,
url
):
result
=
self
.
hateoas_navigator
.
GET
(
url
)
result
=
self
.
hateoas_navigator
.
GET
(
url
)
result
=
json
.
loads
(
result
)
result
=
json
.
loads
(
result
)
if
result
[
'_links'
].
get
(
'action_object_slap'
,
None
)
is
None
:
if
result
[
'_links'
].
get
(
'action_object_slap'
,
None
)
is
None
:
...
@@ -221,7 +207,6 @@ class SlapOSMasterCommunicator(object):
...
@@ -221,7 +207,6 @@ class SlapOSMasterCommunicator(object):
@
retryOnNetworkFailure
@
retryOnNetworkFailure
def
getInformationFromInstance
(
self
,
url
):
def
getInformationFromInstance
(
self
,
url
):
result
=
self
.
hateoas_navigator
.
GET
(
url
)
result
=
self
.
hateoas_navigator
.
GET
(
url
)
result
=
json
.
loads
(
result
)
result
=
json
.
loads
(
result
)
if
result
[
'_links'
].
get
(
'action_object_slap'
,
None
)
is
None
:
if
result
[
'_links'
].
get
(
'action_object_slap'
,
None
)
is
None
:
...
@@ -329,7 +314,7 @@ class SlapOSMasterCommunicator(object):
...
@@ -329,7 +314,7 @@ class SlapOSMasterCommunicator(object):
self
.
_logger
(
'Got an error requesting partition for '
self
.
_logger
(
'Got an error requesting partition for '
'its state'
)
'its state'
)
return
INSTANCE_STATE_UNKNOWN
return
INSTANCE_STATE_UNKNOWN
except
:
except
Exception
:
self
.
_logger
(
"ERROR getting instance state"
)
self
.
_logger
(
"ERROR getting instance state"
)
return
INSTANCE_STATE_UNKNOWN
return
INSTANCE_STATE_UNKNOWN
...
@@ -377,6 +362,7 @@ class SlapOSMasterCommunicator(object):
...
@@ -377,6 +362,7 @@ class SlapOSMasterCommunicator(object):
return
{
'error_message'
:
None
}
return
{
'error_message'
:
None
}
class
SlapOSTester
(
SlapOSMasterCommunicator
):
class
SlapOSTester
(
SlapOSMasterCommunicator
):
def
__init__
(
self
,
def
__init__
(
self
,
name
,
name
,
logger
,
logger
,
...
@@ -387,7 +373,6 @@ class SlapOSTester(SlapOSMasterCommunicator):
...
@@ -387,7 +373,6 @@ class SlapOSTester(SlapOSMasterCommunicator):
computer_guid
=
None
,
# computer for supply if desired
computer_guid
=
None
,
# computer for supply if desired
request_kw
=
None
request_kw
=
None
):
):
super
(
SlapOSTester
,
self
).
__init__
(
super
(
SlapOSTester
,
self
).
__init__
(
slap
,
slap_supply
,
slap_order
,
url
,
logger
)
slap
,
slap_supply
,
slap_order
,
url
,
logger
)
...
@@ -464,7 +449,6 @@ class SoftwareReleaseTester(SlapOSTester):
...
@@ -464,7 +449,6 @@ class SoftwareReleaseTester(SlapOSTester):
software_timeout
=
3600
,
software_timeout
=
3600
,
instance_timeout
=
3600
,
instance_timeout
=
3600
,
):
):
super
(
SoftwareReleaseTester
,
self
).
__init__
(
super
(
SoftwareReleaseTester
,
self
).
__init__
(
name
,
logger
,
slap
,
slap_order
,
slap_supply
,
url
,
computer_guid
,
request_kw
)
name
,
logger
,
slap
,
slap_order
,
slap_supply
,
url
,
computer_guid
,
request_kw
)
...
...
erp5/util/testnode/UnitTestRunner.py
View file @
a01da084
...
@@ -24,28 +24,22 @@
...
@@ -24,28 +24,22 @@
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
#
##############################################################################
##############################################################################
from
datetime
import
datetime
,
timedelta
import
os
import
os
import
subprocess
import
sys
import
time
import
glob
import
glob
import
SlapOSControler
import
json
import
json
import
time
from
.ProcessManager
import
SubprocessError
import
shutil
from
.SlapOSControler
import
SlapOSControler
import
logging
from
.Utils
import
createFolder
import
string
import
random
from
ProcessManager
import
SubprocessError
,
ProcessManager
,
CancellationError
from
subprocess
import
CalledProcessError
from
NodeTestSuite
import
SlapOSInstance
from
Updater
import
Updater
from
Utils
import
dealShebang
from
erp5.util
import
taskdistribution
from
slapos.grid.utils
import
md5digest
from
slapos.grid.utils
import
md5digest
class
UnitTestRunner
():
def
dealShebang
(
run_test_suite_path
):
with
open
(
run_test_suite_path
)
as
f
:
if
f
.
read
(
2
)
==
'#!'
:
return
f
.
readline
().
split
(
None
,
1
)
return
[]
class
UnitTestRunner
(
object
):
def
__init__
(
self
,
testnode
):
def
__init__
(
self
,
testnode
):
self
.
testnode
=
testnode
self
.
testnode
=
testnode
...
@@ -53,12 +47,11 @@ class UnitTestRunner():
...
@@ -53,12 +47,11 @@ class UnitTestRunner():
"""
"""
Create a SlapOSControler
Create a SlapOSControler
"""
"""
return
SlapOSControler
.
SlapOSControler
(
return
SlapOSControler
(
working_directory
,
working_directory
,
self
.
testnode
.
config
,
self
.
testnode
.
config
,
self
.
testnode
.
log
)
self
.
testnode
.
log
)
def
_prepareSlapOS
(
self
,
working_directory
,
slapos_instance
,
log
,
def
_prepareSlapOS
(
self
,
working_directory
,
slapos_instance
,
log
,
create_partition
=
1
,
software_path_list
=
None
,
**
kw
):
create_partition
=
1
,
software_path_list
=
None
,
**
kw
):
"""
"""
...
@@ -66,11 +59,11 @@ class UnitTestRunner():
...
@@ -66,11 +59,11 @@ class UnitTestRunner():
"""
"""
slapproxy_log
=
os
.
path
.
join
(
self
.
testnode
.
config
[
'log_directory'
],
slapproxy_log
=
os
.
path
.
join
(
self
.
testnode
.
config
[
'log_directory'
],
'slapproxy.log'
)
'slapproxy.log'
)
log
(
'Configured slapproxy log to %r'
%
slapproxy_log
)
log
(
'Configured slapproxy log to %r'
,
slapproxy_log
)
reset_software
=
slapos_instance
.
retry_software_count
>
10
reset_software
=
slapos_instance
.
retry_software_count
>
10
if
reset_software
:
if
reset_software
:
slapos_instance
.
retry_software_count
=
0
slapos_instance
.
retry_software_count
=
0
log
(
'testnode, retry_software_count : %r'
%
\
log
(
'testnode, retry_software_count : %r'
,
slapos_instance
.
retry_software_count
)
slapos_instance
.
retry_software_count
)
# XXX Create a new controler because working_directory can be
# XXX Create a new controler because working_directory can be
...
@@ -108,26 +101,22 @@ class UnitTestRunner():
...
@@ -108,26 +101,22 @@ class UnitTestRunner():
"""
"""
# report-url, report-project and suite-url are required to seleniumrunner
# report-url, report-project and suite-url are required to seleniumrunner
# instance. This is a hack which must be removed.
# instance. This is a hack which must be removed.
cluster_configuration
=
{}
config
=
self
.
testnode
.
config
config
=
self
.
testnode
.
config
cluster_configuration
[
'report-url'
]
=
config
.
get
(
"report-url"
,
""
)
return
self
.
_prepareSlapOS
(
config
[
'slapos_directory'
],
cluster_configuration
[
'report-project'
]
=
config
.
get
(
"report-project"
,
""
)
cluster_configuration
[
'suite-url'
]
=
config
.
get
(
"suite-url"
,
""
)
return
self
.
_prepareSlapOS
(
self
.
testnode
.
config
[
'slapos_directory'
],
test_node_slapos
,
self
.
testnode
.
log
,
create_partition
=
0
,
test_node_slapos
,
self
.
testnode
.
log
,
create_partition
=
0
,
software_path_list
=
self
.
testnode
.
config
.
get
(
"software_list"
),
software_path_list
=
config
.
get
(
"software_list"
),
cluster_configuration
=
cluster_configuration
cluster_configuration
=
{
)
'report-url'
:
config
.
get
(
"report-url"
,
""
),
'report-project'
:
config
.
get
(
"report-project"
,
""
),
'suite-url'
:
config
.
get
(
"suite-url"
,
""
),
})
def
prepareSlapOSForTestSuite
(
self
,
node_test_suite
):
def
prepareSlapOSForTestSuite
(
self
,
node_test_suite
):
"""
"""
Build softwares needed by testsuites
Build softwares needed by testsuites
"""
"""
log
=
self
.
testnode
.
log
if
log
is
None
:
log
=
self
.
testnode
.
log
return
self
.
_prepareSlapOS
(
node_test_suite
.
working_directory
,
return
self
.
_prepareSlapOS
(
node_test_suite
.
working_directory
,
node_test_suite
,
log
,
node_test_suite
,
self
.
testnode
.
log
,
software_path_list
=
[
node_test_suite
.
custom_profile_path
],
software_path_list
=
[
node_test_suite
.
custom_profile_path
],
cluster_configuration
=
{
'_'
:
json
.
dumps
(
node_test_suite
.
cluster_configuration
)})
cluster_configuration
=
{
'_'
:
json
.
dumps
(
node_test_suite
.
cluster_configuration
)})
...
@@ -171,8 +160,7 @@ class UnitTestRunner():
...
@@ -171,8 +160,7 @@ class UnitTestRunner():
# From this point, test runner becomes responsible for updating test
# From this point, test runner becomes responsible for updating test
# result. We only do cleanup if the test runner itself is not able
# result. We only do cleanup if the test runner itself is not able
# to run.
# to run.
SlapOSControler
.
createFolder
(
node_test_suite
.
test_suite_directory
,
createFolder
(
node_test_suite
.
test_suite_directory
,
clean
=
True
)
clean
=
True
)
self
.
testnode
.
process_manager
.
spawn
(
*
invocation_list
,
self
.
testnode
.
process_manager
.
spawn
(
*
invocation_list
,
cwd
=
node_test_suite
.
test_suite_directory
,
cwd
=
node_test_suite
.
test_suite_directory
,
log_prefix
=
'runTestSuite'
,
get_output
=
False
)
log_prefix
=
'runTestSuite'
,
get_output
=
False
)
...
...
erp5/util/testnode/Updater.py
View file @
a01da084
...
@@ -28,11 +28,8 @@ import errno
...
@@ -28,11 +28,8 @@ import errno
import
os
import
os
import
re
import
re
import
shutil
import
shutil
import
subprocess
import
sys
import
sys
import
threading
from
.ProcessManager
import
SubprocessError
from
ProcessManager
import
SubprocessError
SVN_UP_REV
=
re
.
compile
(
r'^(?:At|Updated to) revision (\
d+).$
')
SVN_UP_REV
=
re
.
compile
(
r'^(?:At|Updated to) revision (\
d+).$
')
SVN_CHANGED_REV = re.compile(r'
^
Last
Changed
Rev
.
*
:
\
s
*
(
\
d
+
)
', re.MULTILINE)
SVN_CHANGED_REV = re.compile(r'
^
Last
Changed
Rev
.
*
:
\
s
*
(
\
d
+
)
', re.MULTILINE)
...
@@ -44,9 +41,8 @@ SVN_TYPE = 'svn'
...
@@ -44,9 +41,8 @@ SVN_TYPE = 'svn'
class Updater(object):
class Updater(object):
_git_cache = {}
_git_cache = {}
stdin = file(os.devnull)
def __init__(self, repository_path, log, revision=None, git_binary=
None
,
def __init__(self, repository_path, log, revision=None, git_binary=
'
git
'
,
branch=None, realtime_output=True, process_manager=None, url=None,
branch=None, realtime_output=True, process_manager=None, url=None,
working_directory=None):
working_directory=None):
self.log = log
self.log = log
...
@@ -81,7 +77,7 @@ class Updater(object):
...
@@ -81,7 +77,7 @@ class Updater(object):
def deletePycFiles(self, path):
def deletePycFiles(self, path):
"""Delete *.pyc files so that deleted/moved files can not be imported"""
"""Delete *.pyc files so that deleted/moved files can not be imported"""
for path,
dir_list
, file_list in os.walk(path):
for path,
_
, file_list in os.walk(path):
for file in file_list:
for file in file_list:
if file[-4:] in ('
.
pyc
', '
.
pyo
'):
if file[-4:] in ('
.
pyc
', '
.
pyo
'):
# allow several processes clean the same folder at the same time
# allow several processes clean the same folder at the same time
...
@@ -115,17 +111,17 @@ class Updater(object):
...
@@ -115,17 +111,17 @@ class Updater(object):
git_repository_path = os.path.join(self.getRepositoryPath(), '
.
git
')
git_repository_path = os.path.join(self.getRepositoryPath(), '
.
git
')
name = os.path.basename(os.path.normpath(self.getRepositoryPath()))
name = os.path.basename(os.path.normpath(self.getRepositoryPath()))
git_repository_link_path = os.path.join(self.getRepositoryPath(), '
%
s
.
git
' %name)
git_repository_link_path = os.path.join(self.getRepositoryPath(), '
%
s
.
git
' %name)
self.log("checking link %s -> %s.."
self.log("checking link %s -> %s.."
,
%(git_repository_link_path,git_repository_path)
)
git_repository_link_path, git_repository_path
)
if ( not os.path.lexists(git_repository_link_path) and
\
if ( not os.path.lexists(git_repository_link_path) and
\
not os.path.exists(git_repository_link_path) ):
not os.path.exists(git_repository_link_path) ):
try:
try:
os.symlink(git_repository_path, git_repository_link_path)
os.symlink(git_repository_path, git_repository_link_path)
self.log("link: %s -> %s created"
self.log("link: %s -> %s created"
,
%(git_repository_link_path,git_repository_path)
)
git_repository_link_path, git_repository_path
)
except:
except
OSError
:
self.log("Cannot create link from %s -> %s"
self.log("Cannot create link from %s -> %s"
,
%(git_repository_link_path,git_repository_path)
)
git_repository_link_path, git_repository_path
)
def _git_find_rev(self, ref):
def _git_find_rev(self, ref):
try:
try:
...
@@ -152,7 +148,7 @@ class Updater(object):
...
@@ -152,7 +148,7 @@ class Updater(object):
raise NotImplementedError
raise NotImplementedError
def deleteRepository(self):
def deleteRepository(self):
self.log("Wrong repository or wrong url, deleting repos %s"
%
\
self.log("Wrong repository or wrong url, deleting repos %s"
,
self.repository_path)
self.repository_path)
shutil.rmtree(self.repository_path)
shutil.rmtree(self.repository_path)
...
@@ -165,7 +161,7 @@ class Updater(object):
...
@@ -165,7 +161,7 @@ class Updater(object):
remote_url = self._git("config", "--get", "remote.origin.url")
remote_url = self._git("config", "--get", "remote.origin.url")
if remote_url == self.url:
if remote_url == self.url:
correct_url = True
correct_url = True
except
(SubprocessError,) as e
:
except
SubprocessError
:
self.log("SubprocessError", exc_info=sys.exc_info())
self.log("SubprocessError", exc_info=sys.exc_info())
if not(correct_url):
if not(correct_url):
self.deleteRepository()
self.deleteRepository()
...
...
erp5/util/testnode/Utils.py
View file @
a01da084
import
sys
import
os
import
json
import
shutil
import
shutil
import
string
from
random
import
choice
def
createFolder
(
folder
,
clean
=
False
):
if
os
.
path
.
exists
(
folder
):
if
not
clean
:
return
shutil
.
rmtree
(
folder
)
os
.
mkdir
(
folder
)
def
deunicodeData
(
data
):
def
deunicodeData
(
data
):
if
isinstance
(
data
,
list
):
if
isinstance
(
data
,
list
):
new_data
=
[]
return
map
(
deunicodeData
,
data
)
for
sub_data
in
data
:
if
isinstance
(
data
,
unicode
):
new_data
.
append
(
deunicodeData
(
sub_data
))
return
data
.
encode
(
'utf8'
)
elif
isinstance
(
data
,
unicode
):
if
isinstance
(
data
,
dict
):
new_data
=
data
.
encode
(
'utf8'
)
return
{
deunicodeData
(
key
):
deunicodeData
(
value
)
elif
isinstance
(
data
,
dict
):
for
key
,
value
in
data
.
iteritems
()}
new_data
=
{}
return
data
for
key
,
value
in
data
.
iteritems
():
key
=
deunicodeData
(
key
)
value
=
deunicodeData
(
value
)
new_data
[
key
]
=
value
else
:
new_data
=
data
return
new_data
def
dealShebang
(
run_test_suite_path
):
line
=
open
(
run_test_suite_path
,
'r'
).
readline
()
invocation_list
=
[]
if
line
[:
2
]
==
'#!'
:
invocation_list
=
line
[
2
:].
split
()
return
invocation_list
erp5/util/testnode/testnode.py
View file @
a01da084
This diff is collapsed.
Click to expand it.
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment