Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
S
slapos.core
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
Carlos Ramos Carreño
slapos.core
Commits
3bb36903
Commit
3bb36903
authored
Oct 25, 2021
by
Xavier Thompson
Browse files
Options
Browse Files
Download
Plain Diff
slapgrid: Process promises with SR python
See merge request
!329
parents
b1dd96b1
45320b33
Changes
8
Expand all
Hide whitespace changes
Inline
Side-by-side
Showing
8 changed files
with
333 additions
and
132 deletions
+333
-132
slapos/grid/SlapObject.py
slapos/grid/SlapObject.py
+5
-1
slapos/grid/promise/__init__.py
slapos/grid/promise/__init__.py
+56
-59
slapos/grid/promise/runpromises.py
slapos/grid/promise/runpromises.py
+68
-0
slapos/grid/slapgrid.py
slapos/grid/slapgrid.py
+94
-62
slapos/grid/utils.py
slapos/grid/utils.py
+14
-0
slapos/testing/utils.py
slapos/testing/utils.py
+7
-1
slapos/tests/test_slapgrid.py
slapos/tests/test_slapgrid.py
+77
-9
slapos/util.py
slapos/util.py
+12
-0
No files found.
slapos/grid/SlapObject.py
View file @
3bb36903
...
...
@@ -45,7 +45,8 @@ from six.moves.configparser import ConfigParser
from
supervisor
import
xmlrpc
from
slapos.grid.utils
import
(
md5digest
,
getCleanEnvironment
,
SlapPopen
,
dropPrivileges
,
updateFile
)
SlapPopen
,
dropPrivileges
,
updateFile
,
getPythonExecutableFromSoftwarePath
)
from
slapos.grid
import
utils
# for methods that could be mocked, access them through the module
from
slapos.slap.slap
import
NotFoundError
from
slapos.grid.svcbackend
import
getSupervisorRPC
...
...
@@ -472,6 +473,8 @@ class Partition(object):
self
.
instance_min_free_space
=
instance_min_free_space
self
.
instance_python
=
getPythonExecutableFromSoftwarePath
(
self
.
software_path
)
def
check_free_space
(
self
):
required
=
self
.
instance_min_free_space
or
0
...
...
@@ -705,6 +708,7 @@ class Partition(object):
debug
=
self
.
buildout_debug
)
self
.
generateSupervisorConfigurationFile
()
self
.
createRetentionLockDelay
()
self
.
instance_python
=
getPythonExecutableFromSoftwarePath
(
self
.
software_path
)
def
generateSupervisorConfiguration
(
self
):
"""
...
...
slapos/grid/promise/__init__.py
View file @
3bb36903
...
...
@@ -44,7 +44,7 @@ import hashlib
from
datetime
import
datetime
from
multiprocessing
import
Process
,
Queue
as
MQueue
from
six.moves
import
queue
,
reload_module
from
slapos.util
import
str2bytes
,
mkdir_p
,
chownDirectory
from
slapos.util
import
str2bytes
,
mkdir_p
,
chownDirectory
,
listifdir
from
slapos.grid.utils
import
dropPrivileges
,
killProcessTree
from
slapos.grid.promise
import
interface
from
slapos.grid.promise.generic
import
(
GenericPromise
,
PromiseQueueResult
,
...
...
@@ -731,65 +731,63 @@ class PromiseLauncher(object):
error
=
0
success
=
0
promise_name_list
=
[]
if
os
.
path
.
exists
(
self
.
promise_folder
)
and
os
.
path
.
isdir
(
self
.
promise_folder
):
for
promise_name
in
os
.
listdir
(
self
.
promise_folder
):
for
suffix
in
[
'.pyc'
,
'.pyo'
]:
if
promise_name
.
endswith
(
suffix
):
promise_path
=
os
.
path
.
join
(
self
.
promise_folder
,
promise_name
)
if
not
os
.
path
.
exists
(
promise_path
[:
-
1
]):
try
:
os
.
unlink
(
promise_path
)
except
Exception
as
e
:
self
.
logger
.
warning
(
'Failed to remove %r because of %s'
,
promise_path
,
e
)
else
:
self
.
logger
.
debug
(
'Removed stale %r'
,
promise_path
)
if
promise_name
.
startswith
(
'__init__'
)
or
\
not
promise_name
.
endswith
(
'.py'
):
continue
promise_name_list
.
append
(
promise_name
)
if
self
.
run_only_promise_list
is
not
None
and
not
\
promise_name
in
self
.
run_only_promise_list
:
continue
for
promise_name
in
listifdir
(
self
.
promise_folder
):
if
promise_name
.
endswith
((
'.pyc'
,
'.pyo'
)):
promise_path
=
os
.
path
.
join
(
self
.
promise_folder
,
promise_name
)
config
=
{
'path'
:
promise_path
,
'name'
:
promise_name
}
config
.
update
(
base_config
)
promise_result
=
self
.
_launchPromise
(
promise_name
,
promise_path
,
config
)
if
promise_result
:
change_date
=
promise_result
.
date
.
strftime
(
'%Y-%m-%dT%H:%M:%S+0000'
)
if
promise_result
.
hasFailed
():
promise_status
=
'FAILED'
if
not
os
.
path
.
exists
(
promise_path
[:
-
1
]):
try
:
os
.
unlink
(
promise_path
)
except
Exception
as
e
:
self
.
logger
.
warning
(
'Failed to remove %r because of %s'
,
promise_path
,
e
)
else
:
self
.
logger
.
debug
(
'Removed stale %r'
,
promise_path
)
if
promise_name
.
startswith
(
'__init__'
)
or
\
not
promise_name
.
endswith
(
'.py'
):
continue
promise_name_list
.
append
(
promise_name
)
if
self
.
run_only_promise_list
is
not
None
and
not
\
promise_name
in
self
.
run_only_promise_list
:
continue
promise_path
=
os
.
path
.
join
(
self
.
promise_folder
,
promise_name
)
config
=
{
'path'
:
promise_path
,
'name'
:
promise_name
}
config
.
update
(
base_config
)
promise_result
=
self
.
_launchPromise
(
promise_name
,
promise_path
,
config
)
if
promise_result
:
change_date
=
promise_result
.
date
.
strftime
(
'%Y-%m-%dT%H:%M:%S+0000'
)
if
promise_result
.
hasFailed
():
promise_status
=
'FAILED'
error
+=
1
else
:
promise_status
=
"OK"
success
+=
1
if
promise_name
in
previous_state_dict
:
status
,
previous_change_date
,
_
=
previous_state_dict
[
promise_name
]
if
promise_status
==
status
:
change_date
=
previous_change_date
message
=
promise_result
.
message
if
promise_result
.
message
else
""
new_state_dict
[
promise_name
]
=
[
promise_status
,
change_date
,
hashlib
.
md5
(
str2bytes
(
message
)).
hexdigest
()]
if
promise_result
.
hasFailed
()
and
not
failed_promise_name
:
failed_promise_name
=
promise_name
failed_promise_output
=
promise_result
.
message
else
:
# The promise was skip, so for statistic point of view we preserve
# the previous result
if
promise_name
in
new_state_dict
:
if
new_state_dict
[
promise_name
][
0
]
==
"FAILED"
:
error
+=
1
else
:
promise_status
=
"OK"
success
+=
1
if
promise_name
in
previous_state_dict
:
status
,
previous_change_date
,
_
=
previous_state_dict
[
promise_name
]
if
promise_status
==
status
:
change_date
=
previous_change_date
message
=
promise_result
.
message
if
promise_result
.
message
else
""
new_state_dict
[
promise_name
]
=
[
promise_status
,
change_date
,
hashlib
.
md5
(
str2bytes
(
message
)).
hexdigest
()]
if
promise_result
.
hasFailed
()
and
not
failed_promise_name
:
failed_promise_name
=
promise_name
failed_promise_output
=
promise_result
.
message
else
:
# The promise was skip, so for statistic point of view we preserve
# the previous result
if
promise_name
in
new_state_dict
:
if
new_state_dict
[
promise_name
][
0
]
==
"FAILED"
:
error
+=
1
else
:
success
+=
1
if
not
self
.
run_only_promise_list
and
len
(
promise_name_list
)
>
0
:
# cleanup stale json files
...
...
@@ -816,10 +814,9 @@ class PromiseLauncher(object):
if
key
not
in
promise_name_list
:
new_state_dict
.
pop
(
key
,
None
)
if
not
self
.
run_only_promise_list
and
os
.
path
.
exists
(
self
.
legacy_promise_folder
)
\
and
os
.
path
.
isdir
(
self
.
legacy_promise_folder
):
if
not
self
.
run_only_promise_list
:
# run legacy promise styles
for
promise_name
in
os
.
list
dir
(
self
.
legacy_promise_folder
):
for
promise_name
in
listif
dir
(
self
.
legacy_promise_folder
):
promise_path
=
os
.
path
.
join
(
self
.
legacy_promise_folder
,
promise_name
)
if
not
os
.
path
.
isfile
(
promise_path
)
or
\
not
os
.
access
(
promise_path
,
os
.
X_OK
):
...
...
slapos/grid/promise/runpromises.py
0 → 100644
View file @
3bb36903
from
__future__
import
print_function
import
argparse
import
ast
import
os
import
sys
# Parse arguments
parser
=
argparse
.
ArgumentParser
()
parser
.
add_argument
(
'--promise-folder'
,
required
=
True
)
parser
.
add_argument
(
'--legacy-promise-folder'
,
default
=
None
)
parser
.
add_argument
(
'--promise-timeout'
,
type
=
int
,
default
=
20
)
parser
.
add_argument
(
'--partition-folder'
,
default
=
None
)
parser
.
add_argument
(
'--log-folder'
,
default
=
None
)
parser
.
add_argument
(
'--force'
,
action
=
'store_true'
)
parser
.
add_argument
(
'--check-anomaly'
,
action
=
'store_true'
)
parser
.
add_argument
(
'--debug'
,
action
=
'store_true'
)
parser
.
add_argument
(
'--master-url'
,
default
=
None
)
parser
.
add_argument
(
'--partition-cert'
,
default
=
None
)
parser
.
add_argument
(
'--partition-key'
,
default
=
None
)
parser
.
add_argument
(
'--partition-id'
,
default
=
None
)
parser
.
add_argument
(
'--computer-id'
,
default
=
None
)
args
=
parser
.
parse_args
()
# Extract slapos.core path and all dependencies from first promise found
# to import slapos.core
promise_folder
=
args
.
promise_folder
promise_file
=
next
(
p
for
p
in
os
.
listdir
(
promise_folder
)
if
p
.
endswith
(
'.py'
)
and
not
p
.
startswith
(
'__init__'
)
)
with
open
(
os
.
path
.
join
(
promise_folder
,
promise_file
))
as
f
:
promise_content
=
f
.
read
()
tree
=
ast
.
parse
(
promise_content
,
mode
=
'exec'
)
sys
.
path
[
0
:
0
]
=
eval
(
compile
(
ast
.
Expression
(
tree
.
body
[
1
].
value
),
''
,
'eval'
))
from
slapos.grid.promise
import
PromiseLauncher
,
PromiseError
from
slapos.cli.entry
import
SlapOSApp
# Configure promise launcher
# with the same logger as standard slapos command
app
=
SlapOSApp
()
app
.
options
,
_
=
app
.
parser
.
parse_known_args
([])
app
.
configure_logging
()
config
=
{
k
.
replace
(
'_'
,
'-'
)
:
v
for
k
,
v
in
vars
(
args
).
items
()}
promise_checker
=
PromiseLauncher
(
config
=
config
,
logger
=
app
.
log
)
# Run promises
# Redirect stdout to stderr (logger only uses stderr already)
# to reserve stdout for error reporting
out
=
os
.
dup
(
1
)
os
.
dup2
(
2
,
1
)
try
:
promise_checker
.
run
()
except
Exception
as
e
:
os
.
write
(
out
,
str
(
e
))
sys
.
exit
(
2
if
isinstance
(
e
,
PromiseError
)
else
1
)
slapos/grid/slapgrid.py
View file @
3bb36903
This diff is collapsed.
Click to expand it.
slapos/grid/utils.py
View file @
3bb36903
...
...
@@ -164,6 +164,20 @@ def md5digest(url):
return
hashlib
.
md5
(
url
.
encode
(
'utf-8'
)).
hexdigest
()
def
getPythonExecutableFromSoftwarePath
(
software_path
):
"""
Return the path of the python executable installed for the software release
installed as `software_path`.
"""
try
:
with
open
(
os
.
path
.
join
(
software_path
,
'bin'
,
'buildout'
))
as
f
:
shebang
=
f
.
readline
()
except
OSError
:
return
if
shebang
.
startswith
(
'#!'
):
return
shebang
[
2
:].
split
(
None
,
1
)[
0
]
def
getCleanEnvironment
(
logger
,
home_path
=
'/tmp'
):
changed_env
=
{}
removed_env
=
[]
...
...
slapos/testing/utils.py
View file @
3bb36903
...
...
@@ -41,6 +41,8 @@ from contextlib import closing
from
six.moves
import
BaseHTTPServer
from
six.moves
import
urllib_parse
from
..grid.utils
import
getPythonExecutableFromSoftwarePath
try
:
import
typing
if
typing
.
TYPE_CHECKING
:
...
...
@@ -77,8 +79,12 @@ def getPromisePluginParameterDict(filepath):
This allow to check that monitoring plugin are using a proper config.
"""
executable
=
getPythonExecutableFromSoftwarePath
(
os
.
path
.
join
(
os
.
path
.
dirname
(
os
.
path
.
dirname
(
os
.
path
.
dirname
(
filepath
))),
'software_release'
))
extra_config_dict_json
=
subprocess
.
check_output
([
sys
.
executable
,
executable
,
"-c"
,
"""
import json, sys
...
...
slapos/tests/test_slapgrid.py
View file @
3bb36903
...
...
@@ -45,6 +45,7 @@ import json
import
re
import
grp
import
hashlib
import
errno
import
mock
from
mock
import
patch
...
...
@@ -65,6 +66,11 @@ import slapos.grid.SlapObject
from
slapos
import
manager
as
slapmanager
from
slapos.util
import
dumps
from
slapos
import
__path__
as
slapos_path
from
zope
import
__path__
as
zope_path
PROMISE_PATHS
=
sorted
(
set
(
map
(
os
.
path
.
dirname
,
slapos_path
+
zope_path
)))
import
httmock
...
...
@@ -113,6 +119,9 @@ touch worked
"""
PROMISE_CONTENT_TEMPLATE
=
"""
import sys
sys.path[0:0] = %(paths)r
from zope.interface import implementer
from slapos.grid.promise import interface
from slapos.grid.promise import GenericPromise
...
...
@@ -122,7 +131,7 @@ class RunPromise(GenericPromise):
def __init__(self, config):
super(RunPromise, self).__init__(config)
self.setPeriodicity(minute=%(periodicity)
s
)
self.setPeriodicity(minute=%(periodicity)
r
)
def sense(self):
%(content)s
...
...
@@ -133,11 +142,11 @@ class RunPromise(GenericPromise):
self.logger.info("success")
def anomaly(self):
return self._anomaly(result_count=2, failure_amount=%(failure_amount)
s
)
return self._anomaly(result_count=2, failure_amount=%(failure_amount)
r
)
def test(self):
return self._test(result_count=1, failure_amount=%(failure_amount)
s
)
"""
return self._test(result_count=1, failure_amount=%(failure_amount)
r
)
"""
class
BasicMixin
(
object
):
def
setUp
(
self
):
...
...
@@ -150,6 +159,14 @@ class BasicMixin(object):
del
os
.
environ
[
'SLAPGRID_INSTANCE_ROOT'
]
logging
.
basicConfig
(
level
=
logging
.
DEBUG
)
self
.
setSlapgrid
()
self
.
setMock
()
def
setMock
(
self
):
module
=
slapos
.
grid
.
SlapObject
func
=
'getPythonExecutableFromSoftwarePath'
orig
=
getattr
(
module
,
func
)
self
.
addCleanup
(
setattr
,
module
,
func
,
orig
)
setattr
(
module
,
func
,
lambda
software_path
:
None
)
def
setSlapgrid
(
self
,
develop
=
False
,
force_stop
=
False
):
if
getattr
(
self
,
'master_url'
,
None
)
is
None
:
...
...
@@ -576,7 +593,8 @@ class InstanceForTest(object):
{
'success'
:
success
,
'content'
:
promise_content
,
'failure_amount'
:
failure_count
,
'periodicity'
:
periodicity
}
'periodicity'
:
periodicity
,
'paths'
:
PROMISE_PATHS
}
with
open
(
os
.
path
.
join
(
promise_dir
,
promise_name
),
'w'
)
as
f
:
f
.
write
(
_promise_content
)
...
...
@@ -599,7 +617,7 @@ class InstanceForTest(object):
class
SoftwareForTest
(
object
):
"""
Class to prepare and simulate software.
each instance has a so
tf
ware attributed
each instance has a so
ft
ware attributed
"""
def
__init__
(
self
,
software_root
,
name
=
''
):
"""
...
...
@@ -2297,6 +2315,7 @@ exit 1
self
.
assertFalse
(
os
.
path
.
exists
(
promise_file
))
self
.
assertTrue
(
instance
.
error
)
class
TestSlapgridDestructionLock
(
MasterMixin
,
unittest
.
TestCase
):
def
test_retention_lock
(
self
):
"""
...
...
@@ -3999,9 +4018,6 @@ class TestSlapgridPromiseWithMaster(MasterMixin, unittest.TestCase):
self.assertEqual('success', result["
result
"]["
message
"])
def test_one_succeeding_one_timing_out_promises(self):
computer = ComputerForTest(self.software_root, self.instance_root)
with httmock.HTTMock(computer.request_handler):
...
...
@@ -4095,6 +4111,58 @@ class TestSlapgridPromiseWithMaster(MasterMixin, unittest.TestCase):
"
.
slapgrid
/
promise
/
result
/
fail
.
status
.
json
")))
class TestSlapgridPluginPromiseWithInstancePython(TestSlapgridPromiseWithMaster):
expect_plugin = False
def setPython(self):
self.python_called = os.path.join(self.software_root, 'called')
wrapper = """#!/bin/sh
touch %s
exec %s "
$
@
"
""" % (self.python_called, sys.executable)
path = os.path.join(self.software_root, 'python')
with open(path, 'w') as f:
f.write(wrapper)
os.chmod(path, 0o755)
return path
def patchBuildoutSetter(self):
cls = SoftwareForTest
attr = 'setBuildout'
orig = getattr(cls, attr)
def setBuildout(soft):
buildout = "
#!" + self.setPython()
orig
(
soft
,
buildout
)
self
.
addCleanup
(
setattr
,
cls
,
attr
,
orig
)
setattr
(
cls
,
attr
,
setBuildout
)
def
patchPluginSetter
(
self
):
cls
=
InstanceForTest
attr
=
'setPluginPromise'
orig
=
getattr
(
cls
,
attr
)
def
setPluginPromise
(
inst
,
*
args
,
**
kwargs
):
self
.
expect_plugin
=
inst
.
requested_state
==
'started'
return
orig
(
inst
,
*
args
,
**
kwargs
)
self
.
addCleanup
(
setattr
,
cls
,
attr
,
orig
)
setattr
(
cls
,
attr
,
setPluginPromise
)
def
setMock
(
self
):
self
.
patchBuildoutSetter
()
self
.
patchPluginSetter
()
def
tearDown
(
self
):
try
:
os
.
remove
(
self
.
python_called
)
called
=
True
except
OSError
as
e
:
if
e
.
errno
!=
errno
.
ENOENT
:
raise
called
=
False
finally
:
super
(
TestSlapgridPluginPromiseWithInstancePython
,
self
).
tearDown
()
self
.
assertEqual
(
self
.
expect_plugin
,
called
)
class
TestSVCBackend
(
unittest
.
TestCase
):
"""Tests for supervisor backend.
"""
...
...
slapos/util.py
View file @
3bb36903
...
...
@@ -95,6 +95,18 @@ def mkdir_p(path, mode=0o700):
raise
def
listifdir
(
path
):
"""
Like listdir, but returns an empty tuple if the path is not a directory.
"""
try
:
return
os
.
listdir
(
path
)
except
OSError
as
e
:
if
e
.
errno
!=
errno
.
ENOENT
:
raise
return
()
def
chownDirectory
(
path
,
uid
,
gid
):
if
os
.
getuid
()
!=
0
:
# we are probably inside of a webrunner
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment