Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
S
slapos.core
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Labels
Merge Requests
17
Merge Requests
17
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Jobs
Commits
Open sidebar
nexedi
slapos.core
Commits
bbd3e3e0
Commit
bbd3e3e0
authored
Jun 24, 2013
by
Cédric de Saint Martin
Browse files
Options
Browse Files
Download
Plain Diff
Merge remote-tracking branch 'origin/tweak-grid2'
parents
71a21a4b
9be006b0
Changes
3
Hide whitespace changes
Inline
Side-by-side
Showing
3 changed files
with
62 additions
and
71 deletions
+62
-71
setup.py
setup.py
+1
-0
slapos/grid/slapgrid.py
slapos/grid/slapgrid.py
+12
-18
slapos/tests/slapgrid.py
slapos/tests/slapgrid.py
+49
-53
No files found.
setup.py
View file @
bbd3e3e0
...
@@ -54,6 +54,7 @@ setup(name=name,
...
@@ -54,6 +54,7 @@ setup(name=name,
),},
),},
tests_require
=
[
tests_require
=
[
'pyflakes'
,
'pyflakes'
,
'mock'
],
],
zip_safe
=
False
,
# proxy depends on Flask, which has issues with
zip_safe
=
False
,
# proxy depends on Flask, which has issues with
# accessing templates
# accessing templates
...
...
slapos/grid/slapgrid.py
View file @
bbd3e3e0
...
@@ -135,9 +135,6 @@ def merged_options(args, configp):
...
@@ -135,9 +135,6 @@ def merged_options(args, configp):
if
options
.
get
(
'all'
):
if
options
.
get
(
'all'
):
options
[
'develop'
]
=
True
options
[
'develop'
]
=
True
if
options
.
get
(
'maximum_periodicity'
)
is
not
None
:
options
[
'force_periodicity'
]
=
True
# Supervisord configuration location
# Supervisord configuration location
if
not
options
.
get
(
'supervisord_configuration_path'
):
if
not
options
.
get
(
'supervisord_configuration_path'
):
options
[
'supervisord_configuration_path'
]
=
\
options
[
'supervisord_configuration_path'
]
=
\
...
@@ -199,8 +196,7 @@ def create_slapgrid_object(options, logger):
...
@@ -199,8 +196,7 @@ def create_slapgrid_object(options, logger):
supervisord_configuration_path
=
op
[
'supervisord_configuration_path'
],
supervisord_configuration_path
=
op
[
'supervisord_configuration_path'
],
buildout
=
op
.
get
(
'buildout'
),
buildout
=
op
.
get
(
'buildout'
),
logger
=
logger
,
logger
=
logger
,
force_periodicity
=
op
.
get
(
'force_periodicity'
,
False
),
maximum_periodicity
=
op
.
get
(
'maximum_periodicity'
,
86400
),
maximum_periodicity
=
op
.
get
(
'maximum_periodicity'
,
86400
),
key_file
=
op
.
get
(
'key_file'
),
key_file
=
op
.
get
(
'key_file'
),
cert_file
=
op
.
get
(
'cert_file'
),
cert_file
=
op
.
get
(
'cert_file'
),
signature_private_key_file
=
op
.
get
(
'signature_private_key_file'
),
signature_private_key_file
=
op
.
get
(
'signature_private_key_file'
),
...
@@ -256,7 +252,6 @@ class Slapgrid(object):
...
@@ -256,7 +252,6 @@ class Slapgrid(object):
supervisord_configuration_path
,
supervisord_configuration_path
,
buildout
,
buildout
,
logger
,
logger
,
force_periodicity
=
False
,
maximum_periodicity
=
86400
,
maximum_periodicity
=
86400
,
key_file
=
None
,
key_file
=
None
,
cert_file
=
None
,
cert_file
=
None
,
...
@@ -331,7 +326,6 @@ class Slapgrid(object):
...
@@ -331,7 +326,6 @@ class Slapgrid(object):
self
.
computer_partition_filter_list
=
\
self
.
computer_partition_filter_list
=
\
computer_partition_filter_list
.
split
(
","
)
computer_partition_filter_list
.
split
(
","
)
self
.
maximum_periodicity
=
maximum_periodicity
self
.
maximum_periodicity
=
maximum_periodicity
self
.
force_periodicity
=
force_periodicity
def
getWatchdogLine
(
self
):
def
getWatchdogLine
(
self
):
invocation_list
=
[
WATCHDOG_PATH
]
invocation_list
=
[
WATCHDOG_PATH
]
...
@@ -576,15 +570,13 @@ class Slapgrid(object):
...
@@ -576,15 +570,13 @@ class Slapgrid(object):
periodicity
=
self
.
maximum_periodicity
periodicity
=
self
.
maximum_periodicity
if
software_path
:
if
software_path
:
# Get periodicity from periodicity file if not forced
periodicity_path
=
os
.
path
.
join
(
software_path
,
'periodicity'
)
if
not
self
.
force_periodicity
:
if
os
.
path
.
exists
(
periodicity_path
):
periodicity_path
=
os
.
path
.
join
(
software_path
,
'periodicity'
)
try
:
if
os
.
path
.
exists
(
periodicity_path
):
periodicity
=
int
(
open
(
periodicity_path
).
read
())
try
:
except
ValueError
:
periodicity
=
int
(
open
(
periodicity_path
).
read
())
os
.
remove
(
periodicity_path
)
except
ValueError
:
self
.
logger
.
exception
(
''
)
os
.
remove
(
periodicity_path
)
self
.
logger
.
exception
(
''
)
# Check if timestamp from server is more recent than local one.
# Check if timestamp from server is more recent than local one.
# If not: it's not worth processing this partition (nothing has
# If not: it's not worth processing this partition (nothing has
...
@@ -595,13 +587,15 @@ class Slapgrid(object):
...
@@ -595,13 +587,15 @@ class Slapgrid(object):
last_runtime
=
int
(
os
.
path
.
getmtime
(
timestamp_path
))
last_runtime
=
int
(
os
.
path
.
getmtime
(
timestamp_path
))
if
timestamp
:
if
timestamp
:
try
:
try
:
if
int
(
timestamp
)
<=
int
(
old_timestamp
):
if
periodicity
==
0
:
os
.
remove
(
timestamp_path
)
elif
int
(
timestamp
)
<=
int
(
old_timestamp
):
if
computer_partition
.
getState
()
!=
COMPUTER_PARTITION_STARTED_STATE
:
if
computer_partition
.
getState
()
!=
COMPUTER_PARTITION_STARTED_STATE
:
return
return
# Check periodicity, i.e if periodicity is one day, partition
# Check periodicity, i.e if periodicity is one day, partition
# should be processed at least every day.
# should be processed at least every day.
# Only do it for "started" instances
# Only do it for "started" instances
if
int
(
time
.
time
())
<=
(
last_runtime
+
periodicity
):
if
int
(
time
.
time
())
<=
(
last_runtime
+
periodicity
)
or
periodicity
<
0
:
self
.
logger
.
info
(
'Partition already up-to-date, skipping.'
)
self
.
logger
.
info
(
'Partition already up-to-date, skipping.'
)
return
return
else
:
else
:
...
...
slapos/tests/slapgrid.py
View file @
bbd3e3e0
...
@@ -25,6 +25,7 @@
...
@@ -25,6 +25,7 @@
#
#
##############################################################################
##############################################################################
from
__future__
import
absolute_import
import
httplib
import
httplib
import
logging
import
logging
import
os
import
os
...
@@ -40,6 +41,7 @@ import unittest
...
@@ -40,6 +41,7 @@ import unittest
import
urlparse
import
urlparse
import
xml_marshaller
import
xml_marshaller
from
mock
import
patch
import
slapos.slap.slap
import
slapos.slap.slap
import
slapos.grid.utils
import
slapos.grid.utils
...
@@ -49,6 +51,7 @@ from slapos.grid.utils import md5digest
...
@@ -49,6 +51,7 @@ from slapos.grid.utils import md5digest
from
slapos.grid.watchdog
import
Watchdog
,
getWatchdogID
from
slapos.grid.watchdog
import
Watchdog
,
getWatchdogID
from
slapos.grid
import
SlapObject
from
slapos.grid
import
SlapObject
dummylogger
=
logging
.
getLogger
()
dummylogger
=
logging
.
getLogger
()
...
@@ -232,13 +235,13 @@ class MasterMixin(BasicMixin):
...
@@ -232,13 +235,13 @@ class MasterMixin(BasicMixin):
def
_patchHttplib
(
self
):
def
_patchHttplib
(
self
):
"""Overrides httplib"""
"""Overrides httplib"""
import
mock.httplib
import
slapos.tests.
mock.httplib
self
.
saved_httplib
=
{}
self
.
saved_httplib
=
{}
for
fake
in
vars
(
mock
.
httplib
):
for
fake
in
vars
(
slapos
.
tests
.
mock
.
httplib
):
self
.
saved_httplib
[
fake
]
=
getattr
(
httplib
,
fake
,
None
)
self
.
saved_httplib
[
fake
]
=
getattr
(
httplib
,
fake
,
None
)
setattr
(
httplib
,
fake
,
getattr
(
mock
.
httplib
,
fake
))
setattr
(
httplib
,
fake
,
getattr
(
slapos
.
tests
.
mock
.
httplib
,
fake
))
def
_unpatchHttplib
(
self
):
def
_unpatchHttplib
(
self
):
"""Restores httplib overriding"""
"""Restores httplib overriding"""
...
@@ -1021,7 +1024,6 @@ class TestSlapgridCPPartitionProcessing(MasterMixin, unittest.TestCase):
...
@@ -1021,7 +1024,6 @@ class TestSlapgridCPPartitionProcessing(MasterMixin, unittest.TestCase):
instance
.
timestamp
=
timestamp
instance
.
timestamp
=
timestamp
instance
.
requested_state
=
'started'
instance
.
requested_state
=
'started'
instance
.
software
.
setPeriodicity
(
1
)
instance
.
software
.
setPeriodicity
(
1
)
self
.
grid
.
force_periodicity
=
True
self
.
launchSlapgrid
()
self
.
launchSlapgrid
()
partition
=
os
.
path
.
join
(
self
.
instance_root
,
'0'
)
partition
=
os
.
path
.
join
(
self
.
instance_root
,
'0'
)
...
@@ -1037,36 +1039,7 @@ class TestSlapgridCPPartitionProcessing(MasterMixin, unittest.TestCase):
...
@@ -1037,36 +1039,7 @@ class TestSlapgridCPPartitionProcessing(MasterMixin, unittest.TestCase):
self
.
assertItemsEqual
(
os
.
listdir
(
partition
),
self
.
assertItemsEqual
(
os
.
listdir
(
partition
),
[
'.timestamp'
,
'buildout.cfg'
,
'software_release'
,
'worked'
])
[
'.timestamp'
,
'buildout.cfg'
,
'software_release'
,
'worked'
])
def
test_partition_periodicity_is_not_overloaded_if_forced
(
self
):
"""
If periodicity file in software directory but periodicity is forced
periodicity will be the one given by parameter
1. We set force_periodicity parameter to True
2. We put a periodicity file in the software release directory
with an unwanted periodicity
3. We process partition list and wait more than unwanted periodicity
4. We relaunch, partition should not be processed
"""
computer
=
ComputerForTest
(
self
.
software_root
,
self
.
instance_root
)
instance
=
computer
.
instance_list
[
0
]
timestamp
=
str
(
int
(
time
.
time
()))
instance
.
timestamp
=
timestamp
instance
.
requested_state
=
'started'
unwanted_periodicity
=
2
instance
.
software
.
setPeriodicity
(
unwanted_periodicity
)
self
.
grid
.
force_periodicity
=
True
self
.
launchSlapgrid
()
time
.
sleep
(
unwanted_periodicity
+
1
)
self
.
setSlapgrid
()
self
.
grid
.
force_periodicity
=
True
self
.
assertEqual
(
self
.
grid
.
processComputerPartitionList
(),
slapgrid
.
SLAPGRID_SUCCESS
)
self
.
assertNotEqual
(
unwanted_periodicity
,
self
.
grid
.
maximum_periodicity
)
self
.
assertEqual
(
computer
.
sequence
,
[
'getFullComputerInformation'
,
'availableComputerPartition'
,
'startedComputerPartition'
,
'getFullComputerInformation'
])
def
test_one_partition_periodicity_from_file_does_not_disturb_others
(
self
):
def
test_one_partition_periodicity_from_file_does_not_disturb_others
(
self
):
"""
"""
...
@@ -1192,6 +1165,49 @@ class TestSlapgridCPPartitionProcessing(MasterMixin, unittest.TestCase):
...
@@ -1192,6 +1165,49 @@ class TestSlapgridCPPartitionProcessing(MasterMixin, unittest.TestCase):
last_runtime
)
last_runtime
)
self
.
assertNotEqual
(
wanted_periodicity
,
self
.
grid
.
maximum_periodicity
)
self
.
assertNotEqual
(
wanted_periodicity
,
self
.
grid
.
maximum_periodicity
)
def
test_one_partition_is_never_processed_when_periodicity_is_negative
(
self
):
"""
Checks that a partition is not processed when
its periodicity is negative
1. We setup one instance and set periodicity at -1
2. We mock the install method from slapos.grid.slapgrid.Partition
3. We launch slapgrid once so that .timestamp file is created and check that install method is
indeed called (through mocked_method.called
4. We launch slapgrid anew and check that install as not been called again
"""
timestamp
=
str
(
int
(
time
.
time
()))
computer
=
ComputerForTest
(
self
.
software_root
,
self
.
instance_root
,
1
,
1
)
instance
=
computer
.
instance_list
[
0
]
instance
.
software
.
setPeriodicity
(
-
1
)
instance
.
timestamp
=
timestamp
with
patch
.
object
(
slapos
.
grid
.
slapgrid
.
Partition
,
'install'
,
return_value
=
None
)
as
mock_method
:
self
.
launchSlapgrid
()
self
.
assertTrue
(
mock_method
.
called
)
self
.
launchSlapgrid
()
self
.
assertEqual
(
mock_method
.
call_count
,
1
)
def
test_one_partition_is_always_processed_when_periodicity_is_zero
(
self
):
"""
Checks that a partition is always processed when
its periodicity is 0
1. We setup one instance and set periodicity at 0
2. We mock the install method from slapos.grid.slapgrid.Partition
3. We launch slapgrid once so that .timestamp file is created
4. We launch slapgrid anew and check that install has been called twice (one time because of the
new setup and one time because of periodicity = 0)
"""
timestamp
=
str
(
int
(
time
.
time
()))
computer
=
ComputerForTest
(
self
.
software_root
,
self
.
instance_root
,
1
,
1
)
instance
=
computer
.
instance_list
[
0
]
instance
.
software
.
setPeriodicity
(
0
)
instance
.
timestamp
=
timestamp
with
patch
.
object
(
slapos
.
grid
.
slapgrid
.
Partition
,
'install'
,
return_value
=
None
)
as
mock_method
:
self
.
launchSlapgrid
()
self
.
launchSlapgrid
()
self
.
assertEqual
(
mock_method
.
call_count
,
2
)
def
test_one_partition_buildout_fail_does_not_disturb_others
(
self
):
def
test_one_partition_buildout_fail_does_not_disturb_others
(
self
):
"""
"""
1. We set up two instance one using a corrupted buildout
1. We set up two instance one using a corrupted buildout
...
@@ -1552,26 +1568,6 @@ class TestSlapgridArgumentTuple(SlapgridInitialization):
...
@@ -1552,26 +1568,6 @@ class TestSlapgridArgumentTuple(SlapgridInitialization):
slapgrid_object
=
parser
(
*
self
.
default_arg_tuple
)[
0
]
slapgrid_object
=
parser
(
*
self
.
default_arg_tuple
)[
0
]
self
.
assertFalse
(
slapgrid_object
.
develop
)
self
.
assertFalse
(
slapgrid_object
.
develop
)
def
test_force_periodicity_if_periodicity_not_given
(
self
):
"""
Check if not giving --maximum-periodicity triggers "force_periodicity"
option to be false.
"""
parser
=
parseArgumentTupleAndReturnSlapgridObject
slapgrid_object
=
parser
(
*
self
.
default_arg_tuple
)[
0
]
self
.
assertFalse
(
slapgrid_object
.
force_periodicity
)
def
test_force_periodicity_if_periodicity_given
(
self
):
"""
Check if giving --maximum-periodicity triggers "force_periodicity" option.
"""
parser
=
parseArgumentTupleAndReturnSlapgridObject
slapgrid_object
=
parser
(
'--maximum-periodicity'
,
'40'
,
*
self
.
default_arg_tuple
)[
0
]
self
.
assertTrue
(
slapgrid_object
.
force_periodicity
)
class
TestSlapgridConfigurationFile
(
SlapgridInitialization
):
def
test_upload_binary_cache_blacklist
(
self
):
def
test_upload_binary_cache_blacklist
(
self
):
"""
"""
Check if giving --upload-to-binary-cache-url-blacklist triggers option.
Check if giving --upload-to-binary-cache-url-blacklist triggers option.
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment