Commit a403cff3 authored by Alain Takoudjou's avatar Alain Takoudjou Committed by Alain Takoudjou

Introduce new garbage collector for instances ignored by buildout

Buildout cannot request destroy sub instances when it is removed from
instance parameters by the user. Ex: request a cluster of KVM, with
kvm1 and kvm2 instances, edit parameters and remove kvm2 in the list.
Buildout will just ignore that instance and it will be removed only when
the hosting subscription will be destroyed.

                 -- kvm1
root instance --|
                 -X- kvm2

So when processing a partition, we send to master the list of requested sub
instance(s). If the previous list was ['kvm1', 'kvm2'], then after remove kvm2
the list will be ['kvm1'] (for the root instance). Commit this list to the
master will unlink between root instance and kvm2 (predecessor categorie).

A new alarm will search for unlinked instances: Instance which has no parent,
and destroy them (also destroy the sub tree).
parent 4675c289
...@@ -1389,6 +1389,62 @@ class TestSlapOSGarbageCollectDestroyedRootTreeAlarm(testSlapOSMixin): ...@@ -1389,6 +1389,62 @@ class TestSlapOSGarbageCollectDestroyedRootTreeAlarm(testSlapOSMixin):
self.assertEqual('validated', self.assertEqual('validated',
self.requested_software_instance.getValidationState()) self.requested_software_instance.getValidationState())
def test_Instance_tryToGarbageCollect_unlinked_predecessor(self):
self.requested_software_instance.edit(predecessor_list=[])
self.hosting_subscription.archive()
self.portal.portal_workflow._jumpToStateFor(self.hosting_subscription,
'destroy_requested')
self.portal.portal_workflow._jumpToStateFor(self.software_instance,
'destroy_requested')
self.tic()
self.requested_software_instance.Instance_tryToGarbageCollect()
self.tic()
self.assertEqual('destroy_requested',
self.requested_software_instance.getSlapState())
self.assertEqual('validated',
self.requested_software_instance.getValidationState())
def test_Instance_tryToGarbageCollect_destroy_unlinked_with_child(self):
instance_kw = dict(software_release=self.generateNewSoftwareReleaseUrl(),
software_type=self.generateNewSoftwareType(),
instance_xml=self.generateSafeXml(),
sla_xml=self.generateSafeXml(),
shared=False,
software_title='Sub Instance',
state='started'
)
self.requested_software_instance.requestInstance(**instance_kw)
sub_instance = self.requested_software_instance.getPredecessorValue()
self.assertNotEqual(sub_instance, None)
self.requested_software_instance.edit(predecessor_list=[])
self.hosting_subscription.archive()
self.portal.portal_workflow._jumpToStateFor(self.hosting_subscription,
'destroy_requested')
self.portal.portal_workflow._jumpToStateFor(self.software_instance,
'destroy_requested')
self.tic()
self.requested_software_instance.Instance_tryToGarbageCollect()
self.tic()
self.assertEqual('destroy_requested',
self.requested_software_instance.getSlapState())
self.assertEqual('validated',
self.requested_software_instance.getValidationState())
self.assertEqual(self.requested_software_instance.getPredecessorValue(),
None)
self.assertEqual(sub_instance.getSlapState(), 'start_requested')
sub_instance.Instance_tryToGarbageCollect()
self.tic()
self.assertEqual(sub_instance.getSlapState(), 'destroy_requested')
self.assertEqual(sub_instance.getValidationState(), 'validated')
def _simulateInstance_tryToGarbageCollect(self): def _simulateInstance_tryToGarbageCollect(self):
script_name = 'Instance_tryToGarbageCollect' script_name = 'Instance_tryToGarbageCollect'
if script_name in self.portal.portal_skins.custom.objectIds(): if script_name in self.portal.portal_skins.custom.objectIds():
...@@ -1961,6 +2017,187 @@ portal_workflow.doActionFor(context, action='edit_action', comment='Visited by I ...@@ -1961,6 +2017,187 @@ portal_workflow.doActionFor(context, action='edit_action', comment='Visited by I
'Visited by Instance_tryToGarbageCollectNonAllocatedRootTree', 'Visited by Instance_tryToGarbageCollectNonAllocatedRootTree',
instance.workflow_history['edit_workflow'][-1]['comment']) instance.workflow_history['edit_workflow'][-1]['comment'])
class TestSlapOSGarbageCollectUnlinkedInstanceAlarm(testSlapOSMixin):
def createInstance(self):
hosting_subscription = self.portal.hosting_subscription_module\
.template_hosting_subscription.Base_createCloneDocument(batch_mode=1)
hosting_subscription.validate()
hosting_subscription.edit(
title=self.generateNewSoftwareTitle(),
reference="TESTHS-%s" % self.generateNewId(),
)
request_kw = dict(
software_release=\
self.generateNewSoftwareReleaseUrl(),
software_type=self.generateNewSoftwareType(),
instance_xml=self.generateSafeXml(),
sla_xml=self.generateSafeXml(),
shared=False,
software_title=hosting_subscription.getTitle(),
state='started'
)
hosting_subscription.requestStart(**request_kw)
hosting_subscription.requestInstance(**request_kw)
self.hosting_subscription = hosting_subscription
instance = hosting_subscription.getPredecessorValue()
return instance
def createComputerPartition(self):
computer = self.portal.computer_module\
.template_computer.Base_createCloneDocument(batch_mode=1)
computer.validate()
computer.edit(
title=self.generateNewSoftwareTitle(),
reference="TESTCOMP-%s" % self.generateNewId(),
)
partition = computer.newContent(portal_type="Computer Partition")
return partition
def doRequestInstance(self, instance, title, slave=False):
instance_kw = dict(software_release=self.generateNewSoftwareReleaseUrl(),
software_type=self.generateNewSoftwareType(),
instance_xml=self.generateSafeXml(),
sla_xml=self.generateSafeXml(),
shared=slave,
software_title=title,
state='started'
)
instance.requestInstance(**instance_kw)
self.tic()
sub_instance = instance.getPredecessorValue()
partition = self.createComputerPartition()
sub_instance.edit(aggregate_value=partition)
self.tic()
self.assertEqual(self.hosting_subscription.getRelativeUrl(),
sub_instance.getSpecialise())
return sub_instance
def _simulateSoftwareInstance_tryToGarbageUnlinkedInstance(self):
script_name = 'SoftwareInstance_tryToGarbageUnlinkedInstance'
if script_name in self.portal.portal_skins.custom.objectIds():
raise ValueError('Precondition failed: %s exists in custom' % script_name)
createZODBPythonScript(self.portal.portal_skins.custom,
script_name,
'*args, **kwargs',
'# Script body\n'
"""portal_workflow = context.portal_workflow
portal_workflow.doActionFor(context, action='edit_action', comment='Visited by SoftwareInstance_tryToGarbageUnlinkedInstance') """ )
transaction.commit()
def _dropSoftwareInstance_tryToGarbageUnlinkedInstance(self):
script_name = 'SoftwareInstance_tryToGarbageUnlinkedInstance'
if script_name in self.portal.portal_skins.custom.objectIds():
self.portal.portal_skins.custom.manage_delObjects(script_name)
transaction.commit()
def test_SoftwareInstance_tryToGarbageUnlinkedInstance(self):
instance = self.createInstance()
partition = self.createComputerPartition()
instance.edit(aggregate_value=partition)
self.tic()
instance0 = self.doRequestInstance(instance, 'instance0')
self.assertEqual(instance0.getPredecessorRelatedTitle(), instance.getTitle())
# Remove predecessor link
instance.edit(predecessor_list=[])
self.tic()
self.assertEqual(instance0.getPredecessorRelatedTitle(), None)
instance0.SoftwareInstance_tryToGarbageUnlinkedInstance()
self.tic()
self.assertEqual(instance0.getSlapState(), 'destroy_requested')
def test_SoftwareInstance_tryToGarbageUnlinkedInstance_hosting_destroyed(self):
instance = self.createInstance()
partition = self.createComputerPartition()
instance.edit(aggregate_value=partition)
self.tic()
instance0 = self.doRequestInstance(instance, 'instance0')
instance.edit(predecessor_list=[])
self.tic()
self.hosting_subscription.archive()
self.portal.portal_workflow._jumpToStateFor(self.hosting_subscription,
'destroy_requested')
self.portal.portal_workflow._jumpToStateFor(instance, 'destroy_requested')
self.tic()
instance0.SoftwareInstance_tryToGarbageUnlinkedInstance()
self.tic()
# Will not be destroyed by this script
self.assertEqual(instance0.getSlapState(), 'start_requested')
def test_SoftwareInstance_tryToGarbageUnlinkedInstance_will_unlink_children(self):
instance = self.createInstance()
partition = self.createComputerPartition()
instance.edit(aggregate_value=partition)
self.tic()
instance0 = self.doRequestInstance(instance, 'instance0')
instance_instance0 = self.doRequestInstance(instance0, 'Subinstance0')
self.assertEqual(instance_instance0.getPredecessorRelatedTitle(),
'instance0')
instance.edit(predecessor_list=[])
self.tic()
self.assertEqual(instance0.getPredecessorRelatedTitle(), None)
instance0.SoftwareInstance_tryToGarbageUnlinkedInstance()
self.tic()
self.assertEqual(instance0.getSlapState(), 'destroy_requested')
self.assertEqual(instance_instance0.getSlapState(), 'start_requested')
# Link of child removed
self.assertEqual(instance_instance0.getPredecessorRelatedTitle(), None)
def test_alarm_search_inlinked_instance(self):
instance = self.createInstance()
partition = self.createComputerPartition()
instance.edit(aggregate_value=partition)
self.tic()
instance0 = self.doRequestInstance(instance, 'instance0')
self.assertEqual(instance.getPredecessorReference(),
instance0.getReference())
self._simulateSoftwareInstance_tryToGarbageUnlinkedInstance()
try:
self.portal.portal_alarms.slapos_garbage_collect_destroy_unlinked_instance.activeSense()
self.tic()
finally:
self._dropSoftwareInstance_tryToGarbageUnlinkedInstance()
self.assertNotEqual(
'Visited by SoftwareInstance_tryToGarbageUnlinkedInstance',
instance0.workflow_history['edit_workflow'][-1]['comment'])
# Remove predecessor link
instance.edit(predecessor_list=[])
self._simulateSoftwareInstance_tryToGarbageUnlinkedInstance()
self.tic()
try:
self.portal.portal_alarms.slapos_garbage_collect_destroy_unlinked_instance.activeSense()
self.tic()
finally:
self._dropSoftwareInstance_tryToGarbageUnlinkedInstance()
self.assertEqual(
'Visited by SoftwareInstance_tryToGarbageUnlinkedInstance',
instance0.workflow_history['edit_workflow'][-1]['comment'])
def test_alarm_search_inlinked_instance_slave(self):
instance = self.createInstance()
partition = self.createComputerPartition()
instance.edit(aggregate_value=partition)
self.tic()
slave_instance0 = self.doRequestInstance(instance, 'slaveInstance0', True)
self.assertEqual(instance.getPredecessorTitle(), 'slaveInstance0')
self._simulateSoftwareInstance_tryToGarbageUnlinkedInstance()
instance.edit(predecessor_list=[])
self.tic()
try:
self.portal.portal_alarms.slapos_garbage_collect_destroy_unlinked_instance.activeSense()
self.tic()
finally:
self._dropSoftwareInstance_tryToGarbageUnlinkedInstance()
self.assertEqual(
'Visited by SoftwareInstance_tryToGarbageUnlinkedInstance',
slave_instance0.workflow_history['edit_workflow'][-1]['comment'])
class TestSlapOSInvalidateDestroyedInstance(testSlapOSMixin): class TestSlapOSInvalidateDestroyedInstance(testSlapOSMixin):
def createSoftwareInstance(self): def createSoftwareInstance(self):
......
...@@ -1681,6 +1681,123 @@ class TestSlapOSSlapToolInstanceAccess(TestSlapOSSlapToolMixin): ...@@ -1681,6 +1681,123 @@ class TestSlapOSSlapToolInstanceAccess(TestSlapOSSlapToolMixin):
if os.path.exists(self.instance_request_simulator): if os.path.exists(self.instance_request_simulator):
os.unlink(self.instance_request_simulator) os.unlink(self.instance_request_simulator)
def test_updateInstancePredecessorList(self):
self._makeComplexComputer()
partition_id = self.start_requested_software_instance.getAggregateValue(
portal_type='Computer Partition').getReference()
self.login(self.start_requested_software_instance.getReference())
# Atach two software instances
instance_kw = dict(
software_release='http://a.release',
software_type='type',
instance_xml=self.generateSafeXml(),
sla_xml=self.generateSafeXml(),
shared=False,
software_title='Instance0',
state='started'
)
self.start_requested_software_instance.requestInstance(**instance_kw)
instance_kw['software_title'] = 'Instance1'
self.start_requested_software_instance.requestInstance(**instance_kw)
self.tic()
self.assertEqual(len(self.start_requested_software_instance.getPredecessorList()), 2)
self.assertSameSet(['Instance0', 'Instance1'],
self.start_requested_software_instance.getPredecessorTitleList())
# Update with no changes
instance_list_xml = """
<marshal>
<list id="i2"><string>Instance0</string><string>Instance1</string></list>
</marshal>"""
self.portal_slap.updateComputerPartitionRelatedInstanceList(
computer_id=self.computer_id,
computer_partition_id=partition_id,
instance_reference_xml=instance_list_xml)
self.tic()
self.assertSameSet(['Instance0', 'Instance1'],
self.start_requested_software_instance.getPredecessorTitleList())
# Update Instance0 was not requested
instance_list_xml = """
<marshal>
<list id="i2"><string>Instance1</string></list>
</marshal>"""
self.portal_slap.updateComputerPartitionRelatedInstanceList(
computer_id=self.computer_id,
computer_partition_id=partition_id,
instance_reference_xml=instance_list_xml)
self.tic()
self.assertSameSet(['Instance1'],
self.start_requested_software_instance.getPredecessorTitleList())
def test_updateInstancePredecessorList_one_child(self):
self._makeComplexComputer()
partition_id = self.start_requested_software_instance.getAggregateValue(
portal_type='Computer Partition').getReference()
self.login(self.start_requested_software_instance.getReference())
# Atach one software instance
instance_kw = dict(
software_release='http://a.release',
software_type='type',
instance_xml=self.generateSafeXml(),
sla_xml=self.generateSafeXml(),
shared=False,
software_title='Instance0',
state='started'
)
self.start_requested_software_instance.requestInstance(**instance_kw)
self.tic()
self.assertEqual(len(self.start_requested_software_instance.getPredecessorList()), 1)
self.assertSameSet(['Instance0'],
self.start_requested_software_instance.getPredecessorTitleList())
instance_list_xml = '<marshal><list id="i2" /></marshal>'
self.portal_slap.updateComputerPartitionRelatedInstanceList(
computer_id=self.computer_id,
computer_partition_id=partition_id,
instance_reference_xml=instance_list_xml)
self.tic()
self.assertEqual([],
self.start_requested_software_instance.getPredecessorTitleList())
def test_updateInstancePredecessorList_no_child(self):
self._makeComplexComputer()
partition_id = self.start_requested_software_instance.getAggregateValue(
portal_type='Computer Partition').getReference()
self.login(self.start_requested_software_instance.getReference())
self.assertEqual([],
self.start_requested_software_instance.getPredecessorTitleList())
instance_list_xml = '<marshal><list id="i2" /></marshal>'
self.portal_slap.updateComputerPartitionRelatedInstanceList(
computer_id=self.computer_id,
computer_partition_id=partition_id,
instance_reference_xml=instance_list_xml)
self.tic()
self.assertEqual([],
self.start_requested_software_instance.getPredecessorTitleList())
# Try with something that doesn't exist
instance_list_xml = """
<marshal>
<list id="i2"><string>instance0</string></list>
</marshal>"""
self.portal_slap.updateComputerPartitionRelatedInstanceList(
computer_id=self.computer_id,
computer_partition_id=partition_id,
instance_reference_xml=instance_list_xml)
self.tic()
self.assertEqual([],
self.start_requested_software_instance.getPredecessorTitleList())
def test_availableComputerPartition(self): def test_availableComputerPartition(self):
self._makeComplexComputer() self._makeComplexComputer()
partition_id = self.start_requested_software_instance.getAggregateValue( partition_id = self.start_requested_software_instance.getAggregateValue(
......
...@@ -518,6 +518,18 @@ class SlapTool(BaseTool): ...@@ -518,6 +518,18 @@ class SlapTool(BaseTool):
connection_xml, connection_xml,
slave_reference) slave_reference)
security.declareProtected(Permissions.AccessContentsInformation,
'updateComputerPartitionRelatedInstanceList')
def updateComputerPartitionRelatedInstanceList(self, computer_id,
computer_partition_id,
instance_reference_xml):
"""
Update Software Instance predecessor list
"""
return self._updateComputerPartitionRelatedInstanceList(computer_id,
computer_partition_id,
instance_reference_xml)
security.declareProtected(Permissions.AccessContentsInformation, security.declareProtected(Permissions.AccessContentsInformation,
'supplySupply') 'supplySupply')
def supplySupply(self, url, computer_id, state='available'): def supplySupply(self, url, computer_id, state='available'):
...@@ -1365,6 +1377,44 @@ class SlapTool(BaseTool): ...@@ -1365,6 +1377,44 @@ class SlapTool(BaseTool):
software_instance._instance_guid = instance_guid software_instance._instance_guid = instance_guid
return xml_marshaller.xml_marshaller.dumps(software_instance) return xml_marshaller.xml_marshaller.dumps(software_instance)
@UnrestrictedMethod
def _updateComputerPartitionRelatedInstanceList(self, computer_id,
computer_partition_id, instance_reference_xml):
"""
Update Software Instance predecessor list to match the given list. If one
instance was not requested by this computer partition, it should be removed
in the predecessor_list of this instance.
Once the link is removed, this instance will be trashed by Garbage Collect!
instance_reference_xml contain list of title of sub-instances requested by
this instance.
"""
software_instance_document = self.\
_getSoftwareInstanceForComputerPartition(computer_id,
computer_partition_id)
cache_reference = '%s-PREDLIST' % software_instance_document.getReference()
if self._getLastData(cache_reference) != instance_reference_xml:
instance_reference_list = xml_marshaller.xml_marshaller.loads(
instance_reference_xml)
current_predecessor_list = software_instance_document.getPredecessorValueList(
portal_type=['Software Instance', 'Slave Instance'])
current_predecessor_title_list = [i.getTitle() for i in
current_predecessor_list]
# If there are items to remove
if list(set(current_predecessor_title_list).difference(instance_reference_list)) != []:
predecessor_list = [instance.getRelativeUrl() for instance in
current_predecessor_list if instance.getTitle()
in instance_reference_list]
LOG('SlapTool', INFO, '%s, %s: Updating predecessor list to %s' % (
computer_id, computer_partition_id, predecessor_list), error=False)
software_instance_document.edit(predecessor_list=predecessor_list,
comment='predecessor_list edited to unlink non commited instances')
self._storeLastData(cache_reference, instance_reference_xml)
#################################################### ####################################################
# Internals methods # Internals methods
#################################################### ####################################################
......
...@@ -50,6 +50,7 @@ from lxml import etree ...@@ -50,6 +50,7 @@ from lxml import etree
from slapos.slap.slap import NotFoundError from slapos.slap.slap import NotFoundError
from slapos.slap.slap import ServerError from slapos.slap.slap import ServerError
from slapos.slap.slap import COMPUTER_PARTITION_REQUEST_LIST_TEMPLATE_FILENAME
from slapos.util import mkdir_p, chownDirectory, string_to_boolean from slapos.util import mkdir_p, chownDirectory, string_to_boolean
from slapos.grid.exception import BuildoutFailedError from slapos.grid.exception import BuildoutFailedError
from slapos.grid.SlapObject import Software, Partition from slapos.grid.SlapObject import Software, Partition
...@@ -666,6 +667,19 @@ stderr_logfile_backups=1 ...@@ -666,6 +667,19 @@ stderr_logfile_backups=1
if not promise_present: if not promise_present:
self.logger.info("No promise.") self.logger.info("No promise.")
def _endInstallationTransaction(self, computer_partition):
partition_id = computer_partition.getId()
transaction_file_name = COMPUTER_PARTITION_REQUEST_LIST_TEMPLATE_FILENAME % partition_id
transaction_file_path = os.path.join(self.instance_root,
partition_id,
transaction_file_name)
if os.path.exists(transaction_file_path):
with open(transaction_file_path, 'r') as tf:
computer_partition.setComputerPartitionRelatedInstanceList(
[reference for reference in tf.read().split('\n') if reference]
)
def _addFirewallRule(self, rule_command): def _addFirewallRule(self, rule_command):
""" """
""" """
...@@ -904,6 +918,14 @@ stderr_logfile_backups=1 ...@@ -904,6 +918,14 @@ stderr_logfile_backups=1
self.logger.debug('Check if %s requires processing...' % computer_partition_id) self.logger.debug('Check if %s requires processing...' % computer_partition_id)
instance_path = os.path.join(self.instance_root, computer_partition_id) instance_path = os.path.join(self.instance_root, computer_partition_id)
os.environ['SLAPGRID_INSTANCE_ROOT'] = self.instance_root
# Check if transaction file of this partition exists, if the file was created,
# remove it so it will be generate with this new transaction
transaction_file_name = COMPUTER_PARTITION_REQUEST_LIST_TEMPLATE_FILENAME % computer_partition_id
transaction_file_path = os.path.join(instance_path, transaction_file_name)
if os.path.exists(transaction_file_path):
os.unlink(transaction_file_path)
# Try to get partition timestamp (last modification date) # Try to get partition timestamp (last modification date)
timestamp_path = os.path.join( timestamp_path = os.path.join(
...@@ -1027,6 +1049,7 @@ stderr_logfile_backups=1 ...@@ -1027,6 +1049,7 @@ stderr_logfile_backups=1
local_partition.install() local_partition.install()
computer_partition.available() computer_partition.available()
local_partition.start() local_partition.start()
self._endInstallationTransaction(computer_partition)
if self.firewall_conf: if self.firewall_conf:
self._setupComputerPartitionFirewall(computer_partition, self._setupComputerPartitionFirewall(computer_partition,
partition_ip_list) partition_ip_list)
......
...@@ -337,6 +337,14 @@ class IComputerPartition(IBuildoutController, IRequester): ...@@ -337,6 +337,14 @@ class IComputerPartition(IBuildoutController, IRequester):
computer partition. computer partition.
""" """
def setComputerPartitionRelatedInstanceList(instance_reference_list):
"""
Set relation between this Instance and all his children.
instance_reference_list -- list of instances requested by this Computer
Partition.
"""
class IComputer(Interface): class IComputer(Interface):
""" """
Computer interface specification Computer interface specification
......
...@@ -36,6 +36,7 @@ __all__ = ["slap", "ComputerPartition", "Computer", "SoftwareRelease", ...@@ -36,6 +36,7 @@ __all__ = ["slap", "ComputerPartition", "Computer", "SoftwareRelease",
"Supply", "OpenOrder", "NotFoundError", "Supply", "OpenOrder", "NotFoundError",
"ResourceNotReady", "ServerError", "ConnectionError"] "ResourceNotReady", "ServerError", "ConnectionError"]
import os
import json import json
import logging import logging
import re import re
...@@ -67,6 +68,7 @@ fallback_logger.addHandler(fallback_handler) ...@@ -67,6 +68,7 @@ fallback_logger.addHandler(fallback_handler)
DEFAULT_SOFTWARE_TYPE = 'RootSoftwareInstance' DEFAULT_SOFTWARE_TYPE = 'RootSoftwareInstance'
COMPUTER_PARTITION_REQUEST_LIST_TEMPLATE_FILENAME = '.slapos-request-transaction-%s'
class SlapDocument: class SlapDocument:
def __init__(self, connection_helper=None, hateoas_navigator=None): def __init__(self, connection_helper=None, hateoas_navigator=None):
...@@ -81,6 +83,7 @@ class SlapRequester(SlapDocument): ...@@ -81,6 +83,7 @@ class SlapRequester(SlapDocument):
""" """
Abstract class that allow to factor method for subclasses that use "request()" Abstract class that allow to factor method for subclasses that use "request()"
""" """
def _requestComputerPartition(self, request_dict): def _requestComputerPartition(self, request_dict):
try: try:
xml = self._connection_helper.POST('requestComputerPartition', data=request_dict) xml = self._connection_helper.POST('requestComputerPartition', data=request_dict)
...@@ -406,9 +409,38 @@ class ComputerPartition(SlapRequester): ...@@ -406,9 +409,38 @@ class ComputerPartition(SlapRequester):
self._partition_id = partition_id self._partition_id = partition_id
self._request_dict = request_dict self._request_dict = request_dict
# Just create an empty file (for nothing requested yet)
self._updateTransactionFile(partition_reference=None)
def __getinitargs__(self): def __getinitargs__(self):
return (self._computer_id, self._partition_id, ) return (self._computer_id, self._partition_id, )
def _updateTransactionFile(self, partition_reference=None):
"""
Store reference to all Instances requested by this Computer Parition
"""
# Environ variable set by Slapgrid while processing this partition
instance_root = os.environ.get('SLAPGRID_INSTANCE_ROOT', '')
if not instance_root or not self._partition_id:
return
transaction_file_name = COMPUTER_PARTITION_REQUEST_LIST_TEMPLATE_FILENAME % self._partition_id
transaction_file_path = os.path.join(instance_root, self._partition_id,
transaction_file_name)
try:
if partition_reference is None:
if os.access(os.path.join(instance_root, self._partition_id), os.W_OK):
if os.path.exists(transaction_file_path):
return
transac_file = open(transaction_file_path, 'w')
transac_file.close()
else:
with open(transaction_file_path, 'a') as transac_file:
transac_file.write('%s\n' % partition_reference)
except OSError:
return
def request(self, software_release, software_type, partition_reference, def request(self, software_release, software_type, partition_reference,
shared=False, partition_parameter_kw=None, filter_kw=None, shared=False, partition_parameter_kw=None, filter_kw=None,
state=None): state=None):
...@@ -440,6 +472,7 @@ class ComputerPartition(SlapRequester): ...@@ -440,6 +472,7 @@ class ComputerPartition(SlapRequester):
'filter_xml': xml_marshaller.dumps(filter_kw), 'filter_xml': xml_marshaller.dumps(filter_kw),
'state': xml_marshaller.dumps(state), 'state': xml_marshaller.dumps(state),
} }
self._updateTransactionFile(partition_reference)
return self._requestComputerPartition(request_dict) return self._requestComputerPartition(request_dict)
def building(self): def building(self):
...@@ -635,6 +668,15 @@ class ComputerPartition(SlapRequester): ...@@ -635,6 +668,15 @@ class ComputerPartition(SlapRequester):
) )
return xml_marshaller.loads(xml) return xml_marshaller.loads(xml)
def setComputerPartitionRelatedInstanceList(self, instance_reference_list):
self._connection_helper.POST('updateComputerPartitionRelatedInstanceList',
data={
'computer_id': self._computer_id,
'computer_partition_id': self._partition_id,
'instance_reference_xml': xml_marshaller.dumps(instance_reference_list)
}
)
def _addIpv6Brackets(url): def _addIpv6Brackets(url):
# if master_url contains an ipv6 without bracket, add it # if master_url contains an ipv6 without bracket, add it
# Note that this is mostly to limit specific issues with # Note that this is mostly to limit specific issues with
......
...@@ -29,6 +29,7 @@ import logging ...@@ -29,6 +29,7 @@ import logging
import os import os
import unittest import unittest
import urlparse import urlparse
import tempfile
import httmock import httmock
...@@ -53,6 +54,8 @@ class SlapMixin(unittest.TestCase): ...@@ -53,6 +54,8 @@ class SlapMixin(unittest.TestCase):
print 'Testing against SLAP server %r' % self.server_url print 'Testing against SLAP server %r' % self.server_url
self.slap = slapos.slap.slap() self.slap = slapos.slap.slap()
self.partition_id = 'PARTITION_01' self.partition_id = 'PARTITION_01'
if os.environ.has_key('SLAPGRID_INSTANCE_ROOT'):
del os.environ['SLAPGRID_INSTANCE_ROOT']
def tearDown(self): def tearDown(self):
pass pass
...@@ -786,6 +789,84 @@ class TestComputerPartition(SlapMixin): ...@@ -786,6 +789,84 @@ class TestComputerPartition(SlapMixin):
# request was done works correctly # request was done works correctly
self.assertEqual(requested_partition_id, requested_partition.getId()) self.assertEqual(requested_partition_id, requested_partition.getId())
def test_request_with_slapgrid_request_transaction(self):
from slapos.slap.slap import COMPUTER_PARTITION_REQUEST_LIST_TEMPLATE_FILENAME
partition_id = 'PARTITION_01'
instance_root = tempfile.mkdtemp()
partition_root = os.path.join(instance_root, partition_id)
os.mkdir(partition_root)
os.environ['SLAPGRID_INSTANCE_ROOT'] = instance_root
transaction_file_name = COMPUTER_PARTITION_REQUEST_LIST_TEMPLATE_FILENAME % partition_id
transaction_file_path = os.path.join(partition_root, transaction_file_name)
def handler(url, req):
qs = urlparse.parse_qs(url.query)
if (url.path == '/registerComputerPartition'
and 'computer_reference' in qs
and 'computer_partition_reference' in qs):
slap_partition = slapos.slap.ComputerPartition(
qs['computer_reference'][0],
qs['computer_partition_reference'][0])
return {
'status_code': 200,
'content': xml_marshaller.xml_marshaller.dumps(slap_partition)
}
elif (url.path == '/getComputerInformation'
and 'computer_id' in qs):
slap_computer = slapos.slap.Computer(qs['computer_id'][0])
slap_computer._software_release_list = []
slap_partition = slapos.slap.ComputerPartition(
qs['computer_id'][0],
partition_id)
slap_computer._computer_partition_list = [slap_partition]
return {
'status_code': 200,
'content': xml_marshaller.xml_marshaller.dumps(slap_computer)
}
elif url.path == '/requestComputerPartition':
raise RequestWasCalled
else:
return {
'status_code': 404
}
with httmock.HTTMock(handler):
self.computer_guid = self._getTestComputerId()
self.slap = slapos.slap.slap()
self.slap.initializeConnection(self.server_url)
computer_partition = self.slap.registerComputerPartition(
self.computer_guid, partition_id)
self.assertTrue(os.path.exists(transaction_file_path))
with open(transaction_file_path, 'r') as f:
content = f.read()
self.assertEqual(content, '')
self.assertRaises(RequestWasCalled,
computer_partition.request,
'http://server/new/' + self._getTestComputerId(),
'software_type', 'myref')
self.assertTrue(os.path.exists(transaction_file_path))
with open(transaction_file_path, 'r') as f:
content_list = f.read().strip().split('\n')
self.assertEqual(content_list, ['myref'])
# Not override
computer_partition = self.slap.registerComputerPartition(
self.computer_guid, partition_id)
self.assertTrue(os.path.exists(transaction_file_path))
with open(transaction_file_path, 'r') as f:
content_list = f.read().strip().split('\n')
self.assertEqual(content_list, ['myref'])
# Request a second instance
self.assertRaises(RequestWasCalled,
computer_partition.request,
'http://server/new/' + self._getTestComputerId(),
'software_type', 'mysecondref')
with open(transaction_file_path, 'r') as f:
content_list = f.read().strip().split('\n')
self.assertEquals(list(set(content_list)), ['myref', 'mysecondref'])
def _test_new_computer_partition_state(self, state): def _test_new_computer_partition_state(self, state):
""" """
Helper method to automate assertions of failing states on new Computer Helper method to automate assertions of failing states on new Computer
......
...@@ -51,6 +51,7 @@ from slapos.grid.utils import md5digest ...@@ -51,6 +51,7 @@ from slapos.grid.utils import md5digest
from slapos.grid.watchdog import Watchdog from slapos.grid.watchdog import Watchdog
from slapos.grid import SlapObject from slapos.grid import SlapObject
from slapos.grid.SlapObject import WATCHDOG_MARK from slapos.grid.SlapObject import WATCHDOG_MARK
from slapos.slap.slap import COMPUTER_PARTITION_REQUEST_LIST_TEMPLATE_FILENAME
import slapos.grid.SlapObject import slapos.grid.SlapObject
import httmock import httmock
...@@ -107,6 +108,8 @@ class BasicMixin(object): ...@@ -107,6 +108,8 @@ class BasicMixin(object):
self._tempdir = tempfile.mkdtemp() self._tempdir = tempfile.mkdtemp()
self.software_root = os.path.join(self._tempdir, 'software') self.software_root = os.path.join(self._tempdir, 'software')
self.instance_root = os.path.join(self._tempdir, 'instance') self.instance_root = os.path.join(self._tempdir, 'instance')
if os.environ.has_key('SLAPGRID_INSTANCE_ROOT'):
del os.environ['SLAPGRID_INSTANCE_ROOT']
logging.basicConfig(level=logging.DEBUG) logging.basicConfig(level=logging.DEBUG)
self.setSlapgrid() self.setSlapgrid()
...@@ -338,6 +341,8 @@ class ComputerForTest(object): ...@@ -338,6 +341,8 @@ class ComputerForTest(object):
return {'status_code': 200} return {'status_code': 200}
if url.path == '/softwareInstanceBang': if url.path == '/softwareInstanceBang':
return {'status_code': 200} return {'status_code': 200}
if url.path == "/updateComputerPartitionRelatedInstanceList":
return {'status_code': 200}
if url.path == '/softwareInstanceError': if url.path == '/softwareInstanceError':
instance.error_log = '\n'.join( instance.error_log = '\n'.join(
[ [
...@@ -1647,16 +1652,21 @@ class TestSlapgridUsageReport(MasterMixin, unittest.TestCase): ...@@ -1647,16 +1652,21 @@ class TestSlapgridUsageReport(MasterMixin, unittest.TestCase):
# Then run usage report and see if it is still working # Then run usage report and see if it is still working
computer.sequence = [] computer.sequence = []
self.assertEqual(self.grid.agregateAndSendUsage(), slapgrid.SLAPGRID_SUCCESS) self.assertEqual(self.grid.agregateAndSendUsage(), slapgrid.SLAPGRID_SUCCESS)
# registerComputerPartition will create one more file:
from slapos.slap.slap import COMPUTER_PARTITION_REQUEST_LIST_TEMPLATE_FILENAME
request_list_file = COMPUTER_PARTITION_REQUEST_LIST_TEMPLATE_FILENAME % instance.name
self.assertInstanceDirectoryListEqual(['0']) self.assertInstanceDirectoryListEqual(['0'])
self.assertItemsEqual(os.listdir(instance.partition_path), self.assertItemsEqual(os.listdir(instance.partition_path),
['.slapgrid', '.0_wrapper.log', 'buildout.cfg', ['.slapgrid', '.0_wrapper.log', 'buildout.cfg',
'etc', 'software_release', 'worked', '.slapos-retention-lock-delay']) 'etc', 'software_release', 'worked',
'.slapos-retention-lock-delay', request_list_file])
wrapper_log = os.path.join(instance.partition_path, '.0_wrapper.log') wrapper_log = os.path.join(instance.partition_path, '.0_wrapper.log')
self.assertLogContent(wrapper_log, 'Working') self.assertLogContent(wrapper_log, 'Working')
self.assertInstanceDirectoryListEqual(['0']) self.assertInstanceDirectoryListEqual(['0'])
self.assertItemsEqual(os.listdir(instance.partition_path), self.assertItemsEqual(os.listdir(instance.partition_path),
['.slapgrid', '.0_wrapper.log', 'buildout.cfg', ['.slapgrid', '.0_wrapper.log', 'buildout.cfg',
'etc', 'software_release', 'worked', '.slapos-retention-lock-delay']) 'etc', 'software_release', 'worked',
'.slapos-retention-lock-delay', request_list_file])
wrapper_log = os.path.join(instance.partition_path, '.0_wrapper.log') wrapper_log = os.path.join(instance.partition_path, '.0_wrapper.log')
self.assertLogContent(wrapper_log, 'Working') self.assertLogContent(wrapper_log, 'Working')
self.assertEqual(computer.sequence, self.assertEqual(computer.sequence,
...@@ -2312,4 +2322,19 @@ class TestSlapgridCPWithFirewall(MasterMixin, unittest.TestCase): ...@@ -2312,4 +2322,19 @@ class TestSlapgridCPWithFirewall(MasterMixin, unittest.TestCase):
with open(rules_path, 'r') as frules: with open(rules_path, 'r') as frules:
rules_list = json.loads(frules.read()) rules_list = json.loads(frules.read())
self.checkRuleFromIpSource(ip, [source_ip[1]], rules_list) self.checkRuleFromIpSource(ip, [source_ip[1]], rules_list)
\ No newline at end of file class TestSlapgridCPWithTransaction(MasterMixin, unittest.TestCase):
def test_one_partition(self):
computer = ComputerForTest(self.software_root, self.instance_root)
with httmock.HTTMock(computer.request_handler):
instance = computer.instance_list[0]
partition = os.path.join(self.instance_root, '0')
request_list_file = os.path.join(partition,
COMPUTER_PARTITION_REQUEST_LIST_TEMPLATE_FILENAME % instance.name)
with open(request_list_file, 'w') as f:
f.write('some partition')
self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
self.assertInstanceDirectoryListEqual(['0'])
self.assertFalse(os.path.exists(request_list_file))
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment