Merge branch 'master' of https://git.erp5.org/repos/slapos.core into chownDirectory

CHANGES.txt

@@ -14,6 +14,7 @@ Major Improvements:
  * Dramatically speed up slapformat. [Cedric de Saint Martin]
  * Remove CONFIG_SITE env var from Buildout environment, fixing support of OpenSuse 12.x. [Cedric de Saint Martin]
  * RootSoftwareInstance is now the default software type. [Cedric de Saint Martin]
+ * Allow to use SlapOS Client for instances deployed in shared SlapOS Nodes. [Cedric de Saint Martin]
 
 Other fixes:
 
@@ -25,6 +26,7 @@ Other fixes:
  * Slapgrid: Check if SR is upload-blacklisted only if we have upload informations. [Cedric de Saint Martin]
  * Slapformat: Make sure everybody can read slapos configuration directory. [Cedric de Saint Martin]
  * Slapformat: Fix support of slapproxy. [Marco Mariani]
+ * Slapformat: slapos.xml backup: handle corrupted zip files. [Marco Mariani]
 
 
 0.35.1 (2013-02-18)

master/bt5/slapos_slap_tool/TestTemplateItem/testSlapOSSlapTool.py

@@ -1519,7 +1519,7 @@ class TestSlapOSSlapToolInstanceAccess(TestSlapOSSlapToolMixin):
     <unicode>created_at</unicode>
     <unicode>%(created_at)s</unicode>
     <unicode>text</unicode>
-    <unicode>#access instance correctly stopped</unicode>
+    <unicode>#access Instance correctly stopped</unicode>
     <unicode>user</unicode>
     <unicode>%(instance_guid)s</unicode>
   </dictionary>
@@ -1556,7 +1556,7 @@ class TestSlapOSSlapToolInstanceAccess(TestSlapOSSlapToolMixin):
     <unicode>created_at</unicode>
     <unicode>%(created_at)s</unicode>
     <unicode>text</unicode>
-    <unicode>#access instance correctly started</unicode>
+    <unicode>#access Instance correctly started</unicode>
     <unicode>user</unicode>
     <unicode>%(instance_guid)s</unicode>
   </dictionary>
@@ -2167,6 +2167,84 @@ class TestSlapOSSlapToolPersonAccess(TestSlapOSSlapToolMixin):
       if os.path.exists(self.instance_request_simulator):
         os.unlink(self.instance_request_simulator)
 
+  def test_request_allocated_instance(self):
+    self.tic()
+    self.person.edit(
+      default_email_coordinate_text="%s@example.org" % self.person.getReference(),
+      career_role='member',
+    )
+    self._makeComplexComputer(person=self.person)
+    self.start_requested_software_instance.updateLocalRolesOnSecurityGroups()
+    self.tic()
+    self.login(self.person_reference)
+    response = self.portal_slap.requestComputerPartition(
+      software_release=self.start_requested_software_instance.getUrlString(),
+      software_type=self.start_requested_software_instance.getSourceReference(),
+      partition_reference=self.start_requested_software_instance.getTitle(),
+      partition_parameter_xml='<marshal><dictionary id="i2"/></marshal>',
+      filter_xml='<marshal><dictionary id="i2"/></marshal>',
+      state='<marshal><string>started</string></marshal>',
+      shared_xml='<marshal><bool>0</bool></marshal>',
+      )
+    self.assertEqual(type(response), str)
+    # check returned XML
+    xml_fp = StringIO.StringIO()
+
+    xml.dom.ext.PrettyPrint(xml.dom.ext.reader.Sax.FromXml(response),
+        stream=xml_fp)
+    xml_fp.seek(0)
+    got_xml = xml_fp.read()
+    expected_xml = """\
+<?xml version='1.0' encoding='UTF-8'?>
+<marshal>
+  <object id='i2' module='slapos.slap.slap' class='SoftwareInstance'>
+    <tuple/>
+    <dictionary id='i3'>
+      <string>_connection_dict</string>
+      <dictionary id='i4'/>
+      <string>_instance_guid</string>
+      <string>%(instance_guid)s</string>
+      <string>_parameter_dict</string>
+      <dictionary id='i5'/>
+      <string>_requested_state</string>
+      <string>%(state)s</string>
+      <string>ip_list</string>
+      <list id='i6'>
+        <tuple>
+          <string/>
+          <string>%(ip)s</string>
+        </tuple>
+      </list>
+      <string>slap_computer_id</string>
+      <string>%(computer_id)s</string>
+      <string>slap_computer_partition_id</string>
+      <string>%(partition_id)s</string>
+      <string>slap_software_release_url</string>
+      <string>%(url_string)s</string>
+      <string>slap_software_type</string>
+      <string>%(type)s</string>
+      <string>slave_instance_list</string>
+      <list id='i7'/>
+      <string>timestamp</string>
+      <string>%(timestamp)s</string>
+    </dictionary>
+  </object>
+</marshal>
+""" % dict(
+    instance_guid=self.start_requested_software_instance.getReference(),
+    state="started",
+    url_string=self.start_requested_software_instance.getUrlString(),
+    type=self.start_requested_software_instance.getSourceReference(),
+    timestamp=int(self.start_requested_software_instance.getModificationDate()),
+    computer_id=self.computer_id,
+    partition_id=self.start_requested_software_instance.getAggregateId(),
+    ip=self.start_requested_software_instance.getAggregateValue()\
+               .getDefaultNetworkAddressIpAddress(),
+  )
+    self.assertEqual(expected_xml, got_xml,
+      '\n'.join([q for q in difflib.unified_diff(expected_xml.split('\n'),
+                                                 got_xml.split('\n'))]))
+
   def assertSupplySimulator(self, args, kwargs):
     stored = eval(open(self.computer_supply_simulator).read())
     # do the same translation magic as in workflow

master/bt5/slapos_slap_tool/bt/revision

-23
+25
\ No newline at end of file

master/bt5/slapos_web/SkinTemplateItem/portal_skins/vifib_hosting/Computer_viewAllocationScopeUpdateDialog/your_subject_list.xml

@@ -269,7 +269,7 @@
                 </item>
                 <item>
                     <key> <string>title</string> </key>
-                    <value> <string>Friends</string> </value>
+                    <value> <string>Friends (email)</string> </value>
                 </item>
                 <item>
                     <key> <string>unicode</string> </key>

master/bt5/slapos_web/SkinTemplateItem/portal_skins/vifib_hosting/Computer_viewConnectionInformationAsWeb/your_certificate.xml

@@ -259,7 +259,7 @@
                 </item>
                 <item>
                     <key> <string>title</string> </key>
-                    <value> <string>certificate.crt</string> </value>
+                    <value> <string>computer.crt</string> </value>
                 </item>
                 <item>
                     <key> <string>unicode</string> </key>

master/bt5/slapos_web/SkinTemplateItem/portal_skins/vifib_hosting/Computer_viewConnectionInformationAsWeb/your_key.xml

@@ -259,7 +259,7 @@
                 </item>
                 <item>
                     <key> <string>title</string> </key>
-                    <value> <string>key.key</string> </value>
+                    <value> <string>computer.key</string> </value>
                 </item>
                 <item>
                     <key> <string>unicode</string> </key>

master/bt5/slapos_web/bt/revision

-46
+48
\ No newline at end of file

master/product/Vifib/Tool/SlapTool.py

@@ -654,7 +654,6 @@ class SlapTool(BaseTool):
            computer_partition_document.getRelativeUrl())
 
     if software_instance is not None:
-      slap_partition._instance_guid = software_instance.getReference()
       # trick client side, that data has been synchronised already for given
       # document
       slap_partition._synced = True
@@ -677,6 +676,7 @@ class SlapTool(BaseTool):
         parameter_dict.pop('xml'))
       slap_partition._connection_dict = self._instanceXmlToDict(
         parameter_dict.pop('connection_xml'))
+      slap_partition._instance_guid = parameter_dict.pop('instance_guid')
       for slave_instance_dict in parameter_dict.get("slave_instance_list", []):
         if slave_instance_dict.has_key("connection_xml"):
           slave_instance_dict.update(self._instanceXmlToDict(
@@ -788,7 +788,6 @@ class SlapTool(BaseTool):
            computer_partition_document.getRelativeUrl())
 
     if software_instance is not None:
-      slap_partition._instance_guid = software_instance.getReference()
       state = software_instance.getSlapState()
       if state == "stop_requested":
         slap_partition._requested_state = 'stopped'
@@ -808,6 +807,7 @@ class SlapTool(BaseTool):
         parameter_dict.pop('xml'))
       slap_partition._connection_dict = self._instanceXmlToDict(
         parameter_dict.pop('connection_xml'))
+      slap_partition._instance_guid = parameter_dict.pop('instance_guid')
       for slave_instance_dict in parameter_dict.get("slave_instance_list", []):
         if slave_instance_dict.has_key("connection_xml"):
           slave_instance_dict.update(self._instanceXmlToDict(
@@ -975,8 +975,8 @@ class SlapTool(BaseTool):
         computer_partition_id)
     user = self.getPortalObject().portal_membership.getAuthenticatedMember()\
                                                    .getUserName()
-    self._logAccess(user, instance.getReference(), 
-                    '#access instance correctly started')
+    self._logAccess(user, instance.getReference(),
+                    '#access Instance correctly started')
 
   @convertToREST
   def _stoppedComputerPartition(self, computer_id, computer_partition_id):
@@ -988,8 +988,8 @@ class SlapTool(BaseTool):
         computer_partition_id)
     user = self.getPortalObject().portal_membership.getAuthenticatedMember()\
                                                    .getUserName()
-    self._logAccess(user, instance.getReference(), 
-                    '#access instance correctly stopped')
+    self._logAccess(user, instance.getReference(),
+                    '#access Instance correctly stopped')
 
   @convertToREST
   def _destroyedComputerPartition(self, computer_id, computer_partition_id):
@@ -1176,10 +1176,13 @@ class SlapTool(BaseTool):
           parameter_dict.pop('xml'))
         connection_xml = self._instanceXmlToDict(
           parameter_dict.pop('connection_xml'))
+        instance_guid = parameter_dict.pop('instance_guid')
 
         software_instance = SoftwareInstance(**parameter_dict)
         software_instance._parameter_dict = xml
         software_instance._connection_dict = connection_xml
+        software_instance._requested_state = state
+        software_instance._instance_guid = instance_guid
         return xml_marshaller.xml_marshaller.dumps(software_instance)
 
   ####################################################
@@ -1320,6 +1323,7 @@ class SlapTool(BaseTool):
           if (newtimestamp > timestamp):                                            
             timestamp = newtimestamp
     return {
+      'instance_guid': software_instance.getReference(),
       'xml': software_instance.getTextContent(),
       'connection_xml': software_instance.getConnectionXml(),
       'slap_computer_id': computer_partition.getParentValue().getReference(),

slapos-client.cfg.example

@@ -2,13 +2,13 @@
 master_url = https://slap.vifib.com/
 
 [slapconsole]
-# Put here retrieved certificate from vifib.
+# Put here retrieved certificate from SlapOS Master.
 # Beware: put certificate from YOUR account, not the one from your node.
-# You (as identified person from vifib) will request an instance, node your node.
+# You (as identified person from SlapOS Master) will request an instance, node your node.
 # Conclusion: node certificate != person certificate.
 cert_file = certificate file location coming from your slapos master account
 key_file = key file location coming from your slapos master account
-# Below are softwares supported by Vifib
+# Below are softwares maintained by slapos.org and contributors
 alias =
   apache_frontend http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
   dokuwiki http://git.erp5.org/gitweb/slapos.git/blob_plain/refs/tags/slapos-0.158:/software/dokuwiki/software.cfg
@@ -27,4 +27,4 @@ alias =
   slaposwebrunner http://git.erp5.org/gitweb/slapos.git/blob_plain/refs/tags/slapos-0.160:/software/slaprunner/software.cfg
   wordpress http://git.erp5.org/gitweb/slapos.git/blob_plain/refs/tags/slapos-0.153:/software/wordpress/software.cfg
   xwiki http://git.erp5.org/gitweb/slapos.git/blob_plain/refs/tags/slapos-0.46:/software/xwiki/software.cfg
-  zabbixagent http://git.erp5.org/gitweb/slapos.git/blob_plain/refs/tags/slapos-0.95:/software/zabbix-agent/software.cfg
+  zabbixagent http://git.erp5.org/gitweb/slapos.git/blob_plain/refs/tags/slapos-0.162:/software/zabbix-agent/software.cfg

slapos/format.py

@@ -42,12 +42,14 @@ import os
 import pwd
 import random
 import slapos.slap as slap
+import shutil
 import socket
 import struct
 import subprocess
 import sys
 import threading
 import time
+import traceback
 import zipfile
 
 import lxml.etree
@@ -298,7 +300,18 @@ class Computer(object):
           return
 
     if os.path.exists(path_to_xml):
-      self.backup_xml(path_to_archive, path_to_xml)
+      try:
+        self.backup_xml(path_to_archive, path_to_xml)
+      except:
+        # might be a corrupted zip file. let's move it out of the way and retry.
+        shutil.move(path_to_archive,
+                    path_to_archive+time.strftime('_broken_%Y%m%d-%H:%M'))
+        try:
+          self.backup_xml(path_to_archive, path_to_xml)
+        except:
+          # give up trying
+          logger.warning("Can't backup %s: %s" %
+                           (path_to_xml, traceback.format_exc()))
 
     with open(path_to_xml, 'wb') as fout:
       fout.write(new_pretty_xml)

slapos/slap/slap.py

@@ -77,6 +77,35 @@ class SlapDocument:
       # cause failures when accessing _connection_helper property.
       self._connection_helper = connection_helper
 
+class SlapRequester(SlapDocument):
+  """
+  Abstract class that allow to factor method for subclasses that use "request()"
+  """
+  def _requestComputerPartition(self, request_dict):
+    try:
+      self._connection_helper.POST('/requestComputerPartition', request_dict)
+    except ResourceNotReady:
+      return ComputerPartition(
+        request_dict=request_dict,
+        connection_helper=self._connection_helper,
+      )
+    xml = self._connection_helper.response.read()
+    software_instance = xml_marshaller.loads(xml)
+    computer_partition = ComputerPartition(
+      software_instance.slap_computer_id.encode('UTF-8'),
+      software_instance.slap_computer_partition_id.encode('UTF-8'),
+      connection_helper=self._connection_helper,
+    )
+    # Hack to give all object attributes to the ComputerPartition instance
+    computer_partition.__dict__ = software_instance.__dict__.copy()
+    # XXX not generic enough.
+    if xml_marshaller.loads(request_dict['shared_xml']):
+      computer_partition._synced = True
+      computer_partition._connection_dict = software_instance._connection_dict
+      computer_partition._parameter_dict = software_instance._parameter_dict
+    return computer_partition
+
+
 class SoftwareRelease(SlapDocument):
   """
   Contains Software Release information
@@ -182,7 +211,7 @@ class Supply(SlapDocument):
       raise NotFoundError("Computer %s has not been found by SlapOS Master."
           % computer_guid)
 
-class OpenOrder(SlapDocument):
+class OpenOrder(SlapRequester):
 
   zope.interface.implements(interface.IOpenOrder)
 
@@ -208,26 +237,7 @@ class OpenOrder(SlapDocument):
     else:
       # Let's enforce a default software type
       request_dict['software_type'] = DEFAULT_SOFTWARE_TYPE
-    try:
-      self._connection_helper.POST('/requestComputerPartition', request_dict)
-    except ResourceNotReady:
-      return ComputerPartition(
-        request_dict=request_dict,
-        connection_helper=self._connection_helper,
-      )
-    else:
-      xml = self._connection_helper.response.read()
-      software_instance = xml_marshaller.loads(xml)
-      computer_partition = ComputerPartition(
-        software_instance.slap_computer_id.encode('UTF-8'),
-        software_instance.slap_computer_partition_id.encode('UTF-8'),
-        connection_helper=self._connection_helper,
-      )
-      if shared:
-        computer_partition._synced = True
-        computer_partition._connection_dict = software_instance._connection_dict
-        computer_partition._parameter_dict = software_instance._parameter_dict
-      return computer_partition
+    return self._requestComputerPartition(request_dict)
 
   def requestComputer(self, computer_reference):
     """
@@ -321,57 +331,6 @@ class Computer(SlapDocument):
     xml = self._connection_helper.response.read()
     return xml_marshaller.loads(xml)
 
-def _syncComputerPartitionInformation(func):
-  """
-  Synchronize computer partition object with server information
-  """
-  def decorated(self, *args, **kw):
-    if getattr(self, '_synced', 0):
-      return func(self, *args, **kw)
-    if not self._computer_id:
-      # XXX Is it only in case of requesting instance?
-      raise ResourceNotReady("Instance is not ready yet.")
-    # XXX: This is a ugly way to keep backward compatibility,
-    # We should stablise slap library soon.
-    computer = self._connection_helper.getFullComputerInformation(self._computer_id)
-    found_computer_partition = None
-    for computer_partition in computer._computer_partition_list:
-      if computer_partition.getId() == self.getId():
-        found_computer_partition = computer_partition
-        break
-    if found_computer_partition is None:
-      raise NotFoundError("No information for partition %s" %
-          self.getId())
-    else:
-      for key, value in found_computer_partition.__dict__.items():
-        if isinstance(value, unicode):
-          # convert unicode to utf-8
-          setattr(self, key, value.encode('utf-8'))
-        if isinstance(value, dict):
-          new_dict = {}
-          for ink, inv in value.iteritems():
-            if isinstance(inv, (list, tuple)):
-              new_inv = []
-              for elt in inv:
-                if isinstance(elt, (list, tuple)):
-                  new_inv.append([x.encode('utf-8') for x in elt])
-                elif isinstance(elt, dict):
-                  new_inv.append(dict([(x.encode('utf-8'),
-                    y and y.encode("utf-8")) for x,y in elt.iteritems()]))
-                else:
-                  new_inv.append(elt.encode('utf-8'))
-              new_dict[ink.encode('utf-8')] = new_inv
-            elif inv is None:
-              new_dict[ink.encode('utf-8')] = None
-            else:
-              new_dict[ink.encode('utf-8')] = inv.encode('utf-8')
-          setattr(self, key, new_dict)
-        else:
-          setattr(self, key, value)
-    setattr(self, '_synced', True)
-    return func(self, *args, **kw)
-  return decorated
-
 
 class ComputerPartition(SlapDocument):
 
@@ -394,7 +353,6 @@ class ComputerPartition(SlapDocument):
   def __getinitargs__(self):
     return (self._computer_id, self._partition_id, )
 
-  @_syncComputerPartitionInformation
   def request(self, software_release, software_type, partition_reference,
               shared=False, partition_parameter_kw=None, filter_kw=None,
               state=None):
@@ -414,7 +372,8 @@ class ComputerPartition(SlapDocument):
     if software_type is None:
       software_type = DEFAULT_SOFTWARE_TYPE
 
-    request_dict = { 'computer_id': self._computer_id,
+    request_dict = {
+        'computer_id': self._computer_id,
         'computer_partition_id': self._partition_id,
         'software_release': software_release,
         'software_type': software_type,
@@ -424,29 +383,8 @@ class ComputerPartition(SlapDocument):
                                         partition_parameter_kw),
         'filter_xml': xml_marshaller.dumps(filter_kw),
         'state': xml_marshaller.dumps(state),
-      }
-    try:
-      self._connection_helper.POST('/requestComputerPartition', request_dict)
-    except ResourceNotReady:
-      return ComputerPartition(
-        request_dict=request_dict,
-        connection_helper=self._connection_helper,
-      )
-    else:
-      xml = self._connection_helper.response.read()
-      software_instance = xml_marshaller.loads(xml)
-      computer_partition = ComputerPartition(
-        software_instance.slap_computer_id.encode('UTF-8'),
-        software_instance.slap_computer_partition_id.encode('UTF-8'),
-        connection_helper=self._connection_helper,
-      )
-      if shared:
-        computer_partition._synced = True
-        computer_partition._connection_dict = getattr(software_instance,
-          '_connection_dict', None)
-        computer_partition._parameter_dict = getattr(software_instance,
-          '_parameter_dict', None)
-      return computer_partition
+    }
+    return self._requestComputerPartition(request_dict)
 
   def building(self):
     self._connection_helper.POST('/buildingComputerPartition', {
@@ -507,29 +445,24 @@ class ComputerPartition(SlapDocument):
       raise ResourceNotReady()
     return self._partition_id
 
-  @_syncComputerPartitionInformation
   def getInstanceGuid(self):
-    """Sync if not synced, then returns instance_guid"""
+    """Return instance_guid. Raise ResourceNotReady if it doesn't exist."""
     if not getattr(self, '_instance_guid', None):
       raise ResourceNotReady()
     return self._instance_guid
 
-  @_syncComputerPartitionInformation
   def getState(self):
-    """Sync if not synced, then returns _requested_state."""
+    """return _requested_state. Raise ResourceNotReady if it doesn't exist."""
     if not getattr(self, '_requested_state', None):
       raise ResourceNotReady()
     return self._requested_state
 
-  @_syncComputerPartitionInformation
   def getInstanceParameterDict(self):
     return getattr(self, '_parameter_dict', None) or {}
 
-  @_syncComputerPartitionInformation
   def getConnectionParameterDict(self):
     return getattr(self, '_connection_dict', None) or {}
 
-  @_syncComputerPartitionInformation
   def getSoftwareRelease(self):
     """
     Returns the software release associate to the computer partition.
@@ -548,7 +481,6 @@ class ComputerPartition(SlapDocument):
           'connection_xml': xml_marshaller.dumps(connection_dict),
           'slave_reference': slave_reference})
 
-  @_syncComputerPartitionInformation
   def getInstanceParameter(self, key):
     parameter_dict = getattr(self, '_parameter_dict', None) or {}
     if key in parameter_dict:
@@ -556,7 +488,6 @@ class ComputerPartition(SlapDocument):
     else:
       raise NotFoundError("%s not found" % key)
 
-  @_syncComputerPartitionInformation
   def getConnectionParameter(self, key):
     connection_dict = getattr(self, '_connection_dict', None) or {}
     if key in connection_dict:

slapos/tests/util.py

+##############################################################################
+#
+# Copyright (c) 2013 Vifib SARL and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+import os
+import slapos.util
+import tempfile
+import unittest
+
+class TestMkdirP(unittest.TestCase):
+  """
+  Tests methods available in the slapos.util module.
+  """
+
+  def test_mkdir_p_new_directory(self):
+    """
+    Test that mkdir_p recursively creates a directory.
+    """
+    root_directory = tempfile.mkdtemp()
+    wanted_directory = os.path.join(root_directory, 'foo', 'bar')
+    slapos.util.mkdir_p(wanted_directory)
+    self.assertTrue(os.path.isdir(wanted_directory))
+
+  def test_mkdir_already_existing(self):
+    """
+    Check that mkdir_p doesn't raise if directory already exist.
+    """
+    root_directory = tempfile.mkdtemp()
+    slapos.util.mkdir_p(root_directory)
+    self.assertTrue(os.path.isdir(root_directory))
+
+
+if __name__ == '__main__':
+  unittest.main()