Commit ef76b5e2 authored by Marco Mariani's avatar Marco Mariani Committed by Rafael Monnerat

migrate slap.py from (httplib, socket, ssl, urllib) -> requests

parent d405e453
...@@ -53,12 +53,6 @@ class IResourceNotReady(IException): ...@@ -53,12 +53,6 @@ class IResourceNotReady(IException):
ready on the slap server. ready on the slap server.
""" """
class IUnauthorized(IException):
"""
Classes which implement IUnauthorized are used to report missing
permissions on the slap server.
"""
class IRequester(Interface): class IRequester(Interface):
""" """
Classes which implement IRequester can request software instance to the Classes which implement IRequester can request software instance to the
......
...@@ -32,15 +32,11 @@ Simple, easy to (un)marshall classes for slap client/server communication ...@@ -32,15 +32,11 @@ Simple, easy to (un)marshall classes for slap client/server communication
__all__ = ["slap", "ComputerPartition", "Computer", "SoftwareRelease", __all__ = ["slap", "ComputerPartition", "Computer", "SoftwareRelease",
"SoftwareProductCollection", "SoftwareProductCollection",
"Supply", "OpenOrder", "NotFoundError", "Unauthorized", "Supply", "OpenOrder", "NotFoundError",
"ResourceNotReady", "ServerError"] "ResourceNotReady", "ServerError"]
import httplib
import logging import logging
import re import re
import socket
import ssl
import urllib
import urlparse import urlparse
from util import xml2dict from util import xml2dict
...@@ -49,6 +45,12 @@ import zope.interface ...@@ -49,6 +45,12 @@ import zope.interface
from interface import slap as interface from interface import slap as interface
from xml_marshaller import xml_marshaller from xml_marshaller import xml_marshaller
import requests
# silence messages like 'Starting connection' that are logged with INFO
urllib3_logger = logging.getLogger('requests.packages.urllib3')
urllib3_logger.setLevel(logging.WARNING)
# XXX fallback_logger to be deprecated together with the old CLI entry points. # XXX fallback_logger to be deprecated together with the old CLI entry points.
fallback_logger = logging.getLogger(__name__) fallback_logger = logging.getLogger(__name__)
fallback_handler = logging.StreamHandler() fallback_handler = logging.StreamHandler()
...@@ -59,25 +61,12 @@ fallback_logger.addHandler(fallback_handler) ...@@ -59,25 +61,12 @@ fallback_logger.addHandler(fallback_handler)
DEFAULT_SOFTWARE_TYPE = 'RootSoftwareInstance' DEFAULT_SOFTWARE_TYPE = 'RootSoftwareInstance'
# httplib.HTTPSConnection with key verification class AuthenticationError(Exception):
class HTTPSConnectionCA(httplib.HTTPSConnection): pass
"""Patched version of HTTPSConnection which verifies server certificate"""
def __init__(self, *args, **kwargs):
self.ca_file = kwargs.pop('ca_file')
if self.ca_file is None:
raise ValueError('ca_file is required argument.')
httplib.HTTPSConnection.__init__(self, *args, **kwargs)
def connect(self):
"Connect to a host on a given (SSL) port and verify its certificate."
sock = socket.create_connection((self.host, self.port), class ConnectionError(Exception):
self.timeout, self.source_address) pass
if self._tunnel_host:
self.sock = sock
self._tunnel()
self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file,
ca_certs=self.ca_file, cert_reqs=ssl.CERT_REQUIRED)
class SlapDocument: class SlapDocument:
...@@ -94,7 +83,7 @@ class SlapRequester(SlapDocument): ...@@ -94,7 +83,7 @@ class SlapRequester(SlapDocument):
""" """
def _requestComputerPartition(self, request_dict): def _requestComputerPartition(self, request_dict):
try: try:
xml = self._connection_helper.POST('/requestComputerPartition', request_dict) xml = self._connection_helper.POST('requestComputerPartition', request_dict)
except ResourceNotReady: except ResourceNotReady:
return ComputerPartition( return ComputerPartition(
request_dict=request_dict, request_dict=request_dict,
...@@ -156,7 +145,7 @@ class SoftwareRelease(SlapDocument): ...@@ -156,7 +145,7 @@ class SoftwareRelease(SlapDocument):
def error(self, error_log, logger=None): def error(self, error_log, logger=None):
try: try:
# Does not follow interface # Does not follow interface
self._connection_helper.POST('/softwareReleaseError', { self._connection_helper.POST('softwareReleaseError', {
'url': self.getURI(), 'url': self.getURI(),
'computer_id': self.getComputerId(), 'computer_id': self.getComputerId(),
'error_log': error_log}) 'error_log': error_log})
...@@ -164,17 +153,17 @@ class SoftwareRelease(SlapDocument): ...@@ -164,17 +153,17 @@ class SoftwareRelease(SlapDocument):
(logger or fallback_logger).exception('') (logger or fallback_logger).exception('')
def available(self): def available(self):
self._connection_helper.POST('/availableSoftwareRelease', { self._connection_helper.POST('availableSoftwareRelease', {
'url': self.getURI(), 'url': self.getURI(),
'computer_id': self.getComputerId()}) 'computer_id': self.getComputerId()})
def building(self): def building(self):
self._connection_helper.POST('/buildingSoftwareRelease', { self._connection_helper.POST('buildingSoftwareRelease', {
'url': self.getURI(), 'url': self.getURI(),
'computer_id': self.getComputerId()}) 'computer_id': self.getComputerId()})
def destroyed(self): def destroyed(self):
self._connection_helper.POST('/destroyedSoftwareRelease', { self._connection_helper.POST('destroyedSoftwareRelease', {
'url': self.getURI(), 'url': self.getURI(),
'computer_id': self.getComputerId()}) 'computer_id': self.getComputerId()})
...@@ -233,16 +222,12 @@ class NotFoundError(Exception): ...@@ -233,16 +222,12 @@ class NotFoundError(Exception):
zope.interface.implements(interface.INotFoundError) zope.interface.implements(interface.INotFoundError)
class Unauthorized(Exception):
zope.interface.implements(interface.IUnauthorized)
class Supply(SlapDocument): class Supply(SlapDocument):
zope.interface.implements(interface.ISupply) zope.interface.implements(interface.ISupply)
def supply(self, software_release, computer_guid=None, state='available'): def supply(self, software_release, computer_guid=None, state='available'):
try: try:
self._connection_helper.POST('/supplySupply', { self._connection_helper.POST('supplySupply', {
'url': software_release, 'url': software_release,
'computer_id': computer_guid, 'computer_id': computer_guid,
'state': state}) 'state': state})
...@@ -282,7 +267,7 @@ class OpenOrder(SlapRequester): ...@@ -282,7 +267,7 @@ class OpenOrder(SlapRequester):
""" """
Requests a computer. Requests a computer.
""" """
xml = self._connection_helper.POST('/requestComputer', xml = self._connection_helper.POST('requestComputer',
{'computer_title': computer_reference}) {'computer_title': computer_reference})
computer = xml_marshaller.loads(xml) computer = xml_marshaller.loads(xml)
computer._connection_helper = self._connection_helper computer._connection_helper = self._connection_helper
...@@ -341,30 +326,29 @@ class Computer(SlapDocument): ...@@ -341,30 +326,29 @@ class Computer(SlapDocument):
def reportUsage(self, computer_usage): def reportUsage(self, computer_usage):
if computer_usage == "": if computer_usage == "":
return return
self._connection_helper.POST('/useComputer', { self._connection_helper.POST('useComputer', {
'computer_id': self._computer_id, 'computer_id': self._computer_id,
'use_string': computer_usage}) 'use_string': computer_usage})
def updateConfiguration(self, xml): def updateConfiguration(self, xml):
return self._connection_helper.POST( return self._connection_helper.POST(
'/loadComputerConfigurationFromXML', {'xml': xml}) 'loadComputerConfigurationFromXML', data={'xml': xml})
def bang(self, message): def bang(self, message):
self._connection_helper.POST('/computerBang', { self._connection_helper.POST('computerBang', {
'computer_id': self._computer_id, 'computer_id': self._computer_id,
'message': message}) 'message': message})
def getStatus(self): def getStatus(self):
xml = self._connection_helper.GET( xml = self._connection_helper.GET('getComputerStatus', {'computer_id': self._computer_id})
'/getComputerStatus?computer_id=%s' % self._computer_id)
return xml_marshaller.loads(xml) return xml_marshaller.loads(xml)
def revokeCertificate(self): def revokeCertificate(self):
self._connection_helper.POST('/revokeComputerCertificate', { self._connection_helper.POST('revokeComputerCertificate', {
'computer_id': self._computer_id}) 'computer_id': self._computer_id})
def generateCertificate(self): def generateCertificate(self):
xml = self._connection_helper.POST('/generateComputerCertificate', { xml = self._connection_helper.POST('generateComputerCertificate', {
'computer_id': self._computer_id}) 'computer_id': self._computer_id})
return xml_marshaller.loads(xml) return xml_marshaller.loads(xml)
...@@ -436,36 +420,36 @@ class ComputerPartition(SlapRequester): ...@@ -436,36 +420,36 @@ class ComputerPartition(SlapRequester):
return self._requestComputerPartition(request_dict) return self._requestComputerPartition(request_dict)
def building(self): def building(self):
self._connection_helper.POST('/buildingComputerPartition', { self._connection_helper.POST('buildingComputerPartition', {
'computer_id': self._computer_id, 'computer_id': self._computer_id,
'computer_partition_id': self.getId()}) 'computer_partition_id': self.getId()})
def available(self): def available(self):
self._connection_helper.POST('/availableComputerPartition', { self._connection_helper.POST('availableComputerPartition', {
'computer_id': self._computer_id, 'computer_id': self._computer_id,
'computer_partition_id': self.getId()}) 'computer_partition_id': self.getId()})
def destroyed(self): def destroyed(self):
self._connection_helper.POST('/destroyedComputerPartition', { self._connection_helper.POST('destroyedComputerPartition', {
'computer_id': self._computer_id, 'computer_id': self._computer_id,
'computer_partition_id': self.getId(), 'computer_partition_id': self.getId(),
}) })
def started(self): def started(self):
self._connection_helper.POST('/startedComputerPartition', { self._connection_helper.POST('startedComputerPartition', {
'computer_id': self._computer_id, 'computer_id': self._computer_id,
'computer_partition_id': self.getId(), 'computer_partition_id': self.getId(),
}) })
def stopped(self): def stopped(self):
self._connection_helper.POST('/stoppedComputerPartition', { self._connection_helper.POST('stoppedComputerPartition', {
'computer_id': self._computer_id, 'computer_id': self._computer_id,
'computer_partition_id': self.getId(), 'computer_partition_id': self.getId(),
}) })
def error(self, error_log, logger=None): def error(self, error_log, logger=None):
try: try:
self._connection_helper.POST('/softwareInstanceError', { self._connection_helper.POST('softwareInstanceError', {
'computer_id': self._computer_id, 'computer_id': self._computer_id,
'computer_partition_id': self.getId(), 'computer_partition_id': self.getId(),
'error_log': error_log}) 'error_log': error_log})
...@@ -473,7 +457,7 @@ class ComputerPartition(SlapRequester): ...@@ -473,7 +457,7 @@ class ComputerPartition(SlapRequester):
(logger or fallback_logger).exception('') (logger or fallback_logger).exception('')
def bang(self, message): def bang(self, message):
self._connection_helper.POST('/softwareInstanceBang', { self._connection_helper.POST('softwareInstanceBang', {
'computer_id': self._computer_id, 'computer_id': self._computer_id,
'computer_partition_id': self.getId(), 'computer_partition_id': self.getId(),
'message': message}) 'message': message})
...@@ -486,7 +470,7 @@ class ComputerPartition(SlapRequester): ...@@ -486,7 +470,7 @@ class ComputerPartition(SlapRequester):
} }
if slave_reference: if slave_reference:
post_dict['slave_reference'] = slave_reference post_dict['slave_reference'] = slave_reference
self._connection_helper.POST('/softwareInstanceRename', post_dict) self._connection_helper.POST('softwareInstanceRename', post_dict)
def getId(self): def getId(self):
if not getattr(self, '_partition_id', None): if not getattr(self, '_partition_id', None):
...@@ -540,7 +524,7 @@ class ComputerPartition(SlapRequester): ...@@ -540,7 +524,7 @@ class ComputerPartition(SlapRequester):
def setConnectionDict(self, connection_dict, slave_reference=None): def setConnectionDict(self, connection_dict, slave_reference=None):
if self.getConnectionParameterDict() != connection_dict: if self.getConnectionParameterDict() != connection_dict:
self._connection_helper.POST('/setComputerPartitionConnectionXml', { self._connection_helper.POST('setComputerPartitionConnectionXml', {
'computer_id': self._computer_id, 'computer_id': self._computer_id,
'computer_partition_id': self._partition_id, 'computer_partition_id': self._partition_id,
'connection_xml': xml_marshaller.dumps(connection_dict), 'connection_xml': xml_marshaller.dumps(connection_dict),
...@@ -565,39 +549,39 @@ class ComputerPartition(SlapRequester): ...@@ -565,39 +549,39 @@ class ComputerPartition(SlapRequester):
self.usage = usage_log self.usage = usage_log
def getCertificate(self): def getCertificate(self):
xml = self._connection_helper.GET( xml = self._connection_helper.GET('getComputerPartitionCertificate',
'/getComputerPartitionCertificate?computer_id=%s&' {
'computer_partition_id=%s' % (self._computer_id, self._partition_id)) 'computer_id': self._computer_id,
'computer_partition_id': self._partition_id,
}
)
return xml_marshaller.loads(xml) return xml_marshaller.loads(xml)
def getStatus(self): def getStatus(self):
xml = self._connection_helper.GET( xml = self._connection_helper.GET('getComputerPartitionStatus',
'/getComputerPartitionStatus?computer_id=%s&' {
'computer_partition_id=%s' % (self._computer_id, self._partition_id)) 'computer_id': self._computer_id,
'computer_partition_id': self._partition_id,
}
)
return xml_marshaller.loads(xml) return xml_marshaller.loads(xml)
class ConnectionHelper: class ConnectionHelper:
error_message_timeout = "\nThe connection timed out. Please try again later." def __init__(self, master_url, key_file=None,
error_message_connect_fail = "Couldn't connect to the server. Please " \
"double check given master-url argument, and make sure that IPv6 is " \
"enabled on your machine and that the server is available. The " \
"original error was: "
ssl_error_message_connect_fail = "\nCouldn't authenticate computer. Please "\
"check that certificate and key exist and are valid. "
def __init__(self, connection_wrapper, host, path, key_file=None,
cert_file=None, master_ca_file=None, timeout=None): cert_file=None, master_ca_file=None, timeout=None):
self.connection_wrapper = connection_wrapper if master_url.endswith('/'):
self.host = host self.slapgrid_uri = master_url
self.path = path else:
# add a slash or the last path segment will be ignored by urljoin
self.slapgrid_uri = master_url + '/'
self.key_file = key_file self.key_file = key_file
self.cert_file = cert_file self.cert_file = cert_file
self.master_ca_file = master_ca_file self.master_ca_file = master_ca_file
self.timeout = timeout self.timeout = timeout
def getComputerInformation(self, computer_id): def getComputerInformation(self, computer_id):
xml = self.GET('/getComputerInformation?computer_id=%s' % computer_id) xml = self.GET('getComputerInformation', {'computer_id': computer_id})
return xml_marshaller.loads(xml) return xml_marshaller.loads(xml)
def getFullComputerInformation(self, computer_id): def getFullComputerInformation(self, computer_id):
...@@ -605,100 +589,87 @@ class ConnectionHelper: ...@@ -605,100 +589,87 @@ class ConnectionHelper:
Retrieve from SlapOS Master Computer instance containing all needed Retrieve from SlapOS Master Computer instance containing all needed
informations (Software Releases, Computer Partitions, ...). informations (Software Releases, Computer Partitions, ...).
""" """
method = '/getFullComputerInformation?computer_id=%s' % computer_id path = 'getFullComputerInformation'
params = {'computer_id': computer_id}
if not computer_id: if not computer_id:
# XXX-Cedric: should raise something smarter than "NotFound". # XXX-Cedric: should raise something smarter than "NotFound".
raise NotFoundError(method) raise NotFoundError('%r %r' (path, params))
try: try:
xml = self.GET(method) xml = self.GET(path, params)
except NotFoundError: except NotFoundError:
# XXX: This is a ugly way to keep backward compatibility, # XXX: This is a ugly way to keep backward compatibility,
# We should stablise slap library soon. # We should stablise slap library soon.
xml = self.GET('/getComputerInformation?computer_id=%s' % computer_id) xml = self.GET('getComputerInformation', {'computer_id': computer_id})
return xml_marshaller.loads(xml) return xml_marshaller.loads(xml)
def connect(self): def do_request(self, method, path, params=None, data=None, headers=None):
connection_dict = { url = urlparse.urljoin(self.slapgrid_uri, path)
'host': self.host if path.startswith('/'):
} raise ValueError('method path should be relative: %s' % path)
if self.key_file and self.cert_file:
connection_dict['key_file'] = self.key_file
connection_dict['cert_file'] = self.cert_file
if self.master_ca_file:
connection_dict['ca_file'] = self.master_ca_file
self.connection = self.connection_wrapper(**connection_dict)
def GET(self, path):
try: try:
default_timeout = socket.getdefaulttimeout() if url.startswith('https'):
socket.setdefaulttimeout(self.timeout) cert = (self.cert_file, self.key_file)
try: else:
self.connect() cert = None
self.connection.request('GET', self.path + path)
response = self.connection.getresponse() # XXX TODO: handle host cert verify
# If ssl error : may come from bad configuration
except ssl.SSLError as exc: req = method(url=url,
if exc.message == 'The read operation timed out': params=params,
raise socket.error(str(exc) + self.error_message_timeout) cert=cert,
raise ssl.SSLError(str(exc) + self.ssl_error_message_connect_fail) verify=False,
except socket.error as exc: data=data,
if exc.message == 'timed out': headers=headers,
raise socket.error(str(exc) + self.error_message_timeout) timeout=self.timeout)
raise socket.error(self.error_message_connect_fail + str(exc)) req.raise_for_status()
# check self.response.status and raise exception early except (requests.Timeout, requests.ConnectionError) as exc:
if response.status == httplib.REQUEST_TIMEOUT: raise ConnectionError("Couldn't connect to the server. Please "
# resource is not ready "double check given master-url argument, and make sure that IPv6 is "
"enabled on your machine and that the server is available. The "
"original error was:\n%s" % exc)
except requests.HTTPError as exc:
if exc.response.status_code == requests.status_codes.codes.not_found:
msg = url
if params:
msg += ' - %s' % params
raise NotFoundError(msg)
elif exc.response.status_code == requests.status_codes.codes.request_timeout:
# this is explicitly returned by SlapOS master, and does not really mean timeout
raise ResourceNotReady(path) raise ResourceNotReady(path)
elif response.status == httplib.NOT_FOUND: # XXX TODO test request timeout and resource not found
raise NotFoundError(path) else:
elif response.status == httplib.FORBIDDEN: # we don't know how or don't want to handle these (including Unauthorized)
raise Unauthorized(path) req.raise_for_status()
elif response.status != httplib.OK: except requests.exceptions.SSLError as exc:
message = parsed_error_message(response.status, raise AuthenticationError("%s\nCouldn't authenticate computer. Please "
response.read(), "check that certificate and key exist and are valid." % exc)
path)
raise ServerError(message) # XXX TODO parse server messages for client configure and node register
finally: # elif response.status != httplib.OK:
socket.setdefaulttimeout(default_timeout) # message = parsed_error_message(response.status,
# response.read(),
return response.read() # path)
# raise ServerError(message)
def POST(self, path, parameter_dict,
return req
def GET(self, path, params=None):
req = self.do_request(requests.get,
path=path,
params=params)
return req.text
def POST(self, path, params=None, data=None,
content_type='application/x-www-form-urlencoded'): content_type='application/x-www-form-urlencoded'):
try: req = self.do_request(requests.post,
default_timeout = socket.getdefaulttimeout() path=path,
socket.setdefaulttimeout(self.timeout) params=params,
try: data=data,
self.connect() headers={'Content-type': content_type})
header_dict = {'Content-type': content_type} return req.text
self.connection.request("POST", self.path + path,
urllib.urlencode(parameter_dict), header_dict)
# If ssl error : must come from bad configuration
except ssl.SSLError as exc:
raise ssl.SSLError(self.ssl_error_message_connect_fail + str(exc))
except socket.error as exc:
raise socket.error(self.error_message_connect_fail + str(exc))
response = self.connection.getresponse()
# check self.response.status and raise exception early
if response.status == httplib.REQUEST_TIMEOUT:
# resource is not ready
raise ResourceNotReady("%s - %s" % (path, parameter_dict))
elif response.status == httplib.NOT_FOUND:
raise NotFoundError("%s - %s" % (path, parameter_dict))
elif response.status == httplib.FORBIDDEN:
raise Unauthorized("%s - %s" % (path, parameter_dict))
elif response.status != httplib.OK:
message = parsed_error_message(response.status,
response.read(),
path)
raise ServerError(message)
finally:
socket.setdefaulttimeout(default_timeout)
return response.read()
class slap: class slap:
...@@ -706,22 +677,10 @@ class slap: ...@@ -706,22 +677,10 @@ class slap:
def initializeConnection(self, slapgrid_uri, key_file=None, cert_file=None, def initializeConnection(self, slapgrid_uri, key_file=None, cert_file=None,
master_ca_file=None, timeout=60): master_ca_file=None, timeout=60):
scheme, netloc, path, query, fragment = urlparse.urlsplit(slapgrid_uri) if master_ca_file:
if not (query == '' and fragment == ''): raise NotImplementedError('Master certificate not verified in this version: %s' % master_ca_file)
raise AttributeError('Passed URL %r issue: not parseable' % slapgrid_uri)
self._connection_helper = ConnectionHelper(slapgrid_uri, key_file, cert_file, master_ca_file, timeout)
if scheme == 'http':
connection_wrapper = httplib.HTTPConnection
elif scheme == 'https':
if master_ca_file is not None:
connection_wrapper = HTTPSConnectionCA
else:
connection_wrapper = httplib.HTTPSConnection
else:
raise AttributeError('Passed URL %r issue: there is no support '
'for %r protocol' % (slapgrid_uri, scheme))
self._connection_helper = ConnectionHelper(connection_wrapper,
netloc, path, key_file, cert_file, master_ca_file, timeout)
# XXX-Cedric: this method is never used and thus should be removed. # XXX-Cedric: this method is never used and thus should be removed.
def registerSoftwareRelease(self, software_release): def registerSoftwareRelease(self, software_release):
...@@ -749,9 +708,12 @@ class slap: ...@@ -749,9 +708,12 @@ class slap:
# XXX-Cedric: should raise something smarter than NotFound # XXX-Cedric: should raise something smarter than NotFound
raise NotFoundError raise NotFoundError
xml = self._connection_helper.GET('/registerComputerPartition?' \ xml = self._connection_helper.GET('registerComputerPartition',
'computer_reference=%s&computer_partition_reference=%s' % ( {
computer_guid, partition_id)) 'computer_reference': computer_guid,
'computer_partition_reference': partition_id,
}
)
result = xml_marshaller.loads(xml) result = xml_marshaller.loads(xml)
# XXX: dirty hack to make computer partition usable. xml_marshaller is too # XXX: dirty hack to make computer partition usable. xml_marshaller is too
# low-level for our needs here. # low-level for our needs here.
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment