Commit 655fb891 authored by Xavier Thompson's avatar Xavier Thompson

slapproxy: Add minimal hateoas support for cli

Add the minimum hateoas support required for the cli commands:
- `slapos service list`
- `slapos service info <reference>`
- `slapos computer list`
- `slapos computer info <reference>`

To enable hateoas, complete the slapos.cfg configuration with:
```
[slapos]
master_rest_url = <proxyaddr>/hateoas
```

Also: Fix some bugs and python3 incompatibilities in slap/hateoas.py.

See merge request nexedi/slapos.core!290
parent 740021b4
Pipeline #15064 failed with stage
in 0 seconds
......@@ -31,6 +31,7 @@
import random
import string
import time
import re
from datetime import datetime
from slapos.slap.slap import Computer, ComputerPartition, \
SoftwareRelease, SoftwareInstance, NotFoundError
......@@ -39,12 +40,12 @@ import slapos.slap
from slapos.util import bytes2str, unicode2str, sqlite_connect, \
xml2dict, dict2xml
from flask import g, Flask, request, abort
from flask import g, Flask, request, abort, redirect, url_for
from slapos.util import loads, dumps
import six
from six.moves import range
from six.moves.urllib.parse import urlparse
from six.moves.urllib.parse import urlparse, unquote, urljoin
app = Flask(__name__)
......@@ -930,3 +931,186 @@ def getSoftwareReleaseListFromSoftwareProduct():
software_release_url_list = []
return dumps(software_release_url_list)
# hateoas routing
# ---------------
# We only need to handle the hateoas requests made by
# slapos service list
# slapos service info <reference>
# slapos computer list
# slapos computer info <reference>
def unquoted_url_for(method, **kwargs):
return unquote(url_for(method, **kwargs))
def busy_root_partitions_list(title=None):
partitions = []
query = 'SELECT * FROM %s WHERE slap_state<>"free" AND requested_by IS NULL'
args = []
if title:
query += ' AND partition_reference=?'
args.append(title)
for row in execute_db('partition', query, args):
p = dict(row)
p['url_string'] = p['software_release']
p['title'] = p['partition_reference']
p['relative_url'] = url_for('hateoas_partitions', partition_reference=p['partition_reference'])
partitions.append(p)
return partitions
def computers_list(reference=None):
computers = []
query = 'SELECT * FROM %s'
args = []
if reference:
query += ' WHERE reference=?'
args.append(reference)
for row in execute_db('computer', query, args):
c = dict(row)
c['title'] = c['reference']
c['relative_url'] = url_for('hateoas_computers', computer_reference=c['reference'])
computers.append(c)
return computers
r_string = re.compile('"((\\.|[^\\"])*)"')
def is_valid(name):
match = r_string.match(name)
if match.group(0) == name:
return True
return False
p_service_list = 'portal_type:"Hosting Subscription" AND validation_state:validated'
p_service_info = p_service_list + ' AND title:='
p_computer_list = 'portal_type:"Computer" AND validation_state:validated'
p_computer_info = p_computer_list + ' AND reference:='
def parse_query(query):
if query == p_service_list:
return busy_root_partitions_list()
elif query.startswith(p_service_info):
title = query[len(p_service_info):]
if is_valid(title):
return busy_root_partitions_list(title.strip('"'))
elif query == p_computer_list:
return computers_list()
elif query.startswith(p_computer_info):
reference = query[len(p_computer_info):]
if is_valid(reference):
return computers_list(reference.strip('"'))
return None
@app.route('/hateoas/partitions/<partition_reference>', methods=['GET'])
def hateoas_partitions(partition_reference):
partition = execute_db('partition', 'SELECT * FROM %s WHERE partition_reference=?', [partition_reference], one=True)
if partition is None:
abort(404)
return {
'_embedded': {
'_view': {
'form_id': {
'type': 'StringField',
'key': 'partition',
'default': partition['reference'],
},
'my_reference': {
'type': 'StringField',
'key': 'partition_reference',
'default': partition['partition_reference'],
},
'my_slap_state': {
'type': 'StringField',
'key': 'slap_state',
'default': partition['slap_state'],
},
'my_text_content': {
'type': 'StringField',
'key': 'xml',
'default': partition['xml'],
},
'my_connection_parameter_list': {
'type': 'StringField',
'key': 'connection_xml',
'default': partition['connection_xml'],
},
'my_url_string': {
'type': 'StringField',
'key': 'software_release',
'default': partition['software_release'],
},
},
},
'_links': {
'type': {
'name': 'Hosting Subscription',
},
},
}
@app.route('/hateoas/computers/<computer_reference>', methods=['GET'])
def hateoas_computers(computer_reference):
computer = execute_db('computer', 'SELECT * FROM %s WHERE reference=?', [computer_reference], one=True)
if computer is None:
abort(404)
return {
'_embedded': {
'_view': {
'form_id': {
'type': 'StringField',
'key': 'computer',
'default': computer['reference'],
},
'my_reference': {
'type': 'StringField',
'key': 'reference',
'default': computer['reference'],
},
'my_title': {
'type': 'StringField',
'key': 'reference',
'default': computer['reference'],
},
},
},
'_links': {
'type': {
'name': 'Computer',
},
},
}
def hateoas_traverse():
return redirect(request.args.get('relative_url'))
def hateoas_search():
contents = parse_query(request.args.get("query"))
if contents is None:
abort(400)
return { '_embedded': {'contents': contents} }
def hateoas_root():
return {
'_links': {
'raw_search': {
'href': urljoin(request.host_url, unquoted_url_for('hateoas', mode='search', query='{query}', select_list='{select_list}'))
},
'traverse': {
'href': urljoin(request.host_url, unquoted_url_for('hateoas', mode='traverse', relative_url='{relative_url}', view='{view}'))
},
}
}
mode_handlers = {
None: hateoas_root,
'search': hateoas_search,
'traverse': hateoas_traverse,
}
@app.route('/hateoas', methods=['GET'])
def hateoas():
mode = request.args.get('mode')
handler = mode_handlers.get(mode, lambda: abort(400))
resp = handler()
return resp
......@@ -383,7 +383,7 @@ class SlapHateoasNavigator(HateoasNavigator):
hosting_subscription_dict = {}
for hosting_subscription in hosting_subscription_list:
software_instance = TempDocument()
for key, value in hosting_subscription.iteritems():
for key, value in six.iteritems(hosting_subscription):
if key in ['_links', 'url_string']:
continue
setattr(software_instance, '_%s' % key, value)
......@@ -397,7 +397,7 @@ class SlapHateoasNavigator(HateoasNavigator):
computer_dict = {}
for computer_json in computer_list:
computer = TempDocument()
for key, value in computer_json.iteritems():
for key, value in six.iteritems(computer_json):
if key in ['_links']:
continue
setattr(computer, '_%s' % key, value)
......@@ -420,6 +420,7 @@ class SlapHateoasNavigator(HateoasNavigator):
assert len(hosting_subscription_list) <= 1, \
"There are more them one Hosting Subscription for this reference"
hosting_subscription_jio_key= None
for hosting_subscription_candidate in hosting_subscription_list:
if hosting_subscription_candidate.get('title') == reference:
hosting_subscription_jio_key = hosting_subscription_candidate['relative_url']
......@@ -437,6 +438,7 @@ class SlapHateoasNavigator(HateoasNavigator):
assert len(computer_list) <= 1, \
"There are more them one Computer for this reference"
computer_jio_key = None
for computer_candidate in computer_list:
if computer_candidate.get("reference") == reference:
computer_jio_key = computer_candidate['relative_url']
......
......@@ -42,8 +42,12 @@ import sys
import tempfile
import time
import unittest
import json
import mock
import requests
from six.moves.urllib.parse import urljoin
import slapos.proxy
import slapos.proxy.views as views
import slapos.slap
......@@ -1163,6 +1167,185 @@ class TestSlaveRequest(MasterMixin):
self.assertEqual(slave._partition_id, partition._partition_id)
class TestAppSession(requests.Session):
"""
A request session that exposes the necessary interface to seamlessly
replace the object returned by views.app.test_client().
"""
def __init__(self, prefix_url, *args, **kwargs):
super(TestAppSession, self).__init__(*args, **kwargs)
self.prefix_url = prefix_url
def request(self, method, url, *args, **kwargs):
url = urljoin(self.prefix_url, url)
resp = super(TestAppSession, self).request(method, url, *args, **kwargs)
setattr(resp, '_status_code', resp.status_code)
setattr(resp, 'data', resp.content)
return resp
class CliMasterMixin(MasterMixin):
"""
Start a real proxy via the cli so that it will anwser to cli requests.
"""
def createSlapOSConfigurationFile(self):
host = os.environ['SLAPOS_TEST_IPV4']
self.proxyaddr = 'http://%s:8080' % host
with open(self.slapos_cfg, 'w') as f:
f.write("""[slapos]
software_root = %(tempdir)s/opt/slapgrid
instance_root = %(tempdir)s/srv/slapgrid
master_url = %(proxyaddr)s
master_rest_url = %(proxyaddr)s/hateoas
computer_id = computer
[slapproxy]
host = %(host)s
port = 8080
database_uri = %(tempdir)s/lib/proxy.db
""" % {'tempdir': self._tempdir, 'proxyaddr': self.proxyaddr, 'host': host})
def cliDoSlapos(self, command, method=subprocess.check_output, **kwargs):
return method(
(sys.executable, '-m', 'slapos.cli.entry') + command + ('--cfg', self.slapos_cfg),
env={"PYTHONPATH": ':'.join(sys.path)},
cwd=os.chdir(os.path.join(os.path.dirname(slapos.proxy.__file__), os.pardir, os.pardir)),
universal_newlines=True,
**kwargs
)
def startProxy(self):
self.proxy_process = self.cliDoSlapos(
('proxy', 'start'),
method=subprocess.Popen,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
self.app = TestAppSession(self.proxyaddr)
# Wait a bit for proxy to be started
for attempts in range(1, 20):
try:
self.app.get('/')
except requests.ConnectionError:
time.sleep(0.1 * attempts)
else:
break
else:
self.fail('Could not start proxy.')
def tearDown(self):
self.proxy_process.kill()
self.proxy_process.wait()
super(CliMasterMixin, self).tearDown()
class TestCliInformation(CliMasterMixin):
"""
Test minimal hateoas support for cli.
"""
def test_computer_list(self):
self.format_for_number_of_partitions(1)
self.format_for_number_of_partitions(1, 'COMP-1')
self.format_for_number_of_partitions(1, 'COMP-2')
output = self.cliDoSlapos(('computer', 'list'), stderr=subprocess.DEVNULL).splitlines()
self.assertEqual(len(output), 4)
self.assertEqual(output[0], 'List of Computers:')
self.assertEqual(
sorted(output[1:]),
['COMP-1 COMP-1', 'COMP-2 COMP-2', 'computer computer'],
)
def test_computer_info(self):
self.format_for_number_of_partitions(1)
self.format_for_number_of_partitions(1, 'COMP-1')
self.format_for_number_of_partitions(1, 'COMP-2')
output0 = self.cliDoSlapos(('computer', 'info', 'computer'), stderr=subprocess.DEVNULL)
self.assertEqual(
output0.splitlines(),
['Computer Reference: computer', 'Computer Title : computer'],
)
output1 = self.cliDoSlapos(('computer', 'info', 'COMP-1'), stderr=subprocess.DEVNULL)
self.assertEqual(
output1.splitlines(),
['Computer Reference: COMP-1', 'Computer Title : COMP-1'],
)
output2 = self.cliDoSlapos(('computer', 'info', 'COMP-2'), stderr=subprocess.DEVNULL)
self.assertEqual(
output2.splitlines(),
['Computer Reference: COMP-2', 'Computer Title : COMP-2'],
)
def test_service_list(self):
self.format_for_number_of_partitions(4)
self.request('http://sr0//', None, 'MyInstance0', None)
self.request('http://sr1//', None, 'MyInstance1', None)
self.request('http://sr2//', None, 'MyInstance2', None)
self.request('http://sr3//', None, 'MyInstance3', 'slappart0')
output = self.cliDoSlapos(('service', 'list'), stderr=subprocess.DEVNULL).splitlines()
self.assertEqual(len(output), 4)
self.assertEqual(output[0], 'List of services:')
self.assertEqual(
sorted(output[1:]),
['MyInstance0 http://sr0//', 'MyInstance1 http://sr1//', 'MyInstance2 http://sr2//'],
)
def test_service_info(self):
self.format_for_number_of_partitions(3)
self.request('http://sr0//', None, 'MyInstance0', None)
self.request('http://sr1//', None, 'MyInstance1', None, partition_parameter_kw={'couscous': 'hello'})
self.request('http://sr2//', None, 'MyInstance2', 'slappart0')
output0 = self.cliDoSlapos(('service', 'info', 'MyInstance0'), stderr=subprocess.DEVNULL)
self.assertEqual(
output0.splitlines(),
[
'Software Release URL: http://sr0//',
'Instance state: busy',
'Instance parameters:',
'{}',
'Connection parameters:',
'None'
],
)
output1 = self.cliDoSlapos(('service', 'info', 'MyInstance1'), stderr=subprocess.DEVNULL)
self.assertEqual(
output1.splitlines(),
[
'Software Release URL: http://sr1//',
'Instance state: busy',
'Instance parameters:',
"{'couscous': 'hello'}",
'Connection parameters:',
'None'
],
)
try:
output2 = self.cliDoSlapos(('service', 'info', 'MyInstance2'), stderr=subprocess.STDOUT)
self.fail()
except subprocess.CalledProcessError as e:
self.assertIn('Instance MyInstance2 does not exist.', e.output)
def test_invalid_service_names(self):
invalid_names = ('"MyInstance0', 'MyInstance1"', 'My"Instance2', 'title:="MyInstance3"', 'reference:="MyInstance4"')
self.format_for_number_of_partitions(len(invalid_names))
for i, name in enumerate(invalid_names):
self.request('http://sr%d//' % i, None, name)
for i, name in enumerate(invalid_names):
try:
self.cliDoSlapos(('service', 'info', name), method=subprocess.check_output, stderr=subprocess.STDOUT)
self.fail()
except subprocess.CalledProcessError as e:
self.assertIn("HTTPError: 400 Client Error: BAD REQUEST", e.output)
def test_invalid_computer_names(self):
invalid_names = ('"COMP-0', 'COMP-1"', 'COMP"-2', 'title:="COMP-3"', 'reference:="COMP-4"')
for name in invalid_names:
self.format_for_number_of_partitions(1, name)
for name in invalid_names:
try:
self.cliDoSlapos(('computer', 'info', name), method=subprocess.check_output, stderr=subprocess.STDOUT)
self.fail()
except subprocess.CalledProcessError as e:
self.assertIn("HTTPError: 400 Client Error: BAD REQUEST", e.output)
class TestMultiNodeSupport(MasterMixin):
def test_multi_node_support_different_software_release_list(self):
"""
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment