Commit ab0d27e6 authored by Xavier Thompson's avatar Xavier Thompson

slapproxy: Add local software release path

See merge request nexedi/slapos.core!288
parents a01eb331 dd57a2cf
Pipeline #15453 passed with stage
in 0 seconds
...@@ -47,6 +47,8 @@ class ProxyStartCommand(ConfigCommand): ...@@ -47,6 +47,8 @@ class ProxyStartCommand(ConfigCommand):
help='Port to use') help='Port to use')
ap.add_argument('--host', ap.add_argument('--host',
help='Host to use') help='Host to use')
ap.add_argument('--local-software-release-root',
help='Root path for local Software Releases')
return ap return ap
......
...@@ -93,6 +93,7 @@ def setupFlaskConfiguration(conf): ...@@ -93,6 +93,7 @@ def setupFlaskConfiguration(conf):
app.config['DATABASE_URI'] = conf.database_uri app.config['DATABASE_URI'] = conf.database_uri
app.config['software_product_list'] = conf.software_product_list app.config['software_product_list'] = conf.software_product_list
app.config['multimaster'] = conf.multimaster app.config['multimaster'] = conf.multimaster
app.config['local_software_release_root'] = getattr(conf,'local_software_release_root', None)
def connectDB(): def connectDB():
# if first connection, create an empty db at DATABASE_URI path # if first connection, create an empty db at DATABASE_URI path
......
--version:14 --version:15
CREATE TABLE IF NOT EXISTS local_software_release_root%(version)s (
path VARCHAR(255)
);
INSERT INTO local_software_release_root%(version)s VALUES(NULL);
CREATE TABLE IF NOT EXISTS software%(version)s ( CREATE TABLE IF NOT EXISTS software%(version)s (
url VARCHAR(255), url VARCHAR(255),
computer_reference VARCHAR(255) DEFAULT '%(computer)s', computer_reference VARCHAR(255) DEFAULT '%(computer)s',
......
...@@ -32,6 +32,7 @@ import random ...@@ -32,6 +32,7 @@ import random
import string import string
import time import time
import re import re
import os
from datetime import datetime from datetime import datetime
from slapos.slap.slap import Computer, ComputerPartition, \ from slapos.slap.slap import Computer, ComputerPartition, \
SoftwareRelease, SoftwareInstance, NotFoundError SoftwareRelease, SoftwareInstance, NotFoundError
...@@ -177,7 +178,12 @@ def _upgradeDatabaseIfNeeded(): ...@@ -177,7 +178,12 @@ def _upgradeDatabaseIfNeeded():
# Migrate all data to new tables # Migrate all data to new tables
app.logger.info('Old schema detected: Migrating old tables...') app.logger.info('Old schema detected: Migrating old tables...')
for table in ('software', 'computer', 'partition', 'slave', 'partition_network'): table_list = ('software', 'computer', 'partition', 'slave', 'partition_network')
if int(current_schema_version) >= 11:
table_list += ('forwarded_partition_request',)
if int(current_schema_version) >= 15:
table_list += ('local_software_release_root',)
for table in table_list:
for row in execute_db(table, 'SELECT * from %s', db_version=current_schema_version): for row in execute_db(table, 'SELECT * from %s', db_version=current_schema_version):
columns = ', '.join(row.keys()) columns = ', '.join(row.keys())
placeholders = ':'+', :'.join(row.keys()) placeholders = ':'+', :'.join(row.keys())
...@@ -188,6 +194,33 @@ def _upgradeDatabaseIfNeeded(): ...@@ -188,6 +194,33 @@ def _upgradeDatabaseIfNeeded():
g.db.execute("DROP table %s" % previous_table) g.db.execute("DROP table %s" % previous_table)
g.db.commit() g.db.commit()
def _updateLocalSoftwareReleaseRootPathIfNeeded():
"""
Update the local software release root path if it changed,
and rebase all URLs in the database relatively to the new path.
"""
# Retrieve the current root path and replace it with the new one
current_root_path = execute_db('local_software_release_root', 'SELECT * from %s', one=True)['path'] or os.sep
new_root_path = app.config['local_software_release_root'] or os.sep
execute_db('local_software_release_root', 'UPDATE %s SET path=?', [new_root_path])
# Rebase all URLs relative to the new root path
if current_root_path != new_root_path:
app.logger.info('Updating local software release root path: %s --> %s', current_root_path, new_root_path)
def migrate_url(url):
if not url or urlparse(url).scheme:
app.logger.debug('Migrate URL ? N: %s is not a path', url)
return url
rel = os.path.relpath(url, current_root_path)
if rel.startswith(os.pardir + os.sep):
app.logger.debug('Migrate URL ? N: %s is not a subpath', url)
return url
new = os.path.join(new_root_path, rel)
app.logger.debug('Migrate URL ? Y: %s -> %s', url, new)
return new
g.db.create_function('migrate_url', 1, migrate_url)
execute_db('software', 'UPDATE %s SET url=migrate_url(url)')
execute_db('partition', 'UPDATE %s SET software_release=migrate_url(software_release)')
is_schema_already_executed = False is_schema_already_executed = False
@app.before_request @app.before_request
def before_request(): def before_request():
...@@ -195,6 +228,7 @@ def before_request(): ...@@ -195,6 +228,7 @@ def before_request():
global is_schema_already_executed global is_schema_already_executed
if not is_schema_already_executed: if not is_schema_already_executed:
_upgradeDatabaseIfNeeded() _upgradeDatabaseIfNeeded()
_updateLocalSoftwareReleaseRootPathIfNeeded()
is_schema_already_executed = True is_schema_already_executed = True
......
...@@ -211,6 +211,7 @@ class SlapOSConfigWriter(ConfigWriter): ...@@ -211,6 +211,7 @@ class SlapOSConfigWriter(ConfigWriter):
host = {standalone_slapos._server_ip} host = {standalone_slapos._server_ip}
port = {standalone_slapos._server_port} port = {standalone_slapos._server_port}
database_uri = {standalone_slapos._proxy_database} database_uri = {standalone_slapos._proxy_database}
local_software_release_root = {standalone_slapos._local_software_release_root}
{partition_forward_configuration} {partition_forward_configuration}
""").format(**locals())) """).format(**locals()))
...@@ -318,6 +319,7 @@ class StandaloneSlapOS(object): ...@@ -318,6 +319,7 @@ class StandaloneSlapOS(object):
shared_part_root=None, shared_part_root=None,
partition_forward_configuration=(), partition_forward_configuration=(),
slapos_bin='slapos', slapos_bin='slapos',
local_software_release_root=os.sep,
): ):
# type: (str, str, int, str, Iterable[str], Optional[str], Optional[str], Optional[str], Iterable[Union[PartitionForwardConfiguration, PartitionForwardAsPartitionConfiguration]]) -> None # type: (str, str, int, str, Iterable[str], Optional[str], Optional[str], Optional[str], Iterable[Union[PartitionForwardConfiguration, PartitionForwardAsPartitionConfiguration]]) -> None
"""Constructor, creates a standalone slapos in `base_directory`. """Constructor, creates a standalone slapos in `base_directory`.
...@@ -332,6 +334,7 @@ class StandaloneSlapOS(object): ...@@ -332,6 +334,7 @@ class StandaloneSlapOS(object):
* `shared_part_root` -- directory to hold shared parts software, default to "shared" in `base_directory`. * `shared_part_root` -- directory to hold shared parts software, default to "shared" in `base_directory`.
* `partition_forward_configuration` -- configuration of partition request forwarding to external SlapOS master. * `partition_forward_configuration` -- configuration of partition request forwarding to external SlapOS master.
* `slapos_bin` -- slapos executable to use, default to "slapos" (thus depending on the runtime PATH). * `slapos_bin` -- slapos executable to use, default to "slapos" (thus depending on the runtime PATH).
* `local_software_release_root` -- root for local Software Releases paths in the SlapOS proxy, default to `/`.
Error cases: Error cases:
* `PathTooDeepError` when `base_directory` is too deep. Because of limitation * `PathTooDeepError` when `base_directory` is too deep. Because of limitation
...@@ -344,6 +347,7 @@ class StandaloneSlapOS(object): ...@@ -344,6 +347,7 @@ class StandaloneSlapOS(object):
self._server_ip = server_ip self._server_ip = server_ip
self._server_port = server_port self._server_port = server_port
self._master_url = "http://{server_ip}:{server_port}".format(**locals()) self._master_url = "http://{server_ip}:{server_port}".format(**locals())
self._local_software_release_root = local_software_release_root
self._base_directory = base_directory self._base_directory = base_directory
self._shared_part_list = list(shared_part_list) self._shared_part_list = list(shared_part_list)
......
This diff is collapsed.
PRAGMA foreign_keys=OFF;
BEGIN TRANSACTION;
CREATE TABLE software14 (
url VARCHAR(255),
computer_reference VARCHAR(255) DEFAULT 'computer',
requested_state VARCHAR(255) DEFAULT 'available',
CONSTRAINT uniq PRIMARY KEY (url, computer_reference)
);
INSERT INTO "software14" VALUES('/srv/slapgrid//srv//runner/project//slapos/software.cfg','computer','available');
CREATE TABLE computer14 (
reference VARCHAR(255) DEFAULT 'computer',
address VARCHAR(255),
netmask VARCHAR(255),
CONSTRAINT uniq PRIMARY KEY (reference)
);
INSERT INTO "computer14" VALUES('computer','127.0.0.1','255.255.255.255');
CREATE TABLE partition14 (
reference VARCHAR(255),
computer_reference VARCHAR(255) DEFAULT 'computer',
slap_state VARCHAR(255) DEFAULT 'free',
software_release VARCHAR(255),
xml TEXT,
connection_xml TEXT,
slave_instance_list TEXT,
software_type VARCHAR(255),
partition_reference VARCHAR(255), -- name of the instance
requested_by VARCHAR(255), -- only used for debugging,
-- slapproxy does not support proper scope
requested_state VARCHAR(255) NOT NULL DEFAULT 'started',
timestamp REAL,
CONSTRAINT uniq PRIMARY KEY (reference, computer_reference)
);
INSERT INTO "partition14" VALUES('slappart0','computer','busy','/srv/slapgrid//srv//runner/project//slapos/software.cfg','<?xml version=''1.0'' encoding=''utf-8''?>
<instance>
<parameter id="json">{
"site-id": "erp5"
}
}</parameter>
</instance>
',NULL,NULL,'production','slapos',NULL,'started',NULL);
INSERT INTO "partition14" VALUES('slappart1','computer','busy','/srv/slapgrid//srv//runner/project//slapos/software.cfg','<?xml version=''1.0'' encoding=''utf-8''?>
<instance/>
','<?xml version=''1.0'' encoding=''utf-8''?>
<instance>
<parameter id="url">mysql://127.0.0.1:45678/erp5</parameter>
</instance>
',NULL,'mariadb','MariaDB DataBase','slappart0','started',NULL);
INSERT INTO "partition14" VALUES('slappart2','computer','busy','/srv/slapgrid//srv//runner/project//slapos/software.cfg','<?xml version=''1.0'' encoding=''utf-8''?>
<instance>
<parameter id="cloudooo-json"></parameter>
</instance>
','<?xml version=''1.0'' encoding=''utf-8''?>
<instance>
<parameter id="url">cloudooo://127.0.0.1:23000/</parameter>
</instance>
',NULL,'cloudooo','Cloudooo','slappart0','started',NULL);
INSERT INTO "partition14" VALUES('slappart3','computer','busy','/srv/slapgrid//srv//runner/project//slapos/software.cfg','<?xml version=''1.0'' encoding=''utf-8''?>
<instance/>
','<?xml version=''1.0'' encoding=''utf-8''?>
<instance>
<parameter id="url">memcached://127.0.0.1:11000/</parameter>
</instance>
',NULL,'memcached','Memcached','slappart0','started',NULL);
INSERT INTO "partition14" VALUES('slappart4','computer','busy','/srv/slapgrid//srv//runner/project//slapos/software.cfg','<?xml version=''1.0'' encoding=''utf-8''?>
<instance/>
','<?xml version=''1.0'' encoding=''utf-8''?>
<instance>
<parameter id="url">memcached://127.0.0.1:13301/</parameter>
</instance>
',NULL,'kumofs','KumoFS','slappart0','started',NULL);
INSERT INTO "partition14" VALUES('slappart5','computer','busy','/srv/slapgrid//srv//runner/project//slapos/software.cfg','<?xml version=''1.0'' encoding=''utf-8''?>
<instance>
<parameter id="kumofs-url">memcached://127.0.0.1:13301/</parameter>
<parameter id="memcached-url">memcached://127.0.0.1:11000/</parameter>
<parameter id="cloudooo-url">cloudooo://127.0.0.1:23000/</parameter>
</instance>
','<?xml version=''1.0'' encoding=''utf-8''?>
<instance>
<parameter id="url">https://[fc00::1]:10001</parameter>
</instance>
',NULL,'tidstorage','TidStorage','slappart0','started',NULL);
INSERT INTO "partition14" VALUES('slappart6','computer','free',NULL,NULL,NULL,NULL,NULL,NULL,NULL,'started',NULL);
INSERT INTO "partition14" VALUES('slappart7','computer','free',NULL,NULL,NULL,NULL,NULL,NULL,NULL,'started',NULL);
INSERT INTO "partition14" VALUES('slappart8','computer','free',NULL,NULL,NULL,NULL,NULL,NULL,NULL,'started',NULL);
INSERT INTO "partition14" VALUES('slappart9','computer','free',NULL,NULL,NULL,NULL,NULL,NULL,NULL,'started',NULL);
CREATE TABLE slave14 (
reference VARCHAR(255), -- unique slave reference
computer_reference VARCHAR(255) DEFAULT 'computer',
connection_xml TEXT,
hosted_by VARCHAR(255),
asked_by VARCHAR(255) -- only used for debugging,
-- slapproxy does not support proper scope
);
CREATE TABLE partition_network14 (
partition_reference VARCHAR(255),
computer_reference VARCHAR(255) DEFAULT 'computer',
reference VARCHAR(255),
address VARCHAR(255),
netmask VARCHAR(255)
);
INSERT INTO "partition_network14" VALUES('slappart0','computer','slappart0','127.0.0.1','255.255.255.255');
INSERT INTO "partition_network14" VALUES('slappart0','computer','slappart0','fc00::1','ffff:ffff:ffff::');
INSERT INTO "partition_network14" VALUES('slappart1','computer','slappart1','127.0.0.1','255.255.255.255');
INSERT INTO "partition_network14" VALUES('slappart1','computer','slappart1','fc00::1','ffff:ffff:ffff::');
INSERT INTO "partition_network14" VALUES('slappart2','computer','slappart2','127.0.0.1','255.255.255.255');
INSERT INTO "partition_network14" VALUES('slappart2','computer','slappart2','fc00::1','ffff:ffff:ffff::');
INSERT INTO "partition_network14" VALUES('slappart3','computer','slappart3','127.0.0.1','255.255.255.255');
INSERT INTO "partition_network14" VALUES('slappart3','computer','slappart3','fc00::1','ffff:ffff:ffff::');
INSERT INTO "partition_network14" VALUES('slappart4','computer','slappart4','127.0.0.1','255.255.255.255');
INSERT INTO "partition_network14" VALUES('slappart4','computer','slappart4','fc00::1','ffff:ffff:ffff::');
INSERT INTO "partition_network14" VALUES('slappart5','computer','slappart5','127.0.0.1','255.255.255.255');
INSERT INTO "partition_network14" VALUES('slappart5','computer','slappart5','fc00::1','ffff:ffff:ffff::');
INSERT INTO "partition_network14" VALUES('slappart6','computer','slappart6','127.0.0.1','255.255.255.255');
INSERT INTO "partition_network14" VALUES('slappart6','computer','slappart6','fc00::1','ffff:ffff:ffff::');
INSERT INTO "partition_network14" VALUES('slappart7','computer','slappart7','127.0.0.1','255.255.255.255');
INSERT INTO "partition_network14" VALUES('slappart7','computer','slappart7','fc00::1','ffff:ffff:ffff::');
INSERT INTO "partition_network14" VALUES('slappart8','computer','slappart8','127.0.0.1','255.255.255.255');
INSERT INTO "partition_network14" VALUES('slappart8','computer','slappart8','fc00::1','ffff:ffff:ffff::');
INSERT INTO "partition_network14" VALUES('slappart9','computer','slappart9','127.0.0.1','255.255.255.255');
INSERT INTO "partition_network14" VALUES('slappart9','computer','slappart9','fc00::1','ffff:ffff:ffff::');
CREATE TABLE forwarded_partition_request14 (
partition_reference VARCHAR(255), -- a.k.a source_instance_id
master_url VARCHAR(255)
);
COMMIT;
...@@ -41,6 +41,13 @@ EOF ...@@ -41,6 +41,13 @@ EOF
sqlite3 ${TMPD}/proxy.db < $DUMP_BEFORE sqlite3 ${TMPD}/proxy.db < $DUMP_BEFORE
slapos proxy start --cfg ${TMPD}/slapos.cfg & slapos proxy start --cfg ${TMPD}/slapos.cfg &
# If you are running tests locally and you want to refer to the slapos being tested,
# you can use the test python executable in this way:
# cd ../../..
# python -m slapos.cli.entry proxy start --cfg ${TMPD}/slapos.cfg &
# cd -
SLAPOS_PROXY_PID=$! SLAPOS_PROXY_PID=$!
curl --silent --retry-connrefused --retry 3 http://127.0.0.1:${PORT}/getComputerInformation?computer_id=$COMPUTER_ID curl --silent --retry-connrefused --retry 3 http://127.0.0.1:${PORT}/getComputerInformation?computer_id=$COMPUTER_ID
......
[slapos] [slapos]
software_root = %(tempdir)s/opt/slapgrid software_root = %(rootdir)s/opt/slapgrid
instance_root = %(tempdir)s/srv/slapgrid instance_root = %(rootdir)s/srv/slapgrid
master_url = %(proxyaddr)s master_url = %(proxyaddr)s
computer_id = computer computer_id = computer
[slapproxy] [slapproxy]
host = 127.0.0.1 host = 127.0.0.1
port = 8080 port = 8080
database_uri = %(tempdir)s/lib/proxy.db database_uri = %(rootdir)s/lib/proxy.db
# Here goes the list of slapos masters that slapproxy can contact # Here goes the list of slapos masters that slapproxy can contact
# Each section beginning by multimaster is a different SlapOS Master, represented by arbitrary name. # Each section beginning by multimaster is a different SlapOS Master, represented by arbitrary name.
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment