Skip to content

slapos
slapos
Commit 08a696d4 authored by Thomas Gambier's avatar Thomas Gambier 🚴🏼
Browse files
Options

Use slapos.core with python3 

The egg slapos.core is now used with python3 by default (except in slaprunner).

See merge request nexedi/slapos!1154
parents b3b4beb2 eee21d6f
Hide whitespace changes
Inline Side-by-side
Showing with 28 additions and 205 deletions
component/firewalld/buildout.cfg
View file @ 08a696d4
......@@ -60,7 +60,7 @@ environment =
DBUS_GLIB_CFLAGS=-I${dbus-glib:location}/include/dbus-1.0
DBUS_GLIB_LIBS=-L${dbus-glib:location}/lib -ldbus-glib-1
CPPFLAGS=-I${glib:location}/include/glib-2.0 -I${glib:location}/lib/glib-2.0/include
LDFLAGS=-L${glib:location}/lib -Wl,-rpath=${glib:location}/lib
LDFLAGS=-L${glib:location}/lib -Wl,-rpath=${glib:location}/lib -L${gettext:location}/lib -Wl,-rpath=${gettext:location}/lib
[gobject-introspection]
recipe = slapos.recipe.cmmi
......
component/python-slip/buildout.cfg
View file @ 08a696d4
......@@ -6,27 +6,19 @@ parts =
[python-slip]
# BUG: This section does FS changes outside location.
recipe = slapos.recipe.cmmi
url = https://github.com/nphilipp/python-slip/releases/download/python-slip-0.6.1/python-slip-0.6.1.tar.bz2
md5sum = a6d8ee96245fc21785d4c1c062c85f2f
url = https://github.com/nphilipp/python-slip/releases/download/python-slip-0.6.5/python-slip-0.6.5.tar.bz2
md5sum = 28ae5f93853466c44ec96706ba2a1eb4
configure-command = true
make-targets =
make-binary =
make all
cp -ax . ${:slip}/
cp -ax . ${:slip.dbus}/
cp -ax . ${:slip.gtk}/
sed -i '/setup(name="slip.dbus"/,/"gtk", "pango"])/d' ${:slip}/setup.py
sed -i '/setup(name="slip"/,/selinux"])/d' ${:slip.dbus}/setup.py
sed -i '/if sys.version_info.major/,/"gtk", "pango"])/d' ${:slip.dbus}/setup.py
sed -i '/setup(name="slip"/,/xml.etree.ElementTree"])/d' ${:slip.gtk}/setup.py
slip = ${buildout:parts-directory}/slip
slip.dbus = ${buildout:parts-directory}/slip.dbus
slip.gtk = ${buildout:parts-directory}/slip.gtk
[slip.gtk-develop]
recipe = zc.recipe.egg:develop
setup = ${python-slip:slip.gtk}
egg-name = slip.gtk
[slip.dbus-develop]
recipe = zc.recipe.egg:develop
......@@ -43,5 +35,4 @@ recipe = zc.recipe.egg
eggs =
${slip-develop:egg-name}
${slip.dbus-develop:egg-name}
${slip.gtk-develop:egg-name}
component/slapos/obs.cfg
View file @ 08a696d4
......@@ -26,13 +26,14 @@ extends-cache = extends-cache
download-cache = download-cache
[python]
part = python3
init +=
buildout = self.buildout['buildout']
assert buildout['directory'] == buildout['destdir'] + buildout['rootdir'], (
"Buildout MUST BE launched in destdir/rootdir (currently launched in %s but should be launched in %s)",
buildout['directory'], buildout['destdir'] + buildout['rootdir'])
[python2.7]
[python3-common]
configure-options +=
--prefix=${buildout:rootdir}/parts/${:_buildout_section_name_}
environment +=
......@@ -86,8 +87,8 @@ pre-configure =
environment +=
LD_LIBRARY_PATH=${dbus:location}/lib
PYTHON=${python:location}/bin/python${python:version}
PYTHON_INCLUDES=-I${python:location}/include/python${python:version}
PYTHON_LIBS=-L${python:location}/lib -lpython${python:version} -lpthread -ldl -lutil -lm
PYTHON_INCLUDES=-I${python:location}/include/python${python:version}m
PYTHON_LIBS=-L${python:location}/lib -lpython${python:version}m -lpthread -ldl -lutil -lm
post-install =
sed -i 's#${dbus:location}/lib/libdbus-1.la#/opt/slapos/parts/dbus/lib/libdbus-1.la#' ${dbus-glib:location}/lib/libdbus-glib-1.la
......
software/erp5testnode/software.cfg
View file @ 08a696d4
......@@ -22,10 +22,13 @@ parts =
git
apache
[python]
part = python3
[eggs]
recipe = zc.recipe.egg
eggs =
erp5.util[testnode,benchmark,scalability_tester]
erp5.util[testnode,benchmark]
${lxml-python:egg}
......@@ -47,14 +50,13 @@ PyXML = 0.8.5
WebOb = 1.8.5
WebTest = 2.0.33
soupsieve = 1.8
#gitdb2 = 2.0.4
#smmap2 = 2.0.4
waitress = 1.4.4
z3c.etestbrowser = 2.0.1
z3c.etestbrowser = 3.0.1
zope.testbrowser = 5.3.2
WSGIProxy2 = 0.4.6
beautifulsoup4 = 4.7.1
zope.cachedescriptors = 4.3.1
zope.event = 4.4
zope.schema = 4.9.3
slapos.tool.nosqltester = 0.0.4.dev-r45972
zope.deferredimport = 4.3.1
zope.proxy = 4.3.5
software/erp5testnode/testsuite/instance-kvm-resilient-test.cfg.jinja2
View file @ 08a696d4
......@@ -44,7 +44,7 @@ recipe = slapos.cookbook:request
software-url = ${slap-connection:software-release-url}
software-type = kvm-resilient
name = Resilient Instance (Root Instance)
{% for key, value in slapparameter_dict.get('cluster', {}).iteritems() -%}
{% for key, value in slapparameter_dict.get('cluster', {}).items() -%}
config-{{ key }} = {{ dumps(value) }}
{% endfor -%}
config-virtual-hard-drive-url = ${slap-parameter:virtual-hard-drive-url}
......
software/erp5testnode/testsuite/instance-resilient-test.cfg.jinja2
View file @ 08a696d4
......@@ -43,7 +43,7 @@ recipe = slapos.cookbook:request
software-url = ${slap-connection:software-release-url}
software-type = resilient
name = Resilient Instance (Root Instance)
{% for key, value in slapparameter_dict.get('cluster', {}).iteritems() -%}
{% for key, value in slapparameter_dict.get('cluster', {}).items() -%}
config-{{ key }} = {{ dumps(value) }}
{% endfor -%}
config-resiliency-backup-periodicity = */10 * * * *
......
software/erp5testnode/testsuite/kvm.cfg
View file @ 08a696d4
......@@ -12,7 +12,7 @@ md5sum = dd82c771f6f7738fb4b0fc1330ed8236
[template-resilient-test]
filename = instance-kvm-resilient-test.cfg.jinja2
md5sum = 71ddbdeb8769bcb0ebb3c9407ef7e36c
md5sum = 5f255502973530181ee45d93fa00a3c4
# Ingest extra-context, on a the final template-resilient-test rendering
# always ingest raw values.
extra-context =
......
software/erp5testnode/testsuite/slaprunner.cfg
View file @ 08a696d4
......@@ -7,7 +7,7 @@ parts += template-erp5testnode
[template-resilient-test]
filename = instance-resilient-test.cfg.jinja2
md5sum = be4adbc196f07f5e39ca78401dfa53ec
md5sum = e4b04aa6fd3413bc6ae38823abdcc8fa
# We have to use an extra level of indentation here because this is substituted
# during software buildout to generate instance buildout, but the
......
software/slapos-sr-testing/software-py3.cfg
View file @ 08a696d4
......@@ -26,3 +26,5 @@ extra =
headless-chromium ${slapos.test.headless-chromium-setup:setup}
hugo ${slapos.test.hugo-setup:setup}
matomo ${slapos.test.matomo-setup:setup}
erp5testnode ${slapos.test.erp5testnode-setup:setup}
theia ${slapos.test.theia-setup:setup}
software/slapos-sr-testing/software.cfg
View file @ 08a696d4
......@@ -351,8 +351,6 @@ tests =
# here, to check there's no promise issue when slapos node runs with Python 2.
fluentd ${slapos.test.fluentd-setup:setup}
kvm ${slapos.test.kvm-setup:setup}
slaprunner ${slapos.test.slaprunner-setup:setup}
theia ${slapos.test.theia-setup:setup}
metabase ${slapos.test.metabase-setup:setup}
erp5 ${slapos.test.erp5-setup:setup}
###
......@@ -379,8 +377,8 @@ extra =
jscrawler ${slapos.test.jscrawler-setup:setup}
html5as ${slapos.test.html5as-setup:setup}
html5as-base ${slapos.test.html5as-base-setup:setup}
erp5testnode ${slapos.test.erp5testnode-setup:setup}
beremiz-ide ${slapos.test.beremiz-ide-setup:setup}
slaprunner ${slapos.test.slaprunner-setup:setup}
[versions]
# slapos.core is used from the clone always
......
software/slaprunner/README.rst
View file @ 08a696d4
......@@ -7,153 +7,7 @@ Introduction
This software release is used to deploy Slaprunner instances.
Slaprunner is an all-in-one IDE used to develop and test profiles and recipes for SlapOS.
You can learn how to use it here :
http://community.slapos.org/wiki/slapos-Wiki.Home/developer-Lecture.Web.Runner.Extended
This software is only running in Python2 and has been deprecated in favor of theia.
See https://lab.nexedi.com/nexedi/slapos/tree/master/software/theia
Development
-----------
Here is an exemple of parameter XML to develop the slaprunner into another slaprunner :
<?xml version="1.0" encoding="utf-8"?>
<instance>
<parameter id="runner-importer-sshd-port">22224</parameter>
<parameter id="runner-sshd-port">22223</parameter>
<parameter id="no-ipv4-frontend">true</parameter>
</instance>
Parameters
----------
user-authorized-key
~~~~~~~~~~~~~~
You need to specify your SSH public key in order to connect to the SSH server of this instance.
Example of parameter XML::
<?xml version="1.0" encoding="utf-8"?>
<instance>
<parameter id="user-authorized-key">ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCdNp7qZMVAzNc07opHshiIFDmJpYXQpetfcSgUj39a409d42PpsJElp7WsAE/x0nN6gUIoWIl7UiAlMzf6bKEJGJVSOZEPTmiJVlgK1Gp+kE0x9yNcncYg7p38Jny0daVA/NkkpAFyRsAm5kLGzyLtaCcktSvy0cJuy7WSSHU05pd1f8Y8thofE9g5t+/JA2VZvipxPkRfkFAG3aOAGLULlTImTSDFSDFGSDFG5F6mMnl7yvY2d6vEHVBu+K+aKmAwZVfCUwtSpa/tq3i2Lppjrw3UfrxbQSFHZCkzefr+u+l4YYe+tJrX7rYJYXD7LIfZfdSeFTlHFaN/yI1 user@host.local</parameter>
</instance>
instance-amount
~~~~~~~~~~~~~~~
Optional parameter.
Default: 10
Public-directory
----------------
You can serve static files with the webrunner. For that, just put your data in "srv/runner/public". All these files will be served throught the url of the webrunner + "/public/". Useful for developping your own static website.
AUTO-DEPLOYMENT
---------------
for software
~~~~~~~~~~~~
You can automatically deploy a software release while deploying the webrunner itself, using the paramater XML.
To do this, you only need to pass as a parameter named "slapos-software" : "AAA/BBB", where AAA is the folder of slapos.git where is located your BBB software.
For example, to deploy the hello-world software, you need to pass : software/helloworld
This is possible because the slapos.git is automatically downloaded when the webrunner is deployed.
It is also possible to download you own git repository, by providing the url in the "slapos-repository" parameter.
Last but not least, it is also possible to switch the branch with the parameter "slapos-reference" (by default pointing on master)
for instance
~~~~~~~~~~~~
The parameter "auto-deploy-instance" can be explicitly set to allow or prevent the runner to deploy the instance at START TIME (if you manually restart the runner, or if the server reboots). Values : "true" or "false". Default value is "true", except for the instances of import (while type is resilient or test) which is "false"
There also exists the parameter "autorun", which will build&run your software if set to true. For this, you need "auto_deploy" to true, and set the parameter "slapos-software" to the software you want to deploy. Do not hesitate to clone a different repo than "slapos", or to change the tag/branch to use your custom Software Release. (see "slapos-repository" and "slapos-reference" in previous section).
To deploy the instance with some parameters, just give to the runner parameters starting with "parameter-", they will be correctly forwarded to the instance, which will use them for its configuration. For example, if you want to send to the sofware helloworld the parameter "name" with the value "nicolas", here is how to configure the parameter.xml of the webrunner for auto-depolyment :
<?xml version='1.0' encoding='utf-8'?>
<instance>
<parameter id="slapos-software">software/helloworld</parameter>
<parameter id="auto_deploy">true</parameter>
<parameter id="autorun">true</parameter>
<parameter id="parameter-name">nicolas</parameter>
</instance>
Resilience :
------------
To order a resilient runner, you have to request a runner with the option: "--type resilient"
You can then decide on which node you want to deploy each instance, with the parameters.xml :
<?xml version='1.0' encoding='utf-8'?>
<instance>
<parameter id="-sla-runner2-computer_guid">COMP-XXXX</parameter>
<parameter id="-sla-pbs2-computer_guid">COMP-XXXX</parameter>
<parameter id="-sla-pbs1-computer_guid">COMP-XXXX</parameter>
<parameter id="-sla-runner1-computer_guid">COMP-XXXX</parameter>
<parameter id="-sla-runner0-computer_guid">COMP-XXXX</parameter>
</instance>
If you want to check by yourself that the pull-backup instances do their job, you can change directory to the slappart of runner0, and run ./bin/exporter (after creating your account, using the given backend_url or url): it would fill ./srv/backup/runner with data. If you then go to an import instance (runner1 or runner2) on the port 50005, you should be able to sign in the runner.
Tips :
~~~~~
To find in which partition the instance has been deployed, you can open the page of this specific instance, and look for "slappartXX" on the page.
Tests :
-------
To test the runner, start by ordering a runner with default type. Then READ the important note below :
/!\ IMPORTANT NOTE ON THE TESTS /!\ : in order to make them work, you have to comment the last line of the file __init__.py in the runner module (which is just : "run()"). Indeed, this line is used to load the config, when importing this module throught Gunicorn (the wsgi server). But the test suite loads itself the configuration, in fonction of the tested scenarios, etc... (You can find the module in the SR folder directory, under ./eggs/slapos.toolbox, or under parts/slapos.toolbox if you have a development version)
Then, su in the concerned slapuserX and run "./bin/runTestSuite".
For the tests on the resiliency of the webrunner, please refer to the README in slapos.toolbox.git/slapos/resiliencytest
Request custom frontend :
-------------------------
While deploying a server in your instance, you may need to ask for a custom ipv4 frontend.
The way to do that is to send a new parameter to your runner instance, using the parameters XML. The name of it is "custom-frontend-backend-url".
To create the frontend, you now have to wait the slaprunner to be processed.
You can define the type of your backend using "custom-frontend-backend-type". eg: zope
If you deploy a server, which uses basic auth, you also have to declare the parameter "custom-frontend-basic-auth" as true, or your slaprunner instance won't show as correctly instantiated.
Example :
<?xml version='1.0' encoding='utf-8'?>
<instance>
<parameter id="custom-frontend-backend-url">http(s)://[THE_IPV6_ADDRESS]:PORT</parameter>
<parameter id="custom-frontend-backend-type">zope</parameter>
<parameter id="custom-frontend-basic-auth">true</parameter>
</instance>
Things to notice for the nex developer :
----------------------------------------
As you can see in instance-runner-*.cfg, the buildout section extends a hard-coded template file. If one day you need to modify the filename, do not forget to modify it in instance.cfg, but also in these files ! (the problem is that the content of instance.cfg is not known by buildout while the deployment of the software release)
List of ports used by the webrunner:
------------------------------------
8602 : slapproxy, while running tests
9684 : apache (monitoring of slaprunner, git access)
22222 : dropbear
39986 : supervisord
50000 : slapproxy
50005 : webrunner (flask app), webdav access
Tips:
-----
You can use shellinabox in fullscreen, by accessing : https://[IPV6]:50005/shellinabox/
software/slaprunner/software-py3.cfg deleted 100644 → 0
View file @ b3b4beb2
[buildout]
extends =
software.cfg
[python]
part = python3
[python-with-eggs]
eggs -=
# futures is a backport of Py3's concurrent.futures module
futures
software/slaprunner/test/test.py
View file @ 08a696d4
......@@ -46,12 +46,9 @@ from slapos.recipe.librecipe import generateHashFromFiles
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
from slapos.util import bytes2str
skipIfPython3 = unittest.skipIf(six.PY3, 'rdiff-backup is not compatible with Python 3 yet')
setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
os.path.abspath(
os.path.join(os.path.dirname(__file__), '..',
'software%s.cfg' % ("-py3" if six.PY3 else ""))))
os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
class SlaprunnerTestCase(SlapOSInstanceTestCase):
# Slaprunner uses unix sockets, so it needs short paths.
......@@ -499,7 +496,6 @@ class TestCustomFrontend(SlaprunnerTestCase):
parameter_dict['custom-frontend-url'],
'https://www.erp5.com')
@skipIfPython3
class TestResilientInstance(SlaprunnerTestCase):
instance_max_retry = 20
......@@ -526,14 +522,12 @@ class TestResilientInstance(SlaprunnerTestCase):
'url',
'webdav-url']))
@skipIfPython3
class TestResilientCustomFrontend(TestCustomFrontend):
instance_max_retry = 20
@classmethod
def getInstanceSoftwareType(cls):
return 'resilient'
@skipIfPython3
class TestResilientWebInstance(TestWeb):
instance_max_retry = 20
@classmethod
......@@ -544,7 +538,6 @@ class TestResilientWebInstance(TestWeb):
pass # Disable until we can write on runner0 rather them
# on root partition
@skipIfPython3
class TestResilientWebrunnerBasicUsage(TestWebRunnerBasicUsage):
instance_max_retry = 20
@classmethod
......@@ -552,14 +545,12 @@ class TestResilientWebrunnerBasicUsage(TestWebRunnerBasicUsage):
return 'resilient'
@skipIfPython3
class TestResilientWebrunnerAutorun(TestWebRunnerAutorun):
instance_max_retry = 20
@classmethod
def getInstanceSoftwareType(cls):
return 'resilient'
@skipIfPython3
class TestResilientDummyInstance(SlaprunnerTestCase):
instance_max_retry = 20
@classmethod
......
software/theia/software-py3.cfg deleted 100644 → 0
View file @ b3b4beb2
[buildout]
extends =
software.cfg
[python]
part = python3
software/theia/software.cfg
View file @ 08a696d4
......@@ -37,6 +37,8 @@ shared-part-list =
# Versions
# --------
[python]
part = python3
[gcc]
# We keep the gcc part in sync with the one from erp5 software, so that when we install
......
software/theia/test/test.py
View file @ 08a696d4
......@@ -44,8 +44,7 @@ from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass, SlapOSNodeC
from slapos.grid.svcbackend import getSupervisorRPC, _getSupervisordSocketPath
software_cfg = 'software%s.cfg' % ('-py3' if six.PY3 else '')
theia_software_release_url = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', software_cfg))
theia_software_release_url = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'software.cfg'))
setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(theia_software_release_url)
......
stack/slapos.cfg
View file @ 08a696d4
......@@ -223,7 +223,7 @@ atomize = 0.2.0
croniter = 0.3.25
dnspython = 1.16.0
enum34 = 1.1.10
erp5.util = 0.4.72
erp5.util = 0.4.73
feedparser = 5.2.1
functools32 = 3.2.3.post2
attrs = 18.2.0
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7