Commit 589943c2 authored by Kirill Smelkov's avatar Kirill Smelkov

Merge branch 'master' into master+ZODB4-wc2

* master:
  Fix "kvm: Increase default VM parameters to match nowadays requirement"
  kvm: Merge tests
  kvm: Allow to specify network adapter
  software/slapos-sr-testing: install new psycopg2 dependency
  recipe/postgres: support non standard port
  recipe/test_postgres: test with a real postgres server
  software/slapos-testing: install psycopg2 for slapos.cookbook
  software/slapos-testing: make postgresql available for tests
  component/psycopg2: new component
  golang: Switch default to Go1.17
  erp5testnode: update deploy script
  stack/slapos: slapos.cookbook version up (1.0.214)
  Release slapos.cookbook (1.0.214)
  test/recipe: update tests for postgres recipe regarding password update
  recipe/postgres: support changing user password after creation
  component/nxdtest: v↑
  golang: v↑ go1.16 (1.16.8 -> 1.16.10)
  golang: v↑ go1.17 (1.17.1 -> 1.17.3)
parents b88dd5bf 0adc38b3
...@@ -46,8 +46,8 @@ environment-extra = ...@@ -46,8 +46,8 @@ environment-extra =
[golang1.16] [golang1.16]
<= golang-common <= golang-common
url = https://golang.org/dl/go1.16.8.src.tar.gz url = https://golang.org/dl/go1.16.10.src.tar.gz
md5sum = 92e69a5e1bb6ea5e7498d12d03160032 md5sum = 49f0a54f0bdcba297bac194d8dafe431
# go1.16 needs go1.4 to bootstrap # go1.16 needs go1.4 to bootstrap
environment-extra = environment-extra =
...@@ -55,8 +55,8 @@ environment-extra = ...@@ -55,8 +55,8 @@ environment-extra =
[golang1.17] [golang1.17]
<= golang-common <= golang-common
url = https://golang.org/dl/go1.17.1.src.tar.gz url = https://golang.org/dl/go1.17.3.src.tar.gz
md5sum = a78205838c2a7054522cb91c12982f26 md5sum = 3ea82e5966275f405f0db4f52511bb6e
# go1.17 needs go1.4 to bootstrap # go1.17 needs go1.4 to bootstrap
environment-extra = environment-extra =
...@@ -122,7 +122,7 @@ bin = ${gowork.dir:bin} ...@@ -122,7 +122,7 @@ bin = ${gowork.dir:bin}
depends = ${gowork.goinstall:recipe} depends = ${gowork.goinstall:recipe}
# go version used for the workspace (possible to override in applications) # go version used for the workspace (possible to override in applications)
golang = ${golang1.16:location} golang = ${golang1.17:location}
# no special build flags by default # no special build flags by default
buildflags = buildflags =
......
...@@ -20,6 +20,6 @@ egg = nxdtest ...@@ -20,6 +20,6 @@ egg = nxdtest
[nxdtest-repository] [nxdtest-repository]
recipe = slapos.recipe.build:gitclone recipe = slapos.recipe.build:gitclone
repository = https://lab.nexedi.com/nexedi/nxdtest.git repository = https://lab.nexedi.com/nexedi/nxdtest.git
revision = 0ad45a9c revision = 9f413221
location = ${buildout:parts-directory}/nxdtest location = ${buildout:parts-directory}/nxdtest
git-executable = ${git:location}/bin/git git-executable = ${git:location}/bin/git
[buildout]
extends =
../postgresql/buildout.cfg
[psycopg2-env]
PATH = ${postgresql:location}/bin:%(PATH)s
[psycopg2]
recipe = zc.recipe.egg:custom
egg = psycopg2
define = PSYCOPG_EXTENSIONS,PSYCOPG_DISPLAY_SIZE,PSYCOPG_NEW_BOOLEAN,HAVE_PQFREEMEM,HAVE_PQPROTOCOL3
environment = psycopg2-env
include-dirs =
${postgresql:location}/include
library-dirs =
${postgresql:location}/lib
rpath =
${postgresql:location}/lib
...@@ -28,7 +28,7 @@ from setuptools import setup, find_packages ...@@ -28,7 +28,7 @@ from setuptools import setup, find_packages
import glob import glob
import os import os
version = '1.0.213' version = '1.0.214'
name = 'slapos.cookbook' name = 'slapos.cookbook'
long_description = open("README.rst").read() long_description = open("README.rst").read()
...@@ -39,6 +39,7 @@ extras_require = { ...@@ -39,6 +39,7 @@ extras_require = {
'test': ( 'test': (
'jsonschema', 'jsonschema',
'mock', 'mock',
'psycopg2',
'testfixtures', 'testfixtures',
'requests', 'requests',
), ),
......
...@@ -104,6 +104,7 @@ class Recipe(GenericBaseRecipe): ...@@ -104,6 +104,7 @@ class Recipe(GenericBaseRecipe):
else: else:
paths.extend(self.createConfig()) paths.extend(self.createConfig())
paths.extend(self.createRunScript()) paths.extend(self.createRunScript())
self.updateSuperuser()
return paths return paths
...@@ -146,6 +147,7 @@ class Recipe(GenericBaseRecipe): ...@@ -146,6 +147,7 @@ class Recipe(GenericBaseRecipe):
with open(postgres_conf, 'w') as cfg: with open(postgres_conf, 'w') as cfg:
cfg.write(textwrap.dedent("""\ cfg.write(textwrap.dedent("""\
listen_addresses = '%s' listen_addresses = '%s'
port = %s
logging_collector = on logging_collector = on
log_rotation_size = 50MB log_rotation_size = 50MB
max_connections = 100 max_connections = 100
...@@ -161,6 +163,7 @@ class Recipe(GenericBaseRecipe): ...@@ -161,6 +163,7 @@ class Recipe(GenericBaseRecipe):
unix_socket_permissions = 0700 unix_socket_permissions = 0700
""" % ( """ % (
','.join(set(ipv4).union(ipv6)), ','.join(set(ipv4).union(ipv6)),
self.options['port'],
pgdata, pgdata,
))) )))
...@@ -205,9 +208,25 @@ class Recipe(GenericBaseRecipe): ...@@ -205,9 +208,25 @@ class Recipe(GenericBaseRecipe):
# encrypt the password to avoid storing in the logs # encrypt the password to avoid storing in the logs
enc_password = 'md5' + hashlib.md5((password + user).encode()).hexdigest() enc_password = 'md5' + hashlib.md5((password + user).encode()).hexdigest()
change_password_query = """ALTER USER "%s" ENCRYPTED PASSWORD '%s'""" % (user, enc_password)
self.runPostgresCommand(cmd="""ALTER USER "%s" ENCRYPTED PASSWORD '%s'""" % (user, enc_password)) pgdata = self.options['pgdata-directory']
if os.path.exists(os.path.join(pgdata, 'postmaster.pid')):
psql_binary = os.path.join(self.options['bin'], 'psql')
# connect to a running postgres deamon
p = subprocess.Popen([
psql_binary,
'-h', pgdata,
'-p', self.options['port'],
'-U', user,
'-d', self.options['dbname'],
],
stdin=subprocess.PIPE)
p.communicate((change_password_query + '\n').encode())
if p.returncode:
raise UserError("Error updating password")
else:
self.runPostgresCommand(cmd=change_password_query)
def runPostgresCommand(self, cmd): def runPostgresCommand(self, cmd):
"""\ """\
......
import unittest import os
import tempfile
import shutil import shutil
import os.path import tempfile
import textwrap
import time
import unittest
try:
import subprocess32 as subprocess
except ImportError:
import subprocess
import psycopg2
import zc.buildout.testing import zc.buildout.testing
class PostgresTest(unittest.TestCase): class PostgresTest(unittest.TestCase):
ipv4 = os.environ['SLAPOS_TEST_IPV4']
ipv6 = os.environ['SLAPOS_TEST_IPV6']
port = 5432
def setUp(self): def setUp(self):
self.buildout = buildout = zc.buildout.testing.Buildout() self.buildout = buildout = zc.buildout.testing.Buildout()
self.pgdata_directory = tempfile.mkdtemp() self.pgdata_directory = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, self.pgdata_directory) self.addCleanup(shutil.rmtree, self.pgdata_directory)
self.services_directory = tempfile.mkdtemp() self.services_directory = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, self.services_directory) self.addCleanup(shutil.rmtree, self.services_directory)
self.postgres_bin_directory = os.environ['SLAPOS_TEST_POSTGRESQL_PREFIX'] + '/bin'
buildout['postgres'] = { buildout['postgres'] = {
'bin': 'software/parts/postgres/bin/', 'bin': self.postgres_bin_directory,
'dbname': 'dbname', 'dbname': 'dbname',
'ipv4': '127.0.0.1', 'ipv4': self.ipv4,
'ipv6': '::1', 'ipv6': self.ipv6,
'port': '5443', 'port': self.port,
'pgdata-directory': self.pgdata_directory, 'pgdata-directory': os.path.join(self.pgdata_directory, 'pgdata'),
'services': self.services_directory, 'services': self.services_directory,
'superuser': 'superuser', 'superuser': 'superuser',
'password': 'secret', 'password': 'secret',
...@@ -29,21 +44,73 @@ class PostgresTest(unittest.TestCase): ...@@ -29,21 +44,73 @@ class PostgresTest(unittest.TestCase):
'postgres', 'postgres',
buildout['postgres']) buildout['postgres'])
def start_postgres_server(self):
server_process = subprocess.Popen(
[ os.path.join(self.services_directory, 'postgres-start') ],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
self.addCleanup(server_process.wait)
self.addCleanup(server_process.terminate)
# wait for server to accept connections
for i in range(60):
time.sleep(i)
try:
psycopg2.connect(self.buildout['postgres']['url']).close()
except psycopg2.OperationalError as e:
pass
else:
break
def test_options(self): def test_options(self):
self.assertEqual( self.assertEqual(
'postgresql://superuser:secret@[::1]:5443/dbname', self.buildout['postgres']['url'],
self.buildout['postgres']['url']) 'postgresql://superuser:secret@[{self.ipv6}]:{self.port}/dbname'.format(self=self),
)
def test_install(self): def test_install(self):
installed = self.recipe.install() installed = self.recipe.install()
self.assertIn('postgresql.conf', os.listdir(self.pgdata_directory)) pgdata_directory = os.path.join(self.pgdata_directory, 'pgdata')
self.assertIn('pg_hba.conf', os.listdir(self.pgdata_directory)) self.assertIn('postgresql.conf', os.listdir(pgdata_directory))
self.assertIn('pg_hba.conf', os.listdir(pgdata_directory))
self.assertIn('postgres-start', os.listdir(self.services_directory)) self.assertIn('postgres-start', os.listdir(self.services_directory))
self.assertEqual( self.assertEqual(
sorted(installed), sorted(installed),
sorted([ sorted([
os.path.join(self.pgdata_directory, 'postgresql.conf'), os.path.join(pgdata_directory, 'postgresql.conf'),
os.path.join(self.pgdata_directory, 'pg_hba.conf'), os.path.join(pgdata_directory, 'pg_hba.conf'),
os.path.join(self.services_directory, 'postgres-start')])) os.path.join(self.services_directory, 'postgres-start')]))
self.start_postgres_server()
with psycopg2.connect(self.buildout['postgres']['url']) as cnx:
with cnx.cursor() as cursor:
cursor.execute("SELECT 1+1")
self.assertEqual(cursor.fetchone(), (2,))
cnx.close()
def test_update_password(self):
self.recipe.install()
self.start_postgres_server()
self.recipe.options['password'] = 'new'
self.recipe.install()
dsn = self.buildout['postgres']['url']
with psycopg2.connect(psycopg2.extensions.make_dsn(dsn, password='new')) as cnx:
with cnx.cursor() as cursor:
cursor.execute("SELECT 1+1")
self.assertEqual(cursor.fetchone(), (2,))
cnx.close()
# old password can no longer connect
with self.assertRaisesRegexp(
psycopg2.OperationalError,
'password authentication failed'
):
psycopg2.connect(dsn)
class PostgresTestNonStandardPort(PostgresTest):
port = 5433
...@@ -237,7 +237,7 @@ function GetOSVersion { ...@@ -237,7 +237,7 @@ function GetOSVersion {
os_RELEASE=$(lsb_release -r -s) os_RELEASE=$(lsb_release -r -s)
os_UPDATE="" os_UPDATE=""
os_PACKAGE="rpm" os_PACKAGE="rpm"
if [[ "Debian,Ubuntu,LinuxMint" =~ $os_VENDOR ]]; then if [[ "Debian,Ubuntu,LinuxMint,Raspbian" =~ $os_VENDOR ]]; then
os_PACKAGE="deb" os_PACKAGE="deb"
elif [[ "SUSE LINUX" =~ $os_VENDOR ]]; then elif [[ "SUSE LINUX" =~ $os_VENDOR ]]; then
lsb_release -d -s | grep -q openSUSE lsb_release -d -s | grep -q openSUSE
...@@ -1036,6 +1036,7 @@ function update_package_repo { ...@@ -1036,6 +1036,7 @@ function update_package_repo {
if [[ "$REPOS_UPDATED" != "True" || "$RETRY_UPDATE" = "True" ]]; then if [[ "$REPOS_UPDATED" != "True" || "$RETRY_UPDATE" = "True" ]]; then
# if there are transient errors pulling the updates, that's fine. # if there are transient errors pulling the updates, that's fine.
# It may be secondary repositories that we don't really care about. # It may be secondary repositories that we don't really care about.
apt_get update --allow-releaseinfo-change || /bin/true
apt_get update || /bin/true apt_get update || /bin/true
REPOS_UPDATED=True REPOS_UPDATED=True
fi fi
...@@ -1927,6 +1928,7 @@ $XTRACE ...@@ -1927,6 +1928,7 @@ $XTRACE
# mode: shell-script # mode: shell-script
# End: # End:
if [[ $EUID -gt 0 ]]; then if [[ $EUID -gt 0 ]]; then
echo "####################################################" echo "####################################################"
echo "# #" echo "# #"
...@@ -1969,7 +1971,7 @@ fi ...@@ -1969,7 +1971,7 @@ fi
# Warn users who aren't on an explicitly supported distro, but allow them to # Warn users who aren't on an explicitly supported distro, but allow them to
# override check and attempt installation with ``export FORCE=yes`` # override check and attempt installation with ``export FORCE=yes``
if [[ ! ${DISTRO} =~ (wheezy|jessie|trusty|vivid|xenial|rhel7|rhel6|LinuxMint-17.2) ]]; then if [[ ! ${DISTRO} =~ (jessie|stretch|buster|bullseye|xenial|bionic|focal|rhel7) ]]; then
echo "WARNING: this script has not been tested on $DISTRO" echo "WARNING: this script has not been tested on $DISTRO"
if [[ "$FORCE" != "yes" ]]; then if [[ "$FORCE" != "yes" ]]; then
die $LINENO "If you wish to run this script anyway run with FORCE=yes" die $LINENO "If you wish to run this script anyway run with FORCE=yes"
...@@ -1978,10 +1980,10 @@ fi ...@@ -1978,10 +1980,10 @@ fi
# Make sure wheezy backports are available. # Make sure wheezy backports are available.
if [[ $DISTRO == "wheezy" ]]; then if [[ $DISTRO == "wheezy" ]]; then
echo "deb http://ftp.debian.org/debian wheezy-backports main contrib " > /etc/apt/sources.list.d/wheezy-backports.list echo "deb http://archive.debian.org/debian wheezy-backports main contrib " > /etc/apt/sources.list.d/wheezy-backports.list
fi fi
if is_fedora && [[ $DISTRO == "rhel7" ]]; then if is_fedora && [[ $DISTRO =~ rhel7|f20|f19 ]]; then
# RHEL requires EPEL for many Ansible dependencies # RHEL requires EPEL for many Ansible dependencies
# NOTE: We always remove and install latest -- some environments # NOTE: We always remove and install latest -- some environments
...@@ -2007,6 +2009,8 @@ failovermethod=priority ...@@ -2007,6 +2009,8 @@ failovermethod=priority
enabled=0 enabled=0
gpgcheck=0 gpgcheck=0
EOF EOF
yum clean all
yum makecache
yum update -y yum update -y
yum_install yum-utils yum_install yum-utils
# Enable a bootstrap repo. It is removed after finishing # Enable a bootstrap repo. It is removed after finishing
...@@ -2051,18 +2055,26 @@ EOF ...@@ -2051,18 +2055,26 @@ EOF
is_package_installed libselinux-python || install_package libselinux-python is_package_installed libselinux-python || install_package libselinux-python
fi fi
is_package_installed unzip || install_package unzip
is_package_installed openssl || install_package openssl is_package_installed openssl || install_package openssl
# stick to a specific ansible package version is_package_installed ansible || install_package ansible
if is_ubuntu; then
# is_ubuntu also handle debian distro if ! is_ubuntu || [[ $DISTRO = wheezy || $DISTRO == jessie || $DISTRO == stretch || $DISTRO == xenial || $DISTRO == bionic ]]; then
is_package_installed ansible || install_package ansible=1.* is_package_installed python-pip || install_package python-pip
elif is_fedora; then
# is_fedora also handle CentOS distro
is_package_installed ansible || install_package ansible1.9.noarch
else else
is_package_installed ansible || install_package ansible # Ansible is using python3 now
is_package_installed python3-pip || install_package python3-pip
fi
if is_ubuntu && [[ $DISTRO == focal || $DISTRO == buster ]]; then
# install python2 in distro where python2 is not installed by default and ansible python interpreter is still python2
is_package_installed python || install_package python
fi
if is_fedora && [[ $DISTRO == "f26" ]]; then
is_package_installed python2-rpm || install_package python2-rpm
is_package_installed yum || install_package yum
fi fi
is_package_installed python-setuptools || install_package python-setuptools
if is_ubuntu && [[ $DISTRO == "trusty" ]]; then if is_ubuntu && [[ $DISTRO == "trusty" ]]; then
is_package_installed python-apt || install_package python-apt is_package_installed python-apt || install_package python-apt
...@@ -2082,6 +2094,7 @@ fi ...@@ -2082,6 +2094,7 @@ fi
ANSIBLE_PLUGIN_LOCATION="/usr/share/ansible_plugins/callback_plugins/" ANSIBLE_PLUGIN_LOCATION="/usr/share/ansible_plugins/callback_plugins/"
mkdir -p $ANSIBLE_PLUGIN_LOCATION mkdir -p $ANSIBLE_PLUGIN_LOCATION
/bin/cat << EOF > $ANSIBLE_PLUGIN_LOCATION/log_parse.py /bin/cat << EOF > $ANSIBLE_PLUGIN_LOCATION/log_parse.py
from __future__ import absolute_import
import os import os
import time import time
import json import json
...@@ -2111,11 +2124,11 @@ class CallbackModule(baseModule): ...@@ -2111,11 +2124,11 @@ class CallbackModule(baseModule):
else: else:
for filename in os.listdir(self.log_path): for filename in os.listdir(self.log_path):
filepath = os.path.join(self.log_path, filename) filepath = os.path.join(self.log_path, filename)
if os.path.exists(filepath) and os.path.isfile(filepath): if os.path.isfile(filepath):
os.unlink(filepath) os.unlink(filepath)
def writeLog(self, host, category, content): def writeLog(self, host, category, content):
if not self.fd_list.has_key(category): if category not in self.fd_list:
self.fd_list[category] = open( self.fd_list[category] = open(
os.path.join(self.log_path, '%s_%s' % (host, category)), "a" os.path.join(self.log_path, '%s_%s' % (host, category)), "a"
) )
...@@ -2126,12 +2139,11 @@ class CallbackModule(baseModule): ...@@ -2126,12 +2139,11 @@ class CallbackModule(baseModule):
if host == "localhost": if host == "localhost":
host = "127.0.0.1" # keep compatibility host = "127.0.0.1" # keep compatibility
if type(data) == dict: if type(data) is dict:
if '_ansible_verbose_override' in data: if '_ansible_verbose_override' in data:
# avoid logging extraneous data # avoid logging extraneous data
return return
data = data.copy()
content = json.dumps(data) content = json.dumps(data)
if ignore_errors: if ignore_errors:
...@@ -2168,8 +2180,17 @@ class CallbackModule(baseModule): ...@@ -2168,8 +2180,17 @@ class CallbackModule(baseModule):
EOF EOF
ansible localhost -m easy_install -a name=slapcache --connection=local ansible localhost -m pip -a name=setuptools --connection=local
ansible localhost -m easy_install -a name=requests --connection=local ansible localhost -m pip -a name=wheel --connection=local
ansible localhost -m pip -a name=slapcache --connection=local
ansible localhost -m pip -a name=requests --connection=local
if [ ! -f /usr/share/ansible_plugins/mitogen.zip ]; then
wget -O /usr/share/ansible_plugins/mitogen.zip https://shacache.nxdcdn.com/3a935ff257ddc0ad4e0f23d71681e026f14f309f4bed0a8e2a217da9b294be2c676196703f0dde856ece49d711d0221deae70812f035b24aa5cdd0ca02790e85
unzip /usr/share/ansible_plugins/mitogen.zip -d /usr/share/ansible_plugins/mitogen/
mv /usr/share/ansible_plugins/mitogen/mitogen-*/* /usr/share/ansible_plugins/mitogen/
fi
# Include Additional Functions # Include Additional Functions
if [ ! -f /etc/opt/slapcache.cfg ]; then if [ ! -f /etc/opt/slapcache.cfg ]; then
......
...@@ -19,11 +19,11 @@ md5sum = f2b0f1ed27148504f220e06eaceff935 ...@@ -19,11 +19,11 @@ md5sum = f2b0f1ed27148504f220e06eaceff935
[template-kvm] [template-kvm]
filename = instance-kvm.cfg.jinja2 filename = instance-kvm.cfg.jinja2
md5sum = 17c58f74d1ed4cb7dce11bf9af71dd33 md5sum = f902dd10cb052ac262a4a96b9362b3a3
[template-kvm-cluster] [template-kvm-cluster]
filename = instance-kvm-cluster.cfg.jinja2.in filename = instance-kvm-cluster.cfg.jinja2.in
md5sum = fcb35c32ef985e3d69a7914711675dcc md5sum = 67901b7a97b324aa36851d8e8b43cad6
[template-kvm-resilient] [template-kvm-resilient]
filename = instance-kvm-resilient.cfg.jinja2 filename = instance-kvm-resilient.cfg.jinja2
...@@ -55,7 +55,7 @@ md5sum = b7e87479a289f472b634a046b44b5257 ...@@ -55,7 +55,7 @@ md5sum = b7e87479a289f472b634a046b44b5257
[template-kvm-run] [template-kvm-run]
filename = template/template-kvm-run.in filename = template/template-kvm-run.in
md5sum = f840e8b7af83982525f66d7ec12b7085 md5sum = 875261817970d0f83335824373288b9d
[template-kvm-controller] [template-kvm-controller]
filename = template/kvm-controller-run.in filename = template/kvm-controller-run.in
......
...@@ -339,6 +339,22 @@ ...@@ -339,6 +339,22 @@
"type": "string", "type": "string",
"default": "host" "default": "host"
}, },
"network-adapter": {
"title": "Network adapter",
"description": "Network adapter provided to the guest. Allows to support legacy guests, which do not have drivers for default virtio-net-pci.",
"type": "string",
"default": "virtio-net-pci",
"enum": [
"virtio-net-pci",
"e1000",
"ne2k_isa",
"ne2k_pci",
"pcnet",
"rtl8139",
"usb-net",
"vmxnet3"
]
},
"nbd-host": { "nbd-host": {
"title": "NBD hostname or IP", "title": "NBD hostname or IP",
"description": "hostname (or IP) of the NBD server containing the boot image.", "description": "hostname (or IP) of the NBD server containing the boot image.",
......
...@@ -85,7 +85,8 @@ config-disk-size = {{ dumps(kvm_parameter_dict.get('disk-size', 40)) }} ...@@ -85,7 +85,8 @@ config-disk-size = {{ dumps(kvm_parameter_dict.get('disk-size', 40)) }}
config-disk-type = {{ dumps(kvm_parameter_dict.get('disk-type', 'virtio')) }} config-disk-type = {{ dumps(kvm_parameter_dict.get('disk-type', 'virtio')) }}
config-disk-format = {{ dumps(kvm_parameter_dict.get('disk-format', 'qcow2')) }} config-disk-format = {{ dumps(kvm_parameter_dict.get('disk-format', 'qcow2')) }}
config-cpu-count = {{ dumps(kvm_parameter_dict.get('cpu-count', 2)) }} config-cpu-count = {{ dumps(kvm_parameter_dict.get('cpu-count', 2)) }}
config-cpu-max-count = {{ dumps(kvm_parameter_dict.get('cpu-max-count', int(kvm_parameter_dict.get('cpu-count', 1)) + 1)) }} config-cpu-max-count = {{ dumps(kvm_parameter_dict.get('cpu-max-count', int(kvm_parameter_dict.get('cpu-count', 2)) + 1)) }}
config-network-adapter = {{ dumps(kvm_parameter_dict.get('network-adapter', 'virtio-net-pci')) }}
{{ setconfig('numa', kvm_parameter_dict.get('numa', '')) }} {{ setconfig('numa', kvm_parameter_dict.get('numa', '')) }}
{{ setconfig('machine-options', kvm_parameter_dict.get('machine-options', '')) }} {{ setconfig('machine-options', kvm_parameter_dict.get('machine-options', '')) }}
{{ setconfig('nbd-host', kvm_parameter_dict.get('nbd-host', '')) }} {{ setconfig('nbd-host', kvm_parameter_dict.get('nbd-host', '')) }}
......
...@@ -127,6 +127,22 @@ ...@@ -127,6 +127,22 @@
"type": "string", "type": "string",
"default": "host" "default": "host"
}, },
"network-adapter": {
"title": "Network adapter",
"description": "Network adapter provided to the guest. Allows to support legacy guests, which do not have drivers for default virtio-net-pci.",
"type": "string",
"default": "virtio-net-pci",
"enum": [
"virtio-net-pci",
"e1000",
"ne2k_isa",
"ne2k_pci",
"pcnet",
"rtl8139",
"usb-net",
"vmxnet3"
]
},
"nbd-host": { "nbd-host": {
"title": "NBD hostname", "title": "NBD hostname",
"description": "hostname (or IP) of the NBD server containing the boot image.", "description": "hostname (or IP) of the NBD server containing the boot image.",
......
...@@ -434,6 +434,8 @@ disk-format = ${slap-parameter:disk-format} ...@@ -434,6 +434,8 @@ disk-format = ${slap-parameter:disk-format}
disk-device-path = ${slap-parameter:disk-device-path} disk-device-path = ${slap-parameter:disk-device-path}
disk-path = ${directory:srv}/virtual.${slap-parameter:disk-format} disk-path = ${directory:srv}/virtual.${slap-parameter:disk-format}
network-adapter = ${slap-parameter:network-adapter}
pid-file-path = ${kvm-controller-parameter-dict:pid-file} pid-file-path = ${kvm-controller-parameter-dict:pid-file}
socket-path = ${kvm-controller-parameter-dict:socket-path} socket-path = ${kvm-controller-parameter-dict:socket-path}
...@@ -1075,6 +1077,8 @@ disk-type = virtio ...@@ -1075,6 +1077,8 @@ disk-type = virtio
disk-format = qcow2 disk-format = qcow2
disk-device-path = disk-device-path =
network-adapter = virtio-net-pci
cpu-count = 2 cpu-count = 2
disk-cache = disk-cache =
disk-aio = disk-aio =
......
...@@ -19,6 +19,8 @@ qemu_path = {{ repr(parameter_dict["qemu-path"]) }} ...@@ -19,6 +19,8 @@ qemu_path = {{ repr(parameter_dict["qemu-path"]) }}
disk_size = {{ repr(parameter_dict["disk-size"]) }} disk_size = {{ repr(parameter_dict["disk-size"]) }}
disk_type = {{ repr(parameter_dict["disk-type"]) }} disk_type = {{ repr(parameter_dict["disk-type"]) }}
network_adapter = {{ repr(parameter_dict["network-adapter"]) }}
socket_path = '{{ parameter_dict.get("socket-path") }}' socket_path = '{{ parameter_dict.get("socket-path") }}'
nbd_list = (('{{ parameter_dict.get("nbd-host") }}', nbd_list = (('{{ parameter_dict.get("nbd-host") }}',
{{ parameter_dict.get("nbd-port") }}), {{ parameter_dict.get("nbd-port") }}),
...@@ -255,7 +257,7 @@ if use_nat == 'true': ...@@ -255,7 +257,7 @@ if use_nat == 'true':
if use_tap == 'true' and tap_ipv6_addr != '': if use_tap == 'true' and tap_ipv6_addr != '':
rules += ',ipv6=off' rules += ',ipv6=off'
nat_network_parameter = ['-netdev', rules, nat_network_parameter = ['-netdev', rules,
'-device', 'virtio-net-pci,netdev=lan%s,mac=%s' % (number, mac_address)] '-device', '%s,netdev=lan%s,mac=%s' % (network_adapter, number, mac_address)]
if use_tap == 'true': if use_tap == 'true':
number += 1 number += 1
vhost = '' vhost = ''
...@@ -264,7 +266,7 @@ if use_tap == 'true': ...@@ -264,7 +266,7 @@ if use_tap == 'true':
tap_network_parameter = ['-netdev', tap_network_parameter = ['-netdev',
'tap,id=lan%s,ifname=%s,script=no,downscript=no%s' % (number, 'tap,id=lan%s,ifname=%s,script=no,downscript=no%s' % (number,
tap_interface, vhost), tap_interface, vhost),
'-device', 'virtio-net-pci,netdev=lan%s,mac=%s' % (number, tap_mac_address)] '-device', '%s,netdev=lan%s,mac=%s' % (network_adapter, number, tap_mac_address)]
smp = '%s,maxcpus=%s' % (init_smp_count, smp_max_count) smp = '%s,maxcpus=%s' % (init_smp_count, smp_max_count)
ram = '%sM,slots=128,maxmem=%sM' % (init_ram_size, ram_max_size) ram = '%sM,slots=128,maxmem=%sM' % (init_ram_size, ram_max_size)
......
...@@ -1179,27 +1179,6 @@ class TestBootImageUrlSelectKvmCluster(TestBootImageUrlListKvmCluster): ...@@ -1179,27 +1179,6 @@ class TestBootImageUrlSelectKvmCluster(TestBootImageUrlListKvmCluster):
config_file_name = 'boot-image-url-select.json' config_file_name = 'boot-image-url-select.json'
@skipUnlessKvm
class TestCpuMemMaxDynamic(InstanceTestCase):
__partition_reference__ = 'cmm'
@classmethod
def getInstanceParameterDict(cls):
return {
'cpu-count': 2,
'ram-size': 2048
}
def test(self):
with open(os.path.join(
self.computer_partition_root_path, 'bin', 'kvm_raw'), 'r') as fh:
kvm_raw = fh.read()
self.assertIn('smp_count = 2', kvm_raw)
self.assertIn('smp_max_count = 3', kvm_raw)
self.assertIn('ram_size = 2048', kvm_raw)
self.assertIn("ram_max_size = '2560'", kvm_raw)
@skipUnlessKvm @skipUnlessKvm
class TestNatRules(InstanceTestCase): class TestNatRules(InstanceTestCase):
__partition_reference__ = 'nr' __partition_reference__ = 'nr'
...@@ -1611,3 +1590,101 @@ INF: Storing errors in %(error_state_file)s ...@@ -1611,3 +1590,101 @@ INF: Storing errors in %(error_state_file)s
self.assertFalse( self.assertFalse(
os.path.exists( os.path.exists(
os.path.join(self.destination_directory, 'destination'))) os.path.join(self.destination_directory, 'destination')))
@skipUnlessKvm
class TestParameterDefault(InstanceTestCase, KvmMixin):
__partition_reference__ = 'pd'
@classmethod
def getInstanceSoftwareType(cls):
return 'default'
def mangleParameterDict(self, parameter_dict):
return parameter_dict
def _test(self, parameter_dict, expected):
self.rerequestInstance(self.mangleParameterDict(parameter_dict))
self.slap.waitForInstance(max_retry=10)
kvm_raw = glob.glob(os.path.join(
self.slap.instance_directory, '*', 'bin', 'kvm_raw'))
self.assertEqual(len(kvm_raw), 1)
kvm_raw = kvm_raw[0]
with open(kvm_raw, 'r') as fh:
kvm_raw = fh.read()
self.assertIn(expected, kvm_raw)
def test_disk_type_default(self):
self._test({}, "disk_type = 'virtio'")
def test_disk_type_set(self):
self._test({'disk-type': 'ide'}, "disk_type = 'ide'")
def test_network_adapter_default(self):
self._test({}, "network_adapter = 'virtio-net-pci")
def test_network_adapter_set(self):
self._test({'network-adapter': 'e1000'}, "network_adapter = 'e1000'")
def test_cpu_count_default(self):
self._test({}, "init_smp_count = 2")
def test_cpu_count_default_max(self):
self._test({}, "smp_max_count = 3")
def test_cpu_count_set(self):
self._test({'cpu-count': 4}, "init_smp_count = 4")
def test_cpu_count_set_max(self):
self._test({'cpu-count': 4}, "smp_max_count = 5")
def test_ram_size_default(self):
self._test({}, "init_ram_size = 4096")
def test_ram_size_default_max(self):
self._test({}, "ram_max_size = '4608'")
def test_ram_size_set(self):
self._test({'ram-size': 2048}, "init_ram_size = 2048")
def test_ram_size_set_max(self):
self._test({'ram-size': 2048}, "ram_max_size = '2560'")
@skipUnlessKvm
class TestParameterResilient(TestParameterDefault):
__partition_reference__ = 'pr'
@classmethod
def getInstanceSoftwareType(cls):
return 'kvm-resilient'
@skipUnlessKvm
class TestParameterCluster(TestParameterDefault):
__partition_reference__ = 'pc'
parameter_dict = {
"disable-ansible-promise": True
}
@classmethod
def getInstanceParameterDict(cls):
return {'_': json.dumps({
"kvm-partition-dict": {
"KVM0": cls.parameter_dict
}
})}
def mangleParameterDict(self, parameter_dict):
local_parameter_dict = self.parameter_dict.copy()
local_parameter_dict.update(parameter_dict)
return {'_': json.dumps({
"kvm-partition-dict": {
"KVM0": local_parameter_dict
}
})}
@classmethod
def getInstanceSoftwareType(cls):
return 'kvm-cluster'
...@@ -8,6 +8,7 @@ extends = ...@@ -8,6 +8,7 @@ extends =
../../component/pycurl/buildout.cfg ../../component/pycurl/buildout.cfg
../../component/faketime/buildout.cfg ../../component/faketime/buildout.cfg
../../component/pillow/buildout.cfg ../../component/pillow/buildout.cfg
../../component/psycopg2/buildout.cfg
../../component/python-cryptography/buildout.cfg ../../component/python-cryptography/buildout.cfg
../../component/python-mysqlclient/buildout.cfg ../../component/python-mysqlclient/buildout.cfg
../../component/python-pynacl/buildout.cfg ../../component/python-pynacl/buildout.cfg
...@@ -242,6 +243,7 @@ extra-eggs = ...@@ -242,6 +243,7 @@ extra-eggs =
${python-mysqlclient:egg} ${python-mysqlclient:egg}
${backports.lzma:egg} ${backports.lzma:egg}
${bcrypt:egg} ${bcrypt:egg}
${psycopg2:egg}
slapos.libnetworkcache slapos.libnetworkcache
supervisor supervisor
${slapos.cookbook-setup:egg} ${slapos.cookbook-setup:egg}
...@@ -393,3 +395,4 @@ mysqlclient = 1.3.12 ...@@ -393,3 +395,4 @@ mysqlclient = 1.3.12
pexpect = 4.8.0 pexpect = 4.8.0
ptyprocess = 0.6.0 ptyprocess = 0.6.0
typing = 3.7.4.3 typing = 3.7.4.3
psycopg2 = 2.8.6
...@@ -15,4 +15,4 @@ ...@@ -15,4 +15,4 @@
[template] [template]
filename = instance.cfg filename = instance.cfg
md5sum = 84bd6729e9b299c457cea2d1be6d05a4 md5sum = 597c29546519aabe7259e416d0b92095
...@@ -92,6 +92,7 @@ template = inline: ...@@ -92,6 +92,7 @@ template = inline:
export SLAPOS_TEST_IPV6=$${slap-configuration:ipv6-random} export SLAPOS_TEST_IPV6=$${slap-configuration:ipv6-random}
export SLAPOS_TEST_EGGS_DIRECTORY=$${buildout:eggs-directory} export SLAPOS_TEST_EGGS_DIRECTORY=$${buildout:eggs-directory}
export SLAPOS_TEST_DEVELOP_EGGS_DIRECTORY=$${buildout:develop-eggs-directory} export SLAPOS_TEST_DEVELOP_EGGS_DIRECTORY=$${buildout:develop-eggs-directory}
export SLAPOS_TEST_POSTGRESQL_PREFIX=${postgresql:location}
[slapos-test-runner-dot-nxdtest] [slapos-test-runner-dot-nxdtest]
......
...@@ -17,6 +17,8 @@ extends = ...@@ -17,6 +17,8 @@ extends =
../../component/sed/buildout.cfg ../../component/sed/buildout.cfg
../../component/grep/buildout.cfg ../../component/grep/buildout.cfg
../../component/userhosts/buildout.cfg ../../component/userhosts/buildout.cfg
../../component/postgresql/buildout.cfg
../../component/psycopg2/buildout.cfg
../../stack/slapos.cfg ../../stack/slapos.cfg
../../stack/caucase/buildout.cfg ../../stack/caucase/buildout.cfg
../../stack/nxdtest.cfg ../../stack/nxdtest.cfg
...@@ -66,6 +68,7 @@ egg = slapos.cookbook[test] ...@@ -66,6 +68,7 @@ egg = slapos.cookbook[test]
setup = ${slapos.cookbook-repository:location} setup = ${slapos.cookbook-repository:location}
depends = depends =
${slapos.core-setup:egg} ${slapos.core-setup:egg}
${psycopg2:egg}
${slapos-cookbook-dependencies:eggs} ${slapos-cookbook-dependencies:eggs}
[slapos.core-setup] [slapos.core-setup]
...@@ -245,3 +248,4 @@ pyflakes = 2.0.0 ...@@ -245,3 +248,4 @@ pyflakes = 2.0.0
zope.testing = 4.6.2 zope.testing = 4.6.2
urllib3 = 1.24.1 urllib3 = 1.24.1
pathlib = 1.0.1 pathlib = 1.0.1
psycopg2 = 2.8.6
...@@ -190,7 +190,7 @@ setproctitle = 1.1.10 ...@@ -190,7 +190,7 @@ setproctitle = 1.1.10
setuptools-dso = 1.7 setuptools-dso = 1.7
rubygemsrecipe = 0.4.3 rubygemsrecipe = 0.4.3
six = 1.12.0 six = 1.12.0
slapos.cookbook = 1.0.213 slapos.cookbook = 1.0.214
slapos.core = 1.6.19 slapos.core = 1.6.19
slapos.extension.strip = 0.4 slapos.extension.strip = 0.4
slapos.extension.shared = 1.0 slapos.extension.shared = 1.0
......
...@@ -36,6 +36,7 @@ extends = ...@@ -36,6 +36,7 @@ extends =
../../component/stunnel/buildout.cfg ../../component/stunnel/buildout.cfg
../../component/dropbear/buildout.cfg ../../component/dropbear/buildout.cfg
../../component/pycurl/buildout.cfg ../../component/pycurl/buildout.cfg
../../component/psycopg2/buildout.cfg
../slapos.cfg ../slapos.cfg
../resilient/buildout.cfg ../resilient/buildout.cfg
...@@ -127,26 +128,6 @@ md5sum = d95205a5fc2825e9709ed6db295111e2 ...@@ -127,26 +128,6 @@ md5sum = d95205a5fc2825e9709ed6db295111e2
mode = 0644 mode = 0644
#----------------
#-- Postgres driver for Python recipes.
[psycopg2-env]
PATH = ${postgresql:location}/bin:%(PATH)s
[psycopg2]
recipe = zc.recipe.egg:custom
egg = psycopg2
define = PSYCOPG_EXTENSIONS,PSYCOPG_DISPLAY_SIZE,PSYCOPG_NEW_BOOLEAN,HAVE_PQFREEMEM,HAVE_PQPROTOCOL3
environment = psycopg2-env
include-dirs =
${postgresql:location}/include
library-dirs =
${postgresql:location}/lib
rpath =
${postgresql:location}/lib
#---------------- #----------------
#-- #--
#-- Optional part allowing applications using this stack to run a custom #-- Optional part allowing applications using this stack to run a custom
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment