Commit be8ddac7 authored by Kirill Smelkov's avatar Kirill Smelkov

Merge branch 'master+ZODB4-wc2' into y/wc2-next

* master+ZODB4-wc2: (32 commits)
  wendelin.core: v↑ 0.13 -> 2.0.alpha1
  caddy-frontend: Support not resolving backend hostnames
  component/caddy: Use golang 1.17
  software/metabase: configure fonts and tempdir
  software/metabase: generate proper passwords
  software/metabase: version up metabase v0.41.2
  software/metabase: sleep a bit more after "this will erase data" warning
  software/metabase: use temurin JRE
  component/java: add temurin JRE
  Fix "kvm: Increase default VM parameters to match nowadays requirement"
  kvm: Merge tests
  kvm: Allow to specify network adapter
  software/slapos-sr-testing: install new psycopg2 dependency
  recipe/postgres: support non standard port
  recipe/test_postgres: test with a real postgres server
  software/slapos-testing: install psycopg2 for slapos.cookbook
  software/slapos-testing: make postgresql available for tests
  component/psycopg2: new component
  golang: Switch default to Go1.17
  erp5testnode: update deploy script
  ...
parents ad2ff25e a8ef2211
...@@ -13,7 +13,7 @@ repository = https://lab.nexedi.com/nexedi/caddy.git ...@@ -13,7 +13,7 @@ repository = https://lab.nexedi.com/nexedi/caddy.git
revision = nxd-v1.0.3-1-g2c11cedc revision = nxd-v1.0.3-1-g2c11cedc
[gowork] [gowork]
golang = ${golang1.16:location} golang = ${golang1.17:location}
install = install =
${caddy-get:location}:./... ${caddy-get:location}:./...
......
...@@ -46,8 +46,8 @@ environment-extra = ...@@ -46,8 +46,8 @@ environment-extra =
[golang1.16] [golang1.16]
<= golang-common <= golang-common
url = https://golang.org/dl/go1.16.8.src.tar.gz url = https://golang.org/dl/go1.16.10.src.tar.gz
md5sum = 92e69a5e1bb6ea5e7498d12d03160032 md5sum = 49f0a54f0bdcba297bac194d8dafe431
# go1.16 needs go1.4 to bootstrap # go1.16 needs go1.4 to bootstrap
environment-extra = environment-extra =
...@@ -55,8 +55,8 @@ environment-extra = ...@@ -55,8 +55,8 @@ environment-extra =
[golang1.17] [golang1.17]
<= golang-common <= golang-common
url = https://golang.org/dl/go1.17.1.src.tar.gz url = https://golang.org/dl/go1.17.3.src.tar.gz
md5sum = a78205838c2a7054522cb91c12982f26 md5sum = 3ea82e5966275f405f0db4f52511bb6e
# go1.17 needs go1.4 to bootstrap # go1.17 needs go1.4 to bootstrap
environment-extra = environment-extra =
...@@ -122,7 +122,7 @@ bin = ${gowork.dir:bin} ...@@ -122,7 +122,7 @@ bin = ${gowork.dir:bin}
depends = ${gowork.goinstall:recipe} depends = ${gowork.goinstall:recipe}
# go version used for the workspace (possible to override in applications) # go version used for the workspace (possible to override in applications)
golang = ${golang1.16:location} golang = ${golang1.17:location}
# no special build flags by default # no special build flags by default
buildflags = buildflags =
......
[buildout] [buildout]
extends =
../coreutils/buildout.cfg
../patchelf/buildout.cfg
../alsa/buildout.cfg
../libpng/buildout.cfg
../freetype/buildout.cfg
../fontconfig/buildout.cfg
../xorg/buildout.cfg
../zlib/buildout.cfg
parts = parts =
java java
...@@ -9,12 +17,12 @@ parts = ...@@ -9,12 +17,12 @@ parts =
[java-re] [java-re]
<= java-re-7 <= java-re-7
[java-common] [java-sun-common]
recipe = slapos.recipe.build:download-unpacked recipe = slapos.recipe.build:download-unpacked
url = http://javadl.sun.com/webapps/download/AutoDL?BundleId=${:bundle-id} url = https://javadl.sun.com/webapps/download/AutoDL?BundleId=${:bundle-id}
[java-re-7] [java-re-7]
<= java-common <= java-sun-common
# http://java.com/en/download/manual_java7.jsp # http://java.com/en/download/manual_java7.jsp
[java-re-7:linux and platform.machine() == 'i686'] [java-re-7:linux and platform.machine() == 'i686']
...@@ -26,7 +34,7 @@ bundle-id = 97800 ...@@ -26,7 +34,7 @@ bundle-id = 97800
md5sum = 7605134662f6c87131eca5745895fe84 md5sum = 7605134662f6c87131eca5745895fe84
[java-re-8] [java-re-8]
<= java-common <= java-sun-common
# https://www.java.com/en/download/manual.jsp # https://www.java.com/en/download/manual.jsp
# Update 161 # Update 161
...@@ -45,3 +53,25 @@ stop-on-error = true ...@@ -45,3 +53,25 @@ stop-on-error = true
update-command = ${:command} update-command = ${:command}
command = ${coreutils-output:test} -x ${:keytool} command = ${coreutils-output:test} -x ${:keytool}
keytool = ${java-re-8:location}/bin/keytool keytool = ${java-re-8:location}/bin/keytool
[java-re-temurin-11]
recipe = slapos.recipe.build
update =
from zc.buildout import UserError
raise UserError("unsupported platform")
[java-re-temurin-11:linux and platform.machine() == 'x86_64']
recipe = slapos.recipe.cmmi
shared = true
url = https://github.com/adoptium/temurin11-binaries/releases/download/jdk-11.0.13%2B8/OpenJDK11U-jre_x64_linux_hotspot_11.0.13_8.tar.gz
md5sum = 1b06100bcd0923d3f3279c2f09773af0
configure-command = :
make-binary = :
post-install =
mv * %(location)s
for file in %(location)s/bin/* %(location)s/lib/*.so %(location)s/lib/*/*.so ; do
echo appending rpath to $file
${patchelf:location}/bin/patchelf --set-rpath %(rpath)s $file
done
rpath = ${alsa:location}/lib:${freetype:location}/lib:${fontconfig:location}/lib:${libpng:location}/lib:${libXrender:location}/lib:${libXtst:location}/lib:${libX11:location}/lib:${libXau:location}/lib:${libXext:location}/lib:${libXdmcp:location}/lib:${libXi:location}/lib:${libxcb:location}/lib:${zlib:location}/lib:@@LOCATION@@/lib:@@LOCATION@@/lib/server:@@LOCATION@@/lib/jli
...@@ -20,6 +20,6 @@ egg = nxdtest ...@@ -20,6 +20,6 @@ egg = nxdtest
[nxdtest-repository] [nxdtest-repository]
recipe = slapos.recipe.build:gitclone recipe = slapos.recipe.build:gitclone
repository = https://lab.nexedi.com/nexedi/nxdtest.git repository = https://lab.nexedi.com/nexedi/nxdtest.git
revision = 0ad45a9c revision = 9f413221
location = ${buildout:parts-directory}/nxdtest location = ${buildout:parts-directory}/nxdtest
git-executable = ${git:location}/bin/git git-executable = ${git:location}/bin/git
[buildout]
extends =
../postgresql/buildout.cfg
[psycopg2-env]
PATH = ${postgresql:location}/bin:%(PATH)s
[psycopg2]
recipe = zc.recipe.egg:custom
egg = psycopg2
define = PSYCOPG_EXTENSIONS,PSYCOPG_DISPLAY_SIZE,PSYCOPG_NEW_BOOLEAN,HAVE_PQFREEMEM,HAVE_PQPROTOCOL3
environment = psycopg2-env
include-dirs =
${postgresql:location}/include
library-dirs =
${postgresql:location}/lib
rpath =
${postgresql:location}/lib
# SlapOS extension that switches Wendelin.core to Wendelin.core 2 preview.
#
# Should go away once wendelin.core 2 lands to nexedi/wendelin.core@master.
# Must be extended from last, for example:
#
# [buildout]
# extends =
# .../stack/erp5/buildout.cfg
# .../component/wendelin.core/activate-WC2-preview.cfg
[wendelin.core-repository]
repository = https://lab.nexedi.com/kirr/wendelin.core.git
branch = t
revision =
# Wendelin.core 2 targets Go1.17
[gowork]
golang = ${golang1.17:location}
...@@ -50,7 +50,7 @@ CGO_LDFLAGS += -Wl,-rpath=${zlib:location}/lib ...@@ -50,7 +50,7 @@ CGO_LDFLAGS += -Wl,-rpath=${zlib:location}/lib
recipe = slapos.recipe.build:gitclone recipe = slapos.recipe.build:gitclone
repository = https://lab.nexedi.com/nexedi/wendelin.core.git repository = https://lab.nexedi.com/nexedi/wendelin.core.git
branch = master branch = master
revision = v0.13-0-gb26ba55 revision = wendelin.core-2.0.alpha1-0-g49f826b1
# dir is pretty name as top-level recipe # dir is pretty name as top-level recipe
location = ${buildout:parts-directory}/wendelin.core location = ${buildout:parts-directory}/wendelin.core
git-executable = ${git:location}/bin/git git-executable = ${git:location}/bin/git
...@@ -14,6 +14,7 @@ extends = ...@@ -14,6 +14,7 @@ extends =
parts = parts =
# keep neoppod first and in parts so that ZODB is built correctly # keep neoppod first and in parts so that ZODB is built correctly
neoppod-develop
neoppod neoppod
# for instance # for instance
......
...@@ -25,6 +25,7 @@ configure-options = ...@@ -25,6 +25,7 @@ configure-options =
--without-python --without-python
--without-x --without-x
--without-jpg --without-jpg
--without-dbus
environment = environment =
PATH=${autoconf:location}/bin:${automake:location}/bin:${gettext:location}/bin:${libtool:location}/bin:${m4:location}/bin:${bzip2:location}/bin:%(PATH)s PATH=${autoconf:location}/bin:${automake:location}/bin:${gettext:location}/bin:${libtool:location}/bin:${m4:location}/bin:${bzip2:location}/bin:%(PATH)s
CFLAGS= CFLAGS=
...@@ -28,7 +28,7 @@ from setuptools import setup, find_packages ...@@ -28,7 +28,7 @@ from setuptools import setup, find_packages
import glob import glob
import os import os
version = '1.0.213' version = '1.0.214'
name = 'slapos.cookbook' name = 'slapos.cookbook'
long_description = open("README.rst").read() long_description = open("README.rst").read()
...@@ -39,6 +39,7 @@ extras_require = { ...@@ -39,6 +39,7 @@ extras_require = {
'test': ( 'test': (
'jsonschema', 'jsonschema',
'mock', 'mock',
'psycopg2',
'testfixtures', 'testfixtures',
'requests', 'requests',
), ),
......
...@@ -104,6 +104,7 @@ class Recipe(GenericBaseRecipe): ...@@ -104,6 +104,7 @@ class Recipe(GenericBaseRecipe):
else: else:
paths.extend(self.createConfig()) paths.extend(self.createConfig())
paths.extend(self.createRunScript()) paths.extend(self.createRunScript())
self.updateSuperuser()
return paths return paths
...@@ -146,6 +147,7 @@ class Recipe(GenericBaseRecipe): ...@@ -146,6 +147,7 @@ class Recipe(GenericBaseRecipe):
with open(postgres_conf, 'w') as cfg: with open(postgres_conf, 'w') as cfg:
cfg.write(textwrap.dedent("""\ cfg.write(textwrap.dedent("""\
listen_addresses = '%s' listen_addresses = '%s'
port = %s
logging_collector = on logging_collector = on
log_rotation_size = 50MB log_rotation_size = 50MB
max_connections = 100 max_connections = 100
...@@ -161,6 +163,7 @@ class Recipe(GenericBaseRecipe): ...@@ -161,6 +163,7 @@ class Recipe(GenericBaseRecipe):
unix_socket_permissions = 0700 unix_socket_permissions = 0700
""" % ( """ % (
','.join(set(ipv4).union(ipv6)), ','.join(set(ipv4).union(ipv6)),
self.options['port'],
pgdata, pgdata,
))) )))
...@@ -205,9 +208,25 @@ class Recipe(GenericBaseRecipe): ...@@ -205,9 +208,25 @@ class Recipe(GenericBaseRecipe):
# encrypt the password to avoid storing in the logs # encrypt the password to avoid storing in the logs
enc_password = 'md5' + hashlib.md5((password + user).encode()).hexdigest() enc_password = 'md5' + hashlib.md5((password + user).encode()).hexdigest()
change_password_query = """ALTER USER "%s" ENCRYPTED PASSWORD '%s'""" % (user, enc_password)
self.runPostgresCommand(cmd="""ALTER USER "%s" ENCRYPTED PASSWORD '%s'""" % (user, enc_password)) pgdata = self.options['pgdata-directory']
if os.path.exists(os.path.join(pgdata, 'postmaster.pid')):
psql_binary = os.path.join(self.options['bin'], 'psql')
# connect to a running postgres deamon
p = subprocess.Popen([
psql_binary,
'-h', pgdata,
'-p', self.options['port'],
'-U', user,
'-d', self.options['dbname'],
],
stdin=subprocess.PIPE)
p.communicate((change_password_query + '\n').encode())
if p.returncode:
raise UserError("Error updating password")
else:
self.runPostgresCommand(cmd=change_password_query)
def runPostgresCommand(self, cmd): def runPostgresCommand(self, cmd):
"""\ """\
......
import unittest import os
import tempfile
import shutil import shutil
import os.path import tempfile
import textwrap
import time
import unittest
try:
import subprocess32 as subprocess
except ImportError:
import subprocess
import psycopg2
import zc.buildout.testing import zc.buildout.testing
class PostgresTest(unittest.TestCase): class PostgresTest(unittest.TestCase):
ipv4 = os.environ['SLAPOS_TEST_IPV4']
ipv6 = os.environ['SLAPOS_TEST_IPV6']
port = 5432
def setUp(self): def setUp(self):
self.buildout = buildout = zc.buildout.testing.Buildout() self.buildout = buildout = zc.buildout.testing.Buildout()
self.pgdata_directory = tempfile.mkdtemp() self.pgdata_directory = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, self.pgdata_directory) self.addCleanup(shutil.rmtree, self.pgdata_directory)
self.services_directory = tempfile.mkdtemp() self.services_directory = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, self.services_directory) self.addCleanup(shutil.rmtree, self.services_directory)
self.postgres_bin_directory = os.environ['SLAPOS_TEST_POSTGRESQL_PREFIX'] + '/bin'
buildout['postgres'] = { buildout['postgres'] = {
'bin': 'software/parts/postgres/bin/', 'bin': self.postgres_bin_directory,
'dbname': 'dbname', 'dbname': 'dbname',
'ipv4': '127.0.0.1', 'ipv4': self.ipv4,
'ipv6': '::1', 'ipv6': self.ipv6,
'port': '5443', 'port': self.port,
'pgdata-directory': self.pgdata_directory, 'pgdata-directory': os.path.join(self.pgdata_directory, 'pgdata'),
'services': self.services_directory, 'services': self.services_directory,
'superuser': 'superuser', 'superuser': 'superuser',
'password': 'secret', 'password': 'secret',
...@@ -29,21 +44,73 @@ class PostgresTest(unittest.TestCase): ...@@ -29,21 +44,73 @@ class PostgresTest(unittest.TestCase):
'postgres', 'postgres',
buildout['postgres']) buildout['postgres'])
def start_postgres_server(self):
server_process = subprocess.Popen(
[ os.path.join(self.services_directory, 'postgres-start') ],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
self.addCleanup(server_process.wait)
self.addCleanup(server_process.terminate)
# wait for server to accept connections
for i in range(60):
time.sleep(i)
try:
psycopg2.connect(self.buildout['postgres']['url']).close()
except psycopg2.OperationalError as e:
pass
else:
break
def test_options(self): def test_options(self):
self.assertEqual( self.assertEqual(
'postgresql://superuser:secret@[::1]:5443/dbname', self.buildout['postgres']['url'],
self.buildout['postgres']['url']) 'postgresql://superuser:secret@[{self.ipv6}]:{self.port}/dbname'.format(self=self),
)
def test_install(self): def test_install(self):
installed = self.recipe.install() installed = self.recipe.install()
self.assertIn('postgresql.conf', os.listdir(self.pgdata_directory)) pgdata_directory = os.path.join(self.pgdata_directory, 'pgdata')
self.assertIn('pg_hba.conf', os.listdir(self.pgdata_directory)) self.assertIn('postgresql.conf', os.listdir(pgdata_directory))
self.assertIn('pg_hba.conf', os.listdir(pgdata_directory))
self.assertIn('postgres-start', os.listdir(self.services_directory)) self.assertIn('postgres-start', os.listdir(self.services_directory))
self.assertEqual( self.assertEqual(
sorted(installed), sorted(installed),
sorted([ sorted([
os.path.join(self.pgdata_directory, 'postgresql.conf'), os.path.join(pgdata_directory, 'postgresql.conf'),
os.path.join(self.pgdata_directory, 'pg_hba.conf'), os.path.join(pgdata_directory, 'pg_hba.conf'),
os.path.join(self.services_directory, 'postgres-start')])) os.path.join(self.services_directory, 'postgres-start')]))
self.start_postgres_server()
with psycopg2.connect(self.buildout['postgres']['url']) as cnx:
with cnx.cursor() as cursor:
cursor.execute("SELECT 1+1")
self.assertEqual(cursor.fetchone(), (2,))
cnx.close()
def test_update_password(self):
self.recipe.install()
self.start_postgres_server()
self.recipe.options['password'] = 'new'
self.recipe.install()
dsn = self.buildout['postgres']['url']
with psycopg2.connect(psycopg2.extensions.make_dsn(dsn, password='new')) as cnx:
with cnx.cursor() as cursor:
cursor.execute("SELECT 1+1")
self.assertEqual(cursor.fetchone(), (2,))
cnx.close()
# old password can no longer connect
with self.assertRaisesRegexp(
psycopg2.OperationalError,
'password authentication failed'
):
psycopg2.connect(dsn)
class PostgresTestNonStandardPort(PostgresTest):
port = 5433
...@@ -50,7 +50,7 @@ md5sum = 37475d79f28c5f126bc1947fdb938fdb ...@@ -50,7 +50,7 @@ md5sum = 37475d79f28c5f126bc1947fdb938fdb
[template-backend-haproxy-configuration] [template-backend-haproxy-configuration]
_update_hash_filename_ = templates/backend-haproxy.cfg.in _update_hash_filename_ = templates/backend-haproxy.cfg.in
md5sum = 5e126be0f74d8ae390a5594e1e912a59 md5sum = d2851c7ebd2c9baa2edecb3ca3485511
[template-empty] [template-empty]
_update_hash_filename_ = templates/empty.in _update_hash_filename_ = templates/empty.in
......
...@@ -13,6 +13,8 @@ defaults ...@@ -13,6 +13,8 @@ defaults
timeout client {{ configuration['request-timeout'] }}s timeout client {{ configuration['request-timeout'] }}s
timeout connect {{ configuration['backend-connect-timeout'] }}s timeout connect {{ configuration['backend-connect-timeout'] }}s
retries {{ configuration['backend-connect-retries'] }} retries {{ configuration['backend-connect-retries'] }}
{#- Allow to start with not resolved yet servers #}
default-server init-addr last,libc,none
{%- set SCHEME_PREFIX_MAPPING = { 'http': 'http_backend', 'https': 'https_backend'} %} {%- set SCHEME_PREFIX_MAPPING = { 'http': 'http_backend', 'https': 'https_backend'} %}
{%- macro frontend_entry(slave_instance, scheme, wildcard) %} {%- macro frontend_entry(slave_instance, scheme, wildcard) %}
......
...@@ -1310,6 +1310,9 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin): ...@@ -1310,6 +1310,9 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
return { return {
'empty': { 'empty': {
}, },
'bad-backend': {
'url': 'http://bad.backend/',
},
'Url': { 'Url': {
# make URL "incorrect", with whitespace, nevertheless it shall be # make URL "incorrect", with whitespace, nevertheless it shall be
# correctly handled # correctly handled
......
...@@ -110,6 +110,7 @@ class CaucaseService(ManagedResource): ...@@ -110,6 +110,7 @@ class CaucaseService(ManagedResource):
'--netloc', backend_caucased_netloc, '--netloc', backend_caucased_netloc,
'--service-auto-approve-count', '1', '--service-auto-approve-count', '1',
], ],
# capture subprocess output not to pollute test's own stdout
stdout=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, stderr=subprocess.STDOUT,
) )
...@@ -127,6 +128,7 @@ class CaucaseService(ManagedResource): ...@@ -127,6 +128,7 @@ class CaucaseService(ManagedResource):
# type: () -> None # type: () -> None
self._caucased_process.terminate() self._caucased_process.terminate()
self._caucased_process.wait() self._caucased_process.wait()
self._caucased_process.stdout.close()
shutil.rmtree(self.directory) shutil.rmtree(self.directory)
...@@ -525,26 +527,27 @@ class TestHTTP(BalancerTestCase): ...@@ -525,26 +527,27 @@ class TestHTTP(BalancerTestCase):
def test_keep_alive(self): def test_keep_alive(self):
# type: () -> None # type: () -> None
# when doing two requests, connection is established only once # when doing two requests, connection is established only once
session = requests.Session() with requests.Session() as session:
session.verify = False session.verify = False
# do a first request, which establish a first connection # do a first request, which establish a first connection
session.get(self.default_balancer_url).raise_for_status()
# "break" new connection method and check we can make another request
with mock.patch(
"requests.packages.urllib3.connectionpool.HTTPSConnectionPool._new_conn",
) as new_conn:
session.get(self.default_balancer_url).raise_for_status() session.get(self.default_balancer_url).raise_for_status()
new_conn.assert_not_called()
# "break" new connection method and check we can make another request
parsed_url = six.moves.urllib.parse.urlparse(self.default_balancer_url) with mock.patch(
# check that we have an open file for the ip connection "requests.packages.urllib3.connectionpool.HTTPSConnectionPool._new_conn",
self.assertTrue([ ) as new_conn:
c for c in psutil.Process(os.getpid()).connections() session.get(self.default_balancer_url).raise_for_status()
if c.status == 'ESTABLISHED' and c.raddr.ip == parsed_url.hostname new_conn.assert_not_called()
and c.raddr.port == parsed_url.port
]) parsed_url = six.moves.urllib.parse.urlparse(self.default_balancer_url)
# check that we have an open file for the ip connection
self.assertTrue([
c for c in psutil.Process(os.getpid()).connections()
if c.status == 'ESTABLISHED' and c.raddr.ip == parsed_url.hostname
and c.raddr.port == parsed_url.port
])
class ContentTypeHTTPServer(ManagedHTTPServer): class ContentTypeHTTPServer(ManagedHTTPServer):
......
...@@ -64,28 +64,28 @@ class TestPublishedURLIsReachableMixin(object): ...@@ -64,28 +64,28 @@ class TestPublishedURLIsReachableMixin(object):
# erp5 site is not created, with 500 when mysql is not yet reachable, so we # erp5 site is not created, with 500 when mysql is not yet reachable, so we
# configure this requests session to retry. # configure this requests session to retry.
# XXX we should probably add a promise instead # XXX we should probably add a promise instead
session = requests.Session() with requests.Session() as session:
session.mount( session.mount(
base_url, base_url,
requests.adapters.HTTPAdapter( requests.adapters.HTTPAdapter(
max_retries=requests.packages.urllib3.util.retry.Retry( max_retries=requests.packages.urllib3.util.retry.Retry(
total=60, total=60,
backoff_factor=.5, backoff_factor=.5,
status_forcelist=(404, 500, 503)))) status_forcelist=(404, 500, 503))))
r = session.get(virtual_host_url, verify=verify, allow_redirects=False) r = session.get(virtual_host_url, verify=verify, allow_redirects=False)
self.assertEqual(r.status_code, requests.codes.found) self.assertEqual(r.status_code, requests.codes.found)
# access on / are redirected to login form, with virtual host preserved # access on / are redirected to login form, with virtual host preserved
self.assertEqual(r.headers.get('location'), 'https://virtual-host-name:1234/virtual_host_root/login_form') self.assertEqual(r.headers.get('location'), 'https://virtual-host-name:1234/virtual_host_root/login_form')
# login page can be rendered and contain the text "ERP5" # login page can be rendered and contain the text "ERP5"
r = session.get( r = session.get(
six.moves.urllib.parse.urljoin(base_url, '{}/login_form'.format(site_id)), six.moves.urllib.parse.urljoin(base_url, '{}/login_form'.format(site_id)),
verify=verify, verify=verify,
allow_redirects=False, allow_redirects=False,
) )
self.assertEqual(r.status_code, requests.codes.ok) self.assertEqual(r.status_code, requests.codes.ok)
self.assertIn("ERP5", r.text) self.assertIn("ERP5", r.text)
def test_published_family_default_v6_is_reachable(self): def test_published_family_default_v6_is_reachable(self):
"""Tests the IPv6 URL published by the root partition is reachable. """Tests the IPv6 URL published by the root partition is reachable.
......
...@@ -237,7 +237,7 @@ function GetOSVersion { ...@@ -237,7 +237,7 @@ function GetOSVersion {
os_RELEASE=$(lsb_release -r -s) os_RELEASE=$(lsb_release -r -s)
os_UPDATE="" os_UPDATE=""
os_PACKAGE="rpm" os_PACKAGE="rpm"
if [[ "Debian,Ubuntu,LinuxMint" =~ $os_VENDOR ]]; then if [[ "Debian,Ubuntu,LinuxMint,Raspbian" =~ $os_VENDOR ]]; then
os_PACKAGE="deb" os_PACKAGE="deb"
elif [[ "SUSE LINUX" =~ $os_VENDOR ]]; then elif [[ "SUSE LINUX" =~ $os_VENDOR ]]; then
lsb_release -d -s | grep -q openSUSE lsb_release -d -s | grep -q openSUSE
...@@ -1036,7 +1036,8 @@ function update_package_repo { ...@@ -1036,7 +1036,8 @@ function update_package_repo {
if [[ "$REPOS_UPDATED" != "True" || "$RETRY_UPDATE" = "True" ]]; then if [[ "$REPOS_UPDATED" != "True" || "$RETRY_UPDATE" = "True" ]]; then
# if there are transient errors pulling the updates, that's fine. # if there are transient errors pulling the updates, that's fine.
# It may be secondary repositories that we don't really care about. # It may be secondary repositories that we don't really care about.
apt_get update || /bin/true apt_get update --allow-releaseinfo-change || /bin/true
apt_get update || /bin/true
REPOS_UPDATED=True REPOS_UPDATED=True
fi fi
$xtrace $xtrace
...@@ -1927,6 +1928,7 @@ $XTRACE ...@@ -1927,6 +1928,7 @@ $XTRACE
# mode: shell-script # mode: shell-script
# End: # End:
if [[ $EUID -gt 0 ]]; then if [[ $EUID -gt 0 ]]; then
echo "####################################################" echo "####################################################"
echo "# #" echo "# #"
...@@ -1969,7 +1971,7 @@ fi ...@@ -1969,7 +1971,7 @@ fi
# Warn users who aren't on an explicitly supported distro, but allow them to # Warn users who aren't on an explicitly supported distro, but allow them to
# override check and attempt installation with ``export FORCE=yes`` # override check and attempt installation with ``export FORCE=yes``
if [[ ! ${DISTRO} =~ (wheezy|jessie|trusty|vivid|xenial|rhel7|rhel6|LinuxMint-17.2) ]]; then if [[ ! ${DISTRO} =~ (jessie|stretch|buster|bullseye|xenial|bionic|focal|rhel7) ]]; then
echo "WARNING: this script has not been tested on $DISTRO" echo "WARNING: this script has not been tested on $DISTRO"
if [[ "$FORCE" != "yes" ]]; then if [[ "$FORCE" != "yes" ]]; then
die $LINENO "If you wish to run this script anyway run with FORCE=yes" die $LINENO "If you wish to run this script anyway run with FORCE=yes"
...@@ -1978,10 +1980,10 @@ fi ...@@ -1978,10 +1980,10 @@ fi
# Make sure wheezy backports are available. # Make sure wheezy backports are available.
if [[ $DISTRO == "wheezy" ]]; then if [[ $DISTRO == "wheezy" ]]; then
echo "deb http://ftp.debian.org/debian wheezy-backports main contrib " > /etc/apt/sources.list.d/wheezy-backports.list echo "deb http://archive.debian.org/debian wheezy-backports main contrib " > /etc/apt/sources.list.d/wheezy-backports.list
fi fi
if is_fedora && [[ $DISTRO == "rhel7" ]]; then if is_fedora && [[ $DISTRO =~ rhel7|f20|f19 ]]; then
# RHEL requires EPEL for many Ansible dependencies # RHEL requires EPEL for many Ansible dependencies
# NOTE: We always remove and install latest -- some environments # NOTE: We always remove and install latest -- some environments
...@@ -2007,6 +2009,8 @@ failovermethod=priority ...@@ -2007,6 +2009,8 @@ failovermethod=priority
enabled=0 enabled=0
gpgcheck=0 gpgcheck=0
EOF EOF
yum clean all
yum makecache
yum update -y yum update -y
yum_install yum-utils yum_install yum-utils
# Enable a bootstrap repo. It is removed after finishing # Enable a bootstrap repo. It is removed after finishing
...@@ -2051,18 +2055,26 @@ EOF ...@@ -2051,18 +2055,26 @@ EOF
is_package_installed libselinux-python || install_package libselinux-python is_package_installed libselinux-python || install_package libselinux-python
fi fi
is_package_installed unzip || install_package unzip
is_package_installed openssl || install_package openssl is_package_installed openssl || install_package openssl
# stick to a specific ansible package version is_package_installed ansible || install_package ansible
if is_ubuntu; then
# is_ubuntu also handle debian distro if ! is_ubuntu || [[ $DISTRO = wheezy || $DISTRO == jessie || $DISTRO == stretch || $DISTRO == xenial || $DISTRO == bionic ]]; then
is_package_installed ansible || install_package ansible=1.* is_package_installed python-pip || install_package python-pip
elif is_fedora; then
# is_fedora also handle CentOS distro
is_package_installed ansible || install_package ansible1.9.noarch
else else
is_package_installed ansible || install_package ansible # Ansible is using python3 now
is_package_installed python3-pip || install_package python3-pip
fi
if is_ubuntu && [[ $DISTRO == focal || $DISTRO == buster ]]; then
# install python2 in distro where python2 is not installed by default and ansible python interpreter is still python2
is_package_installed python || install_package python
fi
if is_fedora && [[ $DISTRO == "f26" ]]; then
is_package_installed python2-rpm || install_package python2-rpm
is_package_installed yum || install_package yum
fi fi
is_package_installed python-setuptools || install_package python-setuptools
if is_ubuntu && [[ $DISTRO == "trusty" ]]; then if is_ubuntu && [[ $DISTRO == "trusty" ]]; then
is_package_installed python-apt || install_package python-apt is_package_installed python-apt || install_package python-apt
...@@ -2082,6 +2094,7 @@ fi ...@@ -2082,6 +2094,7 @@ fi
ANSIBLE_PLUGIN_LOCATION="/usr/share/ansible_plugins/callback_plugins/" ANSIBLE_PLUGIN_LOCATION="/usr/share/ansible_plugins/callback_plugins/"
mkdir -p $ANSIBLE_PLUGIN_LOCATION mkdir -p $ANSIBLE_PLUGIN_LOCATION
/bin/cat << EOF > $ANSIBLE_PLUGIN_LOCATION/log_parse.py /bin/cat << EOF > $ANSIBLE_PLUGIN_LOCATION/log_parse.py
from __future__ import absolute_import
import os import os
import time import time
import json import json
...@@ -2111,11 +2124,11 @@ class CallbackModule(baseModule): ...@@ -2111,11 +2124,11 @@ class CallbackModule(baseModule):
else: else:
for filename in os.listdir(self.log_path): for filename in os.listdir(self.log_path):
filepath = os.path.join(self.log_path, filename) filepath = os.path.join(self.log_path, filename)
if os.path.exists(filepath) and os.path.isfile(filepath): if os.path.isfile(filepath):
os.unlink(filepath) os.unlink(filepath)
def writeLog(self, host, category, content): def writeLog(self, host, category, content):
if not self.fd_list.has_key(category): if category not in self.fd_list:
self.fd_list[category] = open( self.fd_list[category] = open(
os.path.join(self.log_path, '%s_%s' % (host, category)), "a" os.path.join(self.log_path, '%s_%s' % (host, category)), "a"
) )
...@@ -2126,12 +2139,11 @@ class CallbackModule(baseModule): ...@@ -2126,12 +2139,11 @@ class CallbackModule(baseModule):
if host == "localhost": if host == "localhost":
host = "127.0.0.1" # keep compatibility host = "127.0.0.1" # keep compatibility
if type(data) == dict: if type(data) is dict:
if '_ansible_verbose_override' in data: if '_ansible_verbose_override' in data:
# avoid logging extraneous data # avoid logging extraneous data
return return
data = data.copy()
content = json.dumps(data) content = json.dumps(data)
if ignore_errors: if ignore_errors:
...@@ -2168,8 +2180,17 @@ class CallbackModule(baseModule): ...@@ -2168,8 +2180,17 @@ class CallbackModule(baseModule):
EOF EOF
ansible localhost -m easy_install -a name=slapcache --connection=local ansible localhost -m pip -a name=setuptools --connection=local
ansible localhost -m easy_install -a name=requests --connection=local ansible localhost -m pip -a name=wheel --connection=local
ansible localhost -m pip -a name=slapcache --connection=local
ansible localhost -m pip -a name=requests --connection=local
if [ ! -f /usr/share/ansible_plugins/mitogen.zip ]; then
wget -O /usr/share/ansible_plugins/mitogen.zip https://shacache.nxdcdn.com/3a935ff257ddc0ad4e0f23d71681e026f14f309f4bed0a8e2a217da9b294be2c676196703f0dde856ece49d711d0221deae70812f035b24aa5cdd0ca02790e85
unzip /usr/share/ansible_plugins/mitogen.zip -d /usr/share/ansible_plugins/mitogen/
mv /usr/share/ansible_plugins/mitogen/mitogen-*/* /usr/share/ansible_plugins/mitogen/
fi
# Include Additional Functions # Include Additional Functions
if [ ! -f /etc/opt/slapcache.cfg ]; then if [ ! -f /etc/opt/slapcache.cfg ]; then
......
...@@ -19,11 +19,11 @@ md5sum = f2b0f1ed27148504f220e06eaceff935 ...@@ -19,11 +19,11 @@ md5sum = f2b0f1ed27148504f220e06eaceff935
[template-kvm] [template-kvm]
filename = instance-kvm.cfg.jinja2 filename = instance-kvm.cfg.jinja2
md5sum = 17c58f74d1ed4cb7dce11bf9af71dd33 md5sum = f902dd10cb052ac262a4a96b9362b3a3
[template-kvm-cluster] [template-kvm-cluster]
filename = instance-kvm-cluster.cfg.jinja2.in filename = instance-kvm-cluster.cfg.jinja2.in
md5sum = fcb35c32ef985e3d69a7914711675dcc md5sum = 67901b7a97b324aa36851d8e8b43cad6
[template-kvm-resilient] [template-kvm-resilient]
filename = instance-kvm-resilient.cfg.jinja2 filename = instance-kvm-resilient.cfg.jinja2
...@@ -55,7 +55,7 @@ md5sum = b7e87479a289f472b634a046b44b5257 ...@@ -55,7 +55,7 @@ md5sum = b7e87479a289f472b634a046b44b5257
[template-kvm-run] [template-kvm-run]
filename = template/template-kvm-run.in filename = template/template-kvm-run.in
md5sum = f840e8b7af83982525f66d7ec12b7085 md5sum = 875261817970d0f83335824373288b9d
[template-kvm-controller] [template-kvm-controller]
filename = template/kvm-controller-run.in filename = template/kvm-controller-run.in
......
...@@ -339,6 +339,22 @@ ...@@ -339,6 +339,22 @@
"type": "string", "type": "string",
"default": "host" "default": "host"
}, },
"network-adapter": {
"title": "Network adapter",
"description": "Network adapter provided to the guest. Allows to support legacy guests, which do not have drivers for default virtio-net-pci.",
"type": "string",
"default": "virtio-net-pci",
"enum": [
"virtio-net-pci",
"e1000",
"ne2k_isa",
"ne2k_pci",
"pcnet",
"rtl8139",
"usb-net",
"vmxnet3"
]
},
"nbd-host": { "nbd-host": {
"title": "NBD hostname or IP", "title": "NBD hostname or IP",
"description": "hostname (or IP) of the NBD server containing the boot image.", "description": "hostname (or IP) of the NBD server containing the boot image.",
......
...@@ -85,7 +85,8 @@ config-disk-size = {{ dumps(kvm_parameter_dict.get('disk-size', 40)) }} ...@@ -85,7 +85,8 @@ config-disk-size = {{ dumps(kvm_parameter_dict.get('disk-size', 40)) }}
config-disk-type = {{ dumps(kvm_parameter_dict.get('disk-type', 'virtio')) }} config-disk-type = {{ dumps(kvm_parameter_dict.get('disk-type', 'virtio')) }}
config-disk-format = {{ dumps(kvm_parameter_dict.get('disk-format', 'qcow2')) }} config-disk-format = {{ dumps(kvm_parameter_dict.get('disk-format', 'qcow2')) }}
config-cpu-count = {{ dumps(kvm_parameter_dict.get('cpu-count', 2)) }} config-cpu-count = {{ dumps(kvm_parameter_dict.get('cpu-count', 2)) }}
config-cpu-max-count = {{ dumps(kvm_parameter_dict.get('cpu-max-count', int(kvm_parameter_dict.get('cpu-count', 1)) + 1)) }} config-cpu-max-count = {{ dumps(kvm_parameter_dict.get('cpu-max-count', int(kvm_parameter_dict.get('cpu-count', 2)) + 1)) }}
config-network-adapter = {{ dumps(kvm_parameter_dict.get('network-adapter', 'virtio-net-pci')) }}
{{ setconfig('numa', kvm_parameter_dict.get('numa', '')) }} {{ setconfig('numa', kvm_parameter_dict.get('numa', '')) }}
{{ setconfig('machine-options', kvm_parameter_dict.get('machine-options', '')) }} {{ setconfig('machine-options', kvm_parameter_dict.get('machine-options', '')) }}
{{ setconfig('nbd-host', kvm_parameter_dict.get('nbd-host', '')) }} {{ setconfig('nbd-host', kvm_parameter_dict.get('nbd-host', '')) }}
......
...@@ -127,6 +127,22 @@ ...@@ -127,6 +127,22 @@
"type": "string", "type": "string",
"default": "host" "default": "host"
}, },
"network-adapter": {
"title": "Network adapter",
"description": "Network adapter provided to the guest. Allows to support legacy guests, which do not have drivers for default virtio-net-pci.",
"type": "string",
"default": "virtio-net-pci",
"enum": [
"virtio-net-pci",
"e1000",
"ne2k_isa",
"ne2k_pci",
"pcnet",
"rtl8139",
"usb-net",
"vmxnet3"
]
},
"nbd-host": { "nbd-host": {
"title": "NBD hostname", "title": "NBD hostname",
"description": "hostname (or IP) of the NBD server containing the boot image.", "description": "hostname (or IP) of the NBD server containing the boot image.",
......
...@@ -434,6 +434,8 @@ disk-format = ${slap-parameter:disk-format} ...@@ -434,6 +434,8 @@ disk-format = ${slap-parameter:disk-format}
disk-device-path = ${slap-parameter:disk-device-path} disk-device-path = ${slap-parameter:disk-device-path}
disk-path = ${directory:srv}/virtual.${slap-parameter:disk-format} disk-path = ${directory:srv}/virtual.${slap-parameter:disk-format}
network-adapter = ${slap-parameter:network-adapter}
pid-file-path = ${kvm-controller-parameter-dict:pid-file} pid-file-path = ${kvm-controller-parameter-dict:pid-file}
socket-path = ${kvm-controller-parameter-dict:socket-path} socket-path = ${kvm-controller-parameter-dict:socket-path}
...@@ -1075,6 +1077,8 @@ disk-type = virtio ...@@ -1075,6 +1077,8 @@ disk-type = virtio
disk-format = qcow2 disk-format = qcow2
disk-device-path = disk-device-path =
network-adapter = virtio-net-pci
cpu-count = 2 cpu-count = 2
disk-cache = disk-cache =
disk-aio = disk-aio =
......
...@@ -19,6 +19,8 @@ qemu_path = {{ repr(parameter_dict["qemu-path"]) }} ...@@ -19,6 +19,8 @@ qemu_path = {{ repr(parameter_dict["qemu-path"]) }}
disk_size = {{ repr(parameter_dict["disk-size"]) }} disk_size = {{ repr(parameter_dict["disk-size"]) }}
disk_type = {{ repr(parameter_dict["disk-type"]) }} disk_type = {{ repr(parameter_dict["disk-type"]) }}
network_adapter = {{ repr(parameter_dict["network-adapter"]) }}
socket_path = '{{ parameter_dict.get("socket-path") }}' socket_path = '{{ parameter_dict.get("socket-path") }}'
nbd_list = (('{{ parameter_dict.get("nbd-host") }}', nbd_list = (('{{ parameter_dict.get("nbd-host") }}',
{{ parameter_dict.get("nbd-port") }}), {{ parameter_dict.get("nbd-port") }}),
...@@ -255,7 +257,7 @@ if use_nat == 'true': ...@@ -255,7 +257,7 @@ if use_nat == 'true':
if use_tap == 'true' and tap_ipv6_addr != '': if use_tap == 'true' and tap_ipv6_addr != '':
rules += ',ipv6=off' rules += ',ipv6=off'
nat_network_parameter = ['-netdev', rules, nat_network_parameter = ['-netdev', rules,
'-device', 'virtio-net-pci,netdev=lan%s,mac=%s' % (number, mac_address)] '-device', '%s,netdev=lan%s,mac=%s' % (network_adapter, number, mac_address)]
if use_tap == 'true': if use_tap == 'true':
number += 1 number += 1
vhost = '' vhost = ''
...@@ -264,7 +266,7 @@ if use_tap == 'true': ...@@ -264,7 +266,7 @@ if use_tap == 'true':
tap_network_parameter = ['-netdev', tap_network_parameter = ['-netdev',
'tap,id=lan%s,ifname=%s,script=no,downscript=no%s' % (number, 'tap,id=lan%s,ifname=%s,script=no,downscript=no%s' % (number,
tap_interface, vhost), tap_interface, vhost),
'-device', 'virtio-net-pci,netdev=lan%s,mac=%s' % (number, tap_mac_address)] '-device', '%s,netdev=lan%s,mac=%s' % (network_adapter, number, tap_mac_address)]
smp = '%s,maxcpus=%s' % (init_smp_count, smp_max_count) smp = '%s,maxcpus=%s' % (init_smp_count, smp_max_count)
ram = '%sM,slots=128,maxmem=%sM' % (init_ram_size, ram_max_size) ram = '%sM,slots=128,maxmem=%sM' % (init_ram_size, ram_max_size)
......
...@@ -1179,27 +1179,6 @@ class TestBootImageUrlSelectKvmCluster(TestBootImageUrlListKvmCluster): ...@@ -1179,27 +1179,6 @@ class TestBootImageUrlSelectKvmCluster(TestBootImageUrlListKvmCluster):
config_file_name = 'boot-image-url-select.json' config_file_name = 'boot-image-url-select.json'
@skipUnlessKvm
class TestCpuMemMaxDynamic(InstanceTestCase):
__partition_reference__ = 'cmm'
@classmethod
def getInstanceParameterDict(cls):
return {
'cpu-count': 2,
'ram-size': 2048
}
def test(self):
with open(os.path.join(
self.computer_partition_root_path, 'bin', 'kvm_raw'), 'r') as fh:
kvm_raw = fh.read()
self.assertIn('smp_count = 2', kvm_raw)
self.assertIn('smp_max_count = 3', kvm_raw)
self.assertIn('ram_size = 2048', kvm_raw)
self.assertIn("ram_max_size = '2560'", kvm_raw)
@skipUnlessKvm @skipUnlessKvm
class TestNatRules(InstanceTestCase): class TestNatRules(InstanceTestCase):
__partition_reference__ = 'nr' __partition_reference__ = 'nr'
...@@ -1611,3 +1590,101 @@ INF: Storing errors in %(error_state_file)s ...@@ -1611,3 +1590,101 @@ INF: Storing errors in %(error_state_file)s
self.assertFalse( self.assertFalse(
os.path.exists( os.path.exists(
os.path.join(self.destination_directory, 'destination'))) os.path.join(self.destination_directory, 'destination')))
@skipUnlessKvm
class TestParameterDefault(InstanceTestCase, KvmMixin):
__partition_reference__ = 'pd'
@classmethod
def getInstanceSoftwareType(cls):
return 'default'
def mangleParameterDict(self, parameter_dict):
return parameter_dict
def _test(self, parameter_dict, expected):
self.rerequestInstance(self.mangleParameterDict(parameter_dict))
self.slap.waitForInstance(max_retry=10)
kvm_raw = glob.glob(os.path.join(
self.slap.instance_directory, '*', 'bin', 'kvm_raw'))
self.assertEqual(len(kvm_raw), 1)
kvm_raw = kvm_raw[0]
with open(kvm_raw, 'r') as fh:
kvm_raw = fh.read()
self.assertIn(expected, kvm_raw)
def test_disk_type_default(self):
self._test({}, "disk_type = 'virtio'")
def test_disk_type_set(self):
self._test({'disk-type': 'ide'}, "disk_type = 'ide'")
def test_network_adapter_default(self):
self._test({}, "network_adapter = 'virtio-net-pci")
def test_network_adapter_set(self):
self._test({'network-adapter': 'e1000'}, "network_adapter = 'e1000'")
def test_cpu_count_default(self):
self._test({}, "init_smp_count = 2")
def test_cpu_count_default_max(self):
self._test({}, "smp_max_count = 3")
def test_cpu_count_set(self):
self._test({'cpu-count': 4}, "init_smp_count = 4")
def test_cpu_count_set_max(self):
self._test({'cpu-count': 4}, "smp_max_count = 5")
def test_ram_size_default(self):
self._test({}, "init_ram_size = 4096")
def test_ram_size_default_max(self):
self._test({}, "ram_max_size = '4608'")
def test_ram_size_set(self):
self._test({'ram-size': 2048}, "init_ram_size = 2048")
def test_ram_size_set_max(self):
self._test({'ram-size': 2048}, "ram_max_size = '2560'")
@skipUnlessKvm
class TestParameterResilient(TestParameterDefault):
__partition_reference__ = 'pr'
@classmethod
def getInstanceSoftwareType(cls):
return 'kvm-resilient'
@skipUnlessKvm
class TestParameterCluster(TestParameterDefault):
__partition_reference__ = 'pc'
parameter_dict = {
"disable-ansible-promise": True
}
@classmethod
def getInstanceParameterDict(cls):
return {'_': json.dumps({
"kvm-partition-dict": {
"KVM0": cls.parameter_dict
}
})}
def mangleParameterDict(self, parameter_dict):
local_parameter_dict = self.parameter_dict.copy()
local_parameter_dict.update(parameter_dict)
return {'_': json.dumps({
"kvm-partition-dict": {
"KVM0": local_parameter_dict
}
})}
@classmethod
def getInstanceSoftwareType(cls):
return 'kvm-cluster'
# Metabae # Metabase
https://www.metabase.com/ https://www.metabase.com/
## TODO: ## TODO:
* export backups for resilience * export backups for resilience
* security (proper passwords, verifiable certificate, study metabase encryption option) * security (verifiable certificate, study metabase encryption option)
[instance-profile] [instance-profile]
filename = instance.cfg.in filename = instance.cfg.in
md5sum = 143f46b125389f39905226ec9482ce2a md5sum = 5f2f7c4c2f793d609ad3c4fa0aa2f8a5
...@@ -7,10 +7,28 @@ eggs-directory = ${buildout:eggs-directory} ...@@ -7,10 +7,28 @@ eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory} develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true offline = true
[fontconfig-conf]
recipe = slapos.recipe.template:jinja2
template = ${template-fonts-conf:output}
rendered = $${directory:etc}/fonts.conf
context =
key cachedir directory:fontconfig-cache
key fonts :fonts
key includes :includes
fonts =
${android-fonts:location}
${dejavu-fonts:location}
${ipa-fonts:location}
${ipaex-fonts:location}
${liberation-fonts:location}
${ocrb-fonts:location}
includes =
${fontconfig:location}/etc/fonts/conf.d
[metabase-instance] [metabase-instance]
recipe = slapos.cookbook:wrapper recipe = slapos.cookbook:wrapper
wrapper-path = $${directory:service}/$${:_buildout_section_name_} wrapper-path = $${directory:service}/$${:_buildout_section_name_}
command-line = sh -c "cd $${directory:srv-metabase}; ${java-re-8:location}/bin/java $JAVA_ARGS -jar ${metabase.jar:location}/metabase.jar" command-line = sh -c "cd $${directory:srv-metabase}; ${java:location}/bin/java $JAVA_ARGS -jar ${metabase.jar:location}/metabase.jar"
# https://www.metabase.com/docs/latest/operations-guide/customizing-jetty-webserver.html # https://www.metabase.com/docs/latest/operations-guide/customizing-jetty-webserver.html
# note that we set org.quartz.scheduler.instanceId through $JAVA_ARGS as a workaround for machines # note that we set org.quartz.scheduler.instanceId through $JAVA_ARGS as a workaround for machines
# which cannot resolve their hostnames. See also https://github.com/metabase/metabase/issues/8373 # which cannot resolve their hostnames. See also https://github.com/metabase/metabase/issues/8373
...@@ -28,7 +46,8 @@ environment = ...@@ -28,7 +46,8 @@ environment =
MB_DB_USER=$${postgresql:superuser} MB_DB_USER=$${postgresql:superuser}
MB_DB_PASS=$${postgresql:password} MB_DB_PASS=$${postgresql:password}
MB_DB_HOST=$${postgresql:ipv4} MB_DB_HOST=$${postgresql:ipv4}
JAVA_ARGS=-Dorg.quartz.scheduler.instanceId=$${slap-connection:computer-id}.$${slap-connection:partition-id} FONTCONFIG_FILE=$${fontconfig-conf:rendered}
JAVA_ARGS=-Dorg.quartz.scheduler.instanceId=$${slap-connection:computer-id}.$${slap-connection:partition-id} -Djava.io.tmpdir="$${directory:tmp}"
hash-existing-files = hash-existing-files =
$${buildout:directory}/software_release/buildout.cfg $${buildout:directory}/software_release/buildout.cfg
...@@ -49,19 +68,56 @@ promise = check_url_available ...@@ -49,19 +68,56 @@ promise = check_url_available
name = $${:_buildout_section_name_}.py name = $${:_buildout_section_name_}.py
config-url= $${metabase-instance:url}/api/session/properties config-url= $${metabase-instance:url}/api/session/properties
[metabase-keystore-password]
recipe = slapos.cookbook:generate.password
bytes = 24
[metabase-keystore] [metabase-keystore]
recipe = plone.recipe.command recipe = plone.recipe.command
stop-on-error = true
command = command =
${java-re-8-output:keytool} \ if [ -f $${:file} ]
-genkeypair \ then
-alias "metabase" \ # XXX password used to be "insecure", but we changed to proper password.
-keyalg RSA \ # We try to list the store with the new password and if it fail we change
-keypass "$${:password}" \ # the keystore password.
-dname "CN=$${metabase-instance:ip},OU=Unit,O=Organization,L=City,S=State,C=Country" \ if ! ${java:location}/bin/keytool \
-keystore "$${:file}" \ -list \
-storepass "$${:password}" -keystore "$${:file}" \
-storepass "$${:password}"
then
echo "Migrating keystore password" && \
${java:location}/bin/keytool \
-storepasswd \
-keystore "$${:file}" \
-storepass insecure \
-new "$${:password}" && \
echo "Migrating certificate key password" && \
${java:location}/bin/keytool \
-keypasswd \
-alias "$${:alias}" \
-keypass insecure \
-new "$${:password}" \
-keystore "$${:file}" \
-storepass "$${:password}"
fi
else
${java:location}/bin/keytool \
-genkeypair \
-alias "$${:alias}" \
-keyalg RSA \
-keypass "$${:password}" \
-dname "CN=$${metabase-instance:ip},OU=Unit,O=Organization,L=City,S=State,C=Country" \
-keystore "$${:file}" \
-storepass "$${:password}"
fi
file = $${directory:etc}/.metabase_keystore file = $${directory:etc}/.metabase_keystore
password = insecure password = $${metabase-keystore-password:passwd}
alias = metabase
[postgresql-password]
recipe = slapos.cookbook:generate.password
bytes = 24
[postgresql] [postgresql]
recipe = slapos.cookbook:postgres recipe = slapos.cookbook:postgres
...@@ -69,7 +125,7 @@ bin = ${postgresql10:location}/bin/ ...@@ -69,7 +125,7 @@ bin = ${postgresql10:location}/bin/
services = $${directory:service} services = $${directory:service}
dbname = metabase_db dbname = metabase_db
superuser = metabase-psql superuser = metabase-psql
password = insecure password = $${postgresql-password:passwd}
pgdata-directory = $${directory:srv}/postgresql pgdata-directory = $${directory:srv}/postgresql
ipv4 = $${instance-parameter:ipv4-random} ipv4 = $${instance-parameter:ipv4-random}
...@@ -125,7 +181,7 @@ wrapper-path = $${directory:bin}/$${:_buildout_section_name_} ...@@ -125,7 +181,7 @@ wrapper-path = $${directory:bin}/$${:_buildout_section_name_}
command-line = command-line =
sh -e -c "\ sh -e -c "\
echo 'This will replace current database with latest backup. Hit Ctrl+C to cancel'; echo 'This will replace current database with latest backup. Hit Ctrl+C to cancel';
sleep 5; sleep 30;
$${postgresql:bin}/pg_restore \ $${postgresql:bin}/pg_restore \
--exit-on-error \ --exit-on-error \
-h $${postgresql:pgdata-directory} \ -h $${postgresql:pgdata-directory} \
...@@ -176,7 +232,7 @@ tmp = $${buildout:directory}/tmp ...@@ -176,7 +232,7 @@ tmp = $${buildout:directory}/tmp
service = $${:etc}/service service = $${:etc}/service
srv-metabase = $${:srv}/metabase srv-metabase = $${:srv}/metabase
srv-backup = $${:srv}/backup srv-backup = $${:srv}/backup
fontconfig-cache = $${buildout:directory}/.fontconfig
[publish-connection-parameter] [publish-connection-parameter]
recipe = slapos.cookbook:publish recipe = slapos.cookbook:publish
......
[buildout] [buildout]
extends = extends =
../../component/defaults.cfg ../../component/defaults.cfg
../../component/fontconfig/buildout.cfg
../../component/fonts/buildout.cfg
../../component/java/buildout.cfg ../../component/java/buildout.cfg
../../component/postgresql/buildout.cfg ../../component/postgresql/buildout.cfg
../../component/dcron/buildout.cfg ../../component/dcron/buildout.cfg
...@@ -15,10 +17,13 @@ parts = ...@@ -15,10 +17,13 @@ parts =
[python] [python]
part = python3 part = python3
[java]
<= java-re-temurin-11
[metabase.jar] [metabase.jar]
recipe = slapos.recipe.build:download recipe = slapos.recipe.build:download
url = https://downloads.metabase.com/v0.38.3/metabase.jar url = https://downloads.metabase.com/v0.41.2/metabase.jar
md5sum = b91e4b9ae13c892894fec8bab2fd195f md5sum = 630068d1ccbdc95556931fe9cfc12e61
[instance-profile] [instance-profile]
recipe = slapos.recipe.template recipe = slapos.recipe.template
......
...@@ -85,7 +85,7 @@ class TestMetabaseSetup(MetabaseTestCase): ...@@ -85,7 +85,7 @@ class TestMetabaseSetup(MetabaseTestCase):
"username": email, "username": email,
"password": "wrong" "password": "wrong"
}) })
self.assertEqual(requests.codes.bad_request, resp.status_code) self.assertEqual(resp.status_code, requests.codes.unauthorized)
session = requests.post( session = requests.post(
parse.urljoin(url, '/api/session'), parse.urljoin(url, '/api/session'),
......
[buildout] [buildout]
extends = extends =
../../software/erp5/software.cfg ../../software/erp5/software.cfg
../../component/wendelin.core/activate-WC2-preview.cfg
buildout.hash.cfg buildout.hash.cfg
parts += parts +=
......
...@@ -8,6 +8,7 @@ extends = ...@@ -8,6 +8,7 @@ extends =
../../component/pycurl/buildout.cfg ../../component/pycurl/buildout.cfg
../../component/faketime/buildout.cfg ../../component/faketime/buildout.cfg
../../component/pillow/buildout.cfg ../../component/pillow/buildout.cfg
../../component/psycopg2/buildout.cfg
../../component/python-cryptography/buildout.cfg ../../component/python-cryptography/buildout.cfg
../../component/python-mysqlclient/buildout.cfg ../../component/python-mysqlclient/buildout.cfg
../../component/python-pynacl/buildout.cfg ../../component/python-pynacl/buildout.cfg
...@@ -242,6 +243,7 @@ extra-eggs = ...@@ -242,6 +243,7 @@ extra-eggs =
${python-mysqlclient:egg} ${python-mysqlclient:egg}
${backports.lzma:egg} ${backports.lzma:egg}
${bcrypt:egg} ${bcrypt:egg}
${psycopg2:egg}
slapos.libnetworkcache slapos.libnetworkcache
supervisor supervisor
${slapos.cookbook-setup:egg} ${slapos.cookbook-setup:egg}
...@@ -393,3 +395,4 @@ mysqlclient = 1.3.12 ...@@ -393,3 +395,4 @@ mysqlclient = 1.3.12
pexpect = 4.8.0 pexpect = 4.8.0
ptyprocess = 0.6.0 ptyprocess = 0.6.0
typing = 3.7.4.3 typing = 3.7.4.3
psycopg2 = 2.8.6
...@@ -15,4 +15,4 @@ ...@@ -15,4 +15,4 @@
[template] [template]
filename = instance.cfg filename = instance.cfg
md5sum = 2a82b7d5163d042e85b14a645707a093 md5sum = 597c29546519aabe7259e416d0b92095
...@@ -32,6 +32,16 @@ repository = ${kedifa-repository:location} ...@@ -32,6 +32,16 @@ repository = ${kedifa-repository:location}
<= download-source <= download-source
repository = ${caucase-repository:location} repository = ${caucase-repository:location}
[caucase-test-runner]
recipe = slapos.recipe.template:jinja2
template = inline:#!/bin/sh
export HOSTS="$(mktemp)"
trap 'rm "$HOSTS"' EXIT
printf '%s testhost\n%s testhost\n' "$SLAPOS_TEST_IPV4" "$SLAPOS_TEST_IPV6" > "$HOSTS"
export CAUCASE_NETLOC=testhost:8000 LD_PRELOAD=${userhosts:location}/lib/userhosts.so:$LD_PRELOAD
exec python -m unittest discover -v
rendered = $${caucase:location}/host_setting.sh
[slapos.libnetworkcache] [slapos.libnetworkcache]
<= download-source <= download-source
repository = ${slapos.libnetworkcache-repository:location} repository = ${slapos.libnetworkcache-repository:location}
...@@ -77,11 +87,12 @@ repository = ${rubygemsrecipe-repository:location} ...@@ -77,11 +87,12 @@ repository = ${rubygemsrecipe-repository:location}
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
rendered = $${create-directory:etc}/$${:_buildout_section_name_} rendered = $${create-directory:etc}/$${:_buildout_section_name_}
template = inline: template = inline:
export PATH=${coreutils:location}/bin:${curl:location}/bin:${openssl:location}/bin:${git:location}/bin:${libxslt:location}/bin:${socat:location}/bin:${lmsensors:location}/bin:${rsync:location}/bin/:${buildout:bin-directory}:$PATH export PATH=${coreutils:location}/bin:${curl:location}/bin:${openssl:location}/bin:${jq:location}/bin:${sed:location}/bin:${grep:location}/bin:${git:location}/bin:${libxslt:location}/bin:${socat:location}/bin:${lmsensors:location}/bin:${rsync:location}/bin/:${buildout:bin-directory}:$PATH
export SLAPOS_TEST_IPV4=$${slap-configuration:ipv4-random} export SLAPOS_TEST_IPV4=$${slap-configuration:ipv4-random}
export SLAPOS_TEST_IPV6=$${slap-configuration:ipv6-random} export SLAPOS_TEST_IPV6=$${slap-configuration:ipv6-random}
export SLAPOS_TEST_EGGS_DIRECTORY=$${buildout:eggs-directory} export SLAPOS_TEST_EGGS_DIRECTORY=$${buildout:eggs-directory}
export SLAPOS_TEST_DEVELOP_EGGS_DIRECTORY=$${buildout:develop-eggs-directory} export SLAPOS_TEST_DEVELOP_EGGS_DIRECTORY=$${buildout:develop-eggs-directory}
export SLAPOS_TEST_POSTGRESQL_PREFIX=${postgresql:location}
[slapos-test-runner-dot-nxdtest] [slapos-test-runner-dot-nxdtest]
...@@ -98,9 +109,7 @@ template = inline: ...@@ -98,9 +109,7 @@ template = inline:
) )
TestCase( TestCase(
"caucase", "caucase",
# XXX caucase uses 2to3 dynamically in setup.py, so it only supports ['$${caucase-test-runner:rendered}'],
# runnning tests with python setup.py test
['python', 'setup.py', 'test'],
cwd="""$${caucase:location}""", cwd="""$${caucase:location}""",
summaryf=UnitTest.summary, summaryf=UnitTest.summary,
) )
......
...@@ -13,6 +13,12 @@ extends = ...@@ -13,6 +13,12 @@ extends =
../../component/socat/buildout.cfg ../../component/socat/buildout.cfg
../../component/lmsensors/buildout.cfg ../../component/lmsensors/buildout.cfg
../../component/rsync/buildout.cfg ../../component/rsync/buildout.cfg
../../component/jq/buildout.cfg
../../component/sed/buildout.cfg
../../component/grep/buildout.cfg
../../component/userhosts/buildout.cfg
../../component/postgresql/buildout.cfg
../../component/psycopg2/buildout.cfg
../../stack/slapos.cfg ../../stack/slapos.cfg
../../stack/caucase/buildout.cfg ../../stack/caucase/buildout.cfg
../../stack/nxdtest.cfg ../../stack/nxdtest.cfg
...@@ -62,6 +68,7 @@ egg = slapos.cookbook[test] ...@@ -62,6 +68,7 @@ egg = slapos.cookbook[test]
setup = ${slapos.cookbook-repository:location} setup = ${slapos.cookbook-repository:location}
depends = depends =
${slapos.core-setup:egg} ${slapos.core-setup:egg}
${psycopg2:egg}
${slapos-cookbook-dependencies:eggs} ${slapos-cookbook-dependencies:eggs}
[slapos.core-setup] [slapos.core-setup]
...@@ -241,3 +248,4 @@ pyflakes = 2.0.0 ...@@ -241,3 +248,4 @@ pyflakes = 2.0.0
zope.testing = 4.6.2 zope.testing = 4.6.2
urllib3 = 1.24.1 urllib3 = 1.24.1
pathlib = 1.0.1 pathlib = 1.0.1
psycopg2 = 2.8.6
...@@ -190,7 +190,7 @@ setproctitle = 1.1.10 ...@@ -190,7 +190,7 @@ setproctitle = 1.1.10
setuptools-dso = 1.7 setuptools-dso = 1.7
rubygemsrecipe = 0.4.3 rubygemsrecipe = 0.4.3
six = 1.12.0 six = 1.12.0
slapos.cookbook = 1.0.213 slapos.cookbook = 1.0.214
slapos.core = 1.6.19 slapos.core = 1.6.19
slapos.extension.strip = 0.4 slapos.extension.strip = 0.4
slapos.extension.shared = 1.0 slapos.extension.shared = 1.0
......
...@@ -36,6 +36,7 @@ extends = ...@@ -36,6 +36,7 @@ extends =
../../component/stunnel/buildout.cfg ../../component/stunnel/buildout.cfg
../../component/dropbear/buildout.cfg ../../component/dropbear/buildout.cfg
../../component/pycurl/buildout.cfg ../../component/pycurl/buildout.cfg
../../component/psycopg2/buildout.cfg
../slapos.cfg ../slapos.cfg
../resilient/buildout.cfg ../resilient/buildout.cfg
...@@ -127,26 +128,6 @@ md5sum = d95205a5fc2825e9709ed6db295111e2 ...@@ -127,26 +128,6 @@ md5sum = d95205a5fc2825e9709ed6db295111e2
mode = 0644 mode = 0644
#----------------
#-- Postgres driver for Python recipes.
[psycopg2-env]
PATH = ${postgresql:location}/bin:%(PATH)s
[psycopg2]
recipe = zc.recipe.egg:custom
egg = psycopg2
define = PSYCOPG_EXTENSIONS,PSYCOPG_DISPLAY_SIZE,PSYCOPG_NEW_BOOLEAN,HAVE_PQFREEMEM,HAVE_PQPROTOCOL3
environment = psycopg2-env
include-dirs =
${postgresql:location}/include
library-dirs =
${postgresql:location}/lib
rpath =
${postgresql:location}/lib
#---------------- #----------------
#-- #--
#-- Optional part allowing applications using this stack to run a custom #-- Optional part allowing applications using this stack to run a custom
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment