Commit 3b1c2fd7 authored by Vincent Pelletier's avatar Vincent Pelletier

erp5: Add support for auto-generated hostnames.

Implemented using libuserhosts. Only provided to zope processes so far, may be
extended in the future with each process seeing different sets of aliases,
with potentially different resolutions.
parent 83c5353d
...@@ -39,6 +39,23 @@ ...@@ -39,6 +39,23 @@
"uniqueItems": true, "uniqueItems": true,
"type": "array" "type": "array"
}, },
"hosts": {
"description": "Host entries to be used in addition to and/or overriding auto-generated ones",
"default": {
"erp5-catalog-0": "some-ip",
"erp5-catalog-...": "some-ip",
"erp5-cloudooo": "some-ip",
"erp5-memcached-persistent": "some-ip",
"erp5-memcached-volatile": "some-ip"
},
"patternProperties": {
".*": {
"description": "An IP to which current entry will resolve",
"type": "string"
}
},
"type": "object"
},
"frontend": { "frontend": {
"description": "Front-end slave instance request parameters", "description": "Front-end slave instance request parameters",
"properties": { "properties": {
......
...@@ -66,6 +66,7 @@ extends = ...@@ -66,6 +66,7 @@ extends =
../../component/jsl/buildout.cfg ../../component/jsl/buildout.cfg
../../component/6tunnel/buildout.cfg ../../component/6tunnel/buildout.cfg
../../component/findutils/buildout.cfg ../../component/findutils/buildout.cfg
../../component/libuserhosts/buildout.cfg
parts = parts =
rdiff-backup rdiff-backup
...@@ -101,6 +102,7 @@ parts = ...@@ -101,6 +102,7 @@ parts =
dcron dcron
dash dash
wget wget
libuserhosts
# Buildoutish # Buildoutish
patched-eggs patched-eggs
...@@ -205,6 +207,11 @@ md5sum = 6d52007d9bdc25ed0c83a49d63d59a18 ...@@ -205,6 +207,11 @@ md5sum = 6d52007d9bdc25ed0c83a49d63d59a18
filename = zope.conf.in filename = zope.conf.in
md5sum = 8fe36c41ab784f547b968dc6edd0ca29 md5sum = 8fe36c41ab784f547b968dc6edd0ca29
[template-runzope-userhosts-preloaded]
< = download-base
filename = runzope_userhosts_preloaded.in
md5sum = 479fc3a107ff965dfb6e7e1222d81f28
[template-my-cnf] [template-my-cnf]
< = download-base < = download-base
filename = my.cnf.in filename = my.cnf.in
...@@ -218,19 +225,19 @@ md5sum = b8d2d9af0c4cab45c8337aeac28d5fae ...@@ -218,19 +225,19 @@ md5sum = b8d2d9af0c4cab45c8337aeac28d5fae
[template-create-erp5-site] [template-create-erp5-site]
< = download-base < = download-base
filename = instance-create-erp5-site.cfg.in filename = instance-create-erp5-site.cfg.in
md5sum = 4504b8e58cf6eb0f17ef30c29c04432d md5sum = 2ce65ad372433ae8e11e30f3191bca68
[template-create-erp5-site-real] [template-create-erp5-site-real]
< = download-base < = download-base
filename = instance-create-erp5-site-real.cfg.in filename = instance-create-erp5-site-real.cfg.in
md5sum = ba1b23177e101b5b9f03e1c5009c81fc md5sum = e106cdd73c0a86a9b2c806062573ca8f
[template] [template]
< = template-jinja2-base < = template-jinja2-base
# XXX: "template.cfg" is hardcoded in instanciation recipe # XXX: "template.cfg" is hardcoded in instanciation recipe
filename = template.cfg filename = template.cfg
template = ${:_profile_base_location_}/instance.cfg.in template = ${:_profile_base_location_}/instance.cfg.in
md5sum = ed053189e234a27f0b2f2b8b53c532bf md5sum = 3900ab427f44e41668d9b36411e46149
extra-context = extra-context =
key mariadb_link_binary template-mariadb:link-binary key mariadb_link_binary template-mariadb:link-binary
key zope_link_binary template-zope:link-binary key zope_link_binary template-zope:link-binary
...@@ -266,6 +273,7 @@ extra-context = ...@@ -266,6 +273,7 @@ extra-context =
key libpng12_location libpng12:location key libpng12_location libpng12:location
key libreoffice_bin_location libreoffice-bin:location key libreoffice_bin_location libreoffice-bin:location
key librsvg_location librsvg:location key librsvg_location librsvg:location
key libuserhosts_location libuserhosts:location
key libxcb_location libxcb:location key libxcb_location libxcb:location
key local_bt5_repository local-bt5-repository:list key local_bt5_repository local-bt5-repository:list
key logrotate_location logrotate:location key logrotate_location logrotate:location
...@@ -286,6 +294,7 @@ extra-context = ...@@ -286,6 +294,7 @@ extra-context =
key template_mariadb_initial_setup template-mariadb-initial-setup:target key template_mariadb_initial_setup template-mariadb-initial-setup:target
key template_my_cnf template-my-cnf:target key template_my_cnf template-my-cnf:target
key template_neo template-neo:target key template_neo template-neo:target
key template_runzope_userhosts_preloaded template-runzope-userhosts-preloaded:target
key template_zeo template-zeo:target key template_zeo template-zeo:target
key template_zope template-zope:target key template_zope template-zope:target
key template_zope_conf template-zope-conf:target key template_zope_conf template-zope-conf:target
...@@ -295,7 +304,7 @@ extra-context = ...@@ -295,7 +304,7 @@ extra-context =
[template-erp5] [template-erp5]
< = download-base < = download-base
filename = instance-erp5.cfg.in filename = instance-erp5.cfg.in
md5sum = 62d07372e84d237c6b7c4ef33f98a679 md5sum = b628da98dbed2f7e46f0de207b4bd228
[template-neo] [template-neo]
< = download-base < = download-base
...@@ -310,7 +319,7 @@ md5sum = 7bbb690cb2ea38cd2aa84c8a79c50399 ...@@ -310,7 +319,7 @@ md5sum = 7bbb690cb2ea38cd2aa84c8a79c50399
[template-zope] [template-zope]
< = download-base < = download-base
filename = instance-zope.cfg.in filename = instance-zope.cfg.in
md5sum = 6a566960a2617a59fe0c8c77622dbac9 md5sum = 0db31172368f14eeeb2ae8e22af30d9e
link-binary = link-binary =
${aspell:location}/bin/aspell ${aspell:location}/bin/aspell
${coreutils:location}/bin/basename ${coreutils:location}/bin/basename
......
...@@ -7,8 +7,14 @@ promise = ${:etc}/promise ...@@ -7,8 +7,14 @@ promise = ${:etc}/promise
[erp5-bootstrap] [erp5-bootstrap]
recipe = slapos.cookbook:erp5.bootstrap recipe = slapos.cookbook:erp5.bootstrap
runner-path = ${directory:services}/erp5-bootstrap runner-path = ${directory:services}/erp5-bootstrap
{# Note: a random domain name will be picked if several point to the same IP -#}
{% set reverse_hosts = {} -%}
{% for x, y in publish['hosts-dict'].iteritems() -%}
{% do reverse_hosts.__setitem__(y, x) -%}
{% endfor -%}
{# XXX: Expect the first database to be the one to use for catalog. -#} {# XXX: Expect the first database to be the one to use for catalog. -#}
mysql-url = {{ publish['mariadb-database-list'][0] }} {% set mysql_parsed = urlparse.urlparse(publish['mariadb-database-list'][0]) -%}
mysql-url = {{ urlparse.urlunparse(mysql_parsed[:1] + (reverse_hosts.get(mysql_parsed.hostname, mysql_parsed.hostname) ~ ':' ~ mysql_parsed.port, ) + mysql_parsed[2:]) }}
{# Pick the first http[s] family found, they should be all equivalent anyway. -#} {# Pick the first http[s] family found, they should be all equivalent anyway. -#}
{% set family_list = [] -%} {% set family_list = [] -%}
{% for key, value in publish.items() -%} {% for key, value in publish.items() -%}
......
...@@ -16,6 +16,7 @@ template = {{ parameter_dict['template-create-erp5-site-real'] }} ...@@ -16,6 +16,7 @@ template = {{ parameter_dict['template-create-erp5-site-real'] }}
rendered = ${buildout:directory}/instance-create-erp5-site-real.cfg rendered = ${buildout:directory}/instance-create-erp5-site-real.cfg
extensions = jinja2.ext.do extensions = jinja2.ext.do
context = context =
import urlparse urlparse
section publish publish section publish publish
section parameter_dict instance-create-erp5-site-real-parameters section parameter_dict instance-create-erp5-site-real-parameters
key eggs_directory buildout:eggs-directory key eggs_directory buildout:eggs-directory
......
...@@ -67,12 +67,14 @@ recipe = slapos.cookbook:generate.password ...@@ -67,12 +67,14 @@ recipe = slapos.cookbook:generate.password
< = request-common < = request-common
return = return =
zope-address-list zope-address-list
hosts-dict
extra-config = extra-config =
bt5 bt5
bt5-repository-url bt5-repository-url
cloudooo-url cloudooo-url
deadlock-debugger-password deadlock-debugger-password
developer-list developer-list
hosts-dict
inituser-login inituser-login
inituser-password inituser-password
instance-count instance-count
...@@ -99,6 +101,7 @@ config-bt5-repository-url = {{ dumps(slapparameter_dict.get('bt5-repository-url' ...@@ -99,6 +101,7 @@ config-bt5-repository-url = {{ dumps(slapparameter_dict.get('bt5-repository-url'
config-cloudooo-url = ${request-cloudooo:connection-url} config-cloudooo-url = ${request-cloudooo:connection-url}
config-deadlock-debugger-password = ${deadlock-debugger-password:passwd} config-deadlock-debugger-password = ${deadlock-debugger-password:passwd}
config-developer-list = {{ dumps(slapparameter_dict.get('developer-list', [inituser_login])) }} config-developer-list = {{ dumps(slapparameter_dict.get('developer-list', [inituser_login])) }}
config-hosts-dict = {{ dumps(slapparameter_dict.get('hosts-dict', {})) }}
config-inituser-login = {{ dumps(inituser_login) }} config-inituser-login = {{ dumps(inituser_login) }}
config-inituser-password = ${inituser-password:passwd} config-inituser-password = ${inituser-password:passwd}
config-kumofs-url = ${request-memcached-persistent:connection-url} config-kumofs-url = ${request-memcached-persistent:connection-url}
...@@ -208,6 +211,11 @@ return = site_url ...@@ -208,6 +211,11 @@ return = site_url
recipe = slapos.cookbook:publish.serialised recipe = slapos.cookbook:publish.serialised
deadlock-debugger-password = ${deadlock-debugger-password:passwd} deadlock-debugger-password = ${deadlock-debugger-password:passwd}
inituser-password = ${inituser-password:passwd} inituser-password = ${inituser-password:passwd}
{#
Pick any published hosts-dict, they are expected to be identical - and there is
no way to check here.
-#}
hosts-dict = {{ '${' ~ zope_address_list_id_dict.keys()[0] ~ ':connection-hosts-dict}' }}
{% for name, value in publish_dict.items() -%} {% for name, value in publish_dict.items() -%}
{{ name }} = {{ value }} {{ name }} = {{ value }}
{% endfor -%} {% endfor -%}
......
...@@ -164,6 +164,53 @@ ipv4 = ...@@ -164,6 +164,53 @@ ipv4 =
ipv4-port = ipv4-port =
{% endif -%} {% endif -%}
{% set hosts_dict = {} -%}
{% for alias, url in (
('erp5-memcached-volatile', slapparameter_dict['memcached-url']),
('erp5-memcached-persistent', slapparameter_dict['kumofs-url']),
('erp5-cloudooo', slapparameter_dict['cloudooo-url']),
) -%}
{% do hosts_dict.__setitem__(
alias,
urlparse.urlparse(url).hostname,
) -%}
{%- endfor %}
{# jinja2 does not support enumerate... -#}
{% set catalog_counter = 0 %}
{% for url in slapparameter_dict['mysql-url-list'] -%}
{% do hosts_dict.__setitem__(
'erp5-catalog-' ~ catalog_counter,
urlparse.urlparse(url).hostname,
) -%}
{% set catalog_counter = catalog_counter + 1 -%}
{%- endfor %}
{% do hosts_dict.update(slapparameter_dict['hosts-dict']) -%}
[hosts-parameter]
host-dict = {{ dumps(hosts_dict) }}
[hosts]
recipe = slapos.recipe.template:jinja2
template = inline: {{ '
{% for alias, aliased in host_dict.items() -%}
{{ aliased }} {{ alias }}
{% endfor %}
' }}
rendered = ${directory:etc}/hosts
context = key host_dict hosts-parameter:host-dict
[preload-userhosts-runzope-parameter]
runzope-binary = {{ bin_directory }}/runzope
libuserhosts = {{ parameter_dict['libuserhosts'] }}
shell-path = {{ parameter_dict['dash'] }}/bin/dash
hosts = ${hosts:rendered}
[preload-userhosts-runzope]
recipe = slapos.recipe.template:jinja2
rendered = ${directory:bin}/runzope_userhosts_preloaded
context = section parameter_dict preload-userhosts-runzope-parameter
template = {{ parameter_dict['runzope-userhosts-preloaded-template'] }}
mode = 755
[zope-base] [zope-base]
recipe = slapos.cookbook:generic.zope.zeo.client recipe = slapos.cookbook:generic.zope.zeo.client
inituser = ${directory:instance}/inituser inituser = ${directory:instance}/inituser
...@@ -173,7 +220,7 @@ timezone = {{ dumps(slapparameter_dict['timezone']) }} ...@@ -173,7 +220,7 @@ timezone = {{ dumps(slapparameter_dict['timezone']) }}
tmp-path = ${directory:tmp} tmp-path = ${directory:tmp}
bin-path = ${directory:bin} bin-path = ${directory:bin}
site-zcml = ${directory:instance-etc}/site.zcml site-zcml = ${directory:instance-etc}/site.zcml
runzope-binary = {{ bin_directory }}/runzope runzope-binary = ${preload-userhosts-runzope:rendered}
bt5-repository = bt5-repository =
[zope-conf-parameter-base] [zope-conf-parameter-base]
...@@ -295,6 +342,14 @@ post = {{ bin_directory }}/killpidfromfile {{ '${' ~ conf_parameter_name ~ ':pid ...@@ -295,6 +342,14 @@ post = {{ bin_directory }}/killpidfromfile {{ '${' ~ conf_parameter_name ~ ':pid
[publish-zope] [publish-zope]
recipe = slapos.cookbook:publish.serialised recipe = slapos.cookbook:publish.serialised
zope-address-list = {{ dumps(publish_list) }} zope-address-list = {{ dumps(publish_list) }}
{#
Note: hosts_dist is generated at zope level rather than at erp5 (root partition)
level, as it is easier: we can access urls as python values trivially here.
This has the downside of making each zope partition publish the (hopefuly) same
dict toward erp5 partition, violating the DRY principle and making the intent
hard to guess.
-#}
hosts-dict = {{ dumps(hosts_dict) }}
[erp5-promise] [erp5-promise]
recipe = slapos.cookbook:erp5.promise recipe = slapos.cookbook:erp5.promise
......
...@@ -141,6 +141,8 @@ buildout-bin-directory = {{ buildout_bin_directory }} ...@@ -141,6 +141,8 @@ buildout-bin-directory = {{ buildout_bin_directory }}
dash = {{ dash_location }} dash = {{ dash_location }}
jsl = {{ jsl_location }} jsl = {{ jsl_location }}
link-binary = {{ dumps(zope_link_binary) }} link-binary = {{ dumps(zope_link_binary) }}
libuserhosts = {{ libuserhosts_location }}
runzope-userhosts-preloaded-template = {{ template_runzope_userhosts_preloaded }}
[dynamic-template-zope] [dynamic-template-zope]
< = jinja2-template-base < = jinja2-template-base
...@@ -150,6 +152,7 @@ extensions = jinja2.ext.do ...@@ -150,6 +152,7 @@ extensions = jinja2.ext.do
extra-context = extra-context =
key buildout_directory buildout:directory key buildout_directory buildout:directory
section parameter_dict dynamic-template-zope-parameters section parameter_dict dynamic-template-zope-parameters
import urlparse urlparse
# Must match the key id in [switch-softwaretype] which uses this section. # Must match the key id in [switch-softwaretype] which uses this section.
key software_type :software-type key software_type :software-type
software-type = zope software-type = zope
......
#!{{ parameter_dict['shell-path'] }}
export HOSTS='{{ parameter_dict['hosts'] }}' LD_PRELOAD='{{ parameter_dict['libuserhosts'] }}'
exec {{ parameter_dict['runzope-binary'] }} "$@"
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment