erp5: Add support for auto-generated hostnames.

Implemented using libuserhosts. Only provided to zope processes so far, may be
extended in the future with each process seeing different sets of aliases,
with potentially different resolutions.

software/erp5/instance-erp5-input-schema.json

@@ -39,6 +39,23 @@
       "uniqueItems": true,
       "type": "array"
     },
+    "hosts": {
+      "description": "Host entries to be used in addition to and/or overriding auto-generated ones",
+      "default": {
+        "erp5-catalog-0": "some-ip",
+        "erp5-catalog-...": "some-ip",
+        "erp5-cloudooo": "some-ip",
+        "erp5-memcached-persistent": "some-ip",
+        "erp5-memcached-volatile": "some-ip"
+      },
+      "patternProperties": {
+        ".*": {
+          "description": "An IP to which current entry will resolve",
+          "type": "string"
+        }
+      },
+      "type": "object"
+    },
     "frontend": {
       "description": "Front-end slave instance request parameters",
       "properties": {

stack/erp5/buildout.cfg

@@ -66,6 +66,7 @@ extends =
   ../../component/jsl/buildout.cfg
   ../../component/6tunnel/buildout.cfg
   ../../component/findutils/buildout.cfg
+  ../../component/libuserhosts/buildout.cfg
 
 parts =
   rdiff-backup
@@ -101,6 +102,7 @@ parts =
   dcron
   dash
   wget
+  libuserhosts
 
 # Buildoutish
   patched-eggs
@@ -205,6 +207,11 @@ md5sum = 6d52007d9bdc25ed0c83a49d63d59a18
 filename = zope.conf.in
 md5sum = 8fe36c41ab784f547b968dc6edd0ca29
 
+[template-runzope-userhosts-preloaded]
+< = download-base
+filename = runzope_userhosts_preloaded.in
+md5sum = 479fc3a107ff965dfb6e7e1222d81f28
+
 [template-my-cnf]
 < = download-base
 filename = my.cnf.in
@@ -218,19 +225,19 @@ md5sum = b8d2d9af0c4cab45c8337aeac28d5fae
 [template-create-erp5-site]
 < = download-base
 filename = instance-create-erp5-site.cfg.in
-md5sum = 4504b8e58cf6eb0f17ef30c29c04432d
+md5sum = 2ce65ad372433ae8e11e30f3191bca68
 
 [template-create-erp5-site-real]
 < = download-base
 filename = instance-create-erp5-site-real.cfg.in
-md5sum = ba1b23177e101b5b9f03e1c5009c81fc
+md5sum = e106cdd73c0a86a9b2c806062573ca8f
 
 [template]
 < = template-jinja2-base
 # XXX: "template.cfg" is hardcoded in instanciation recipe
 filename = template.cfg
 template = ${:_profile_base_location_}/instance.cfg.in
-md5sum = ed053189e234a27f0b2f2b8b53c532bf
+md5sum = 3900ab427f44e41668d9b36411e46149
 extra-context =
     key mariadb_link_binary template-mariadb:link-binary
     key zope_link_binary template-zope:link-binary
@@ -266,6 +273,7 @@ extra-context =
     key libpng12_location libpng12:location
     key libreoffice_bin_location libreoffice-bin:location
     key librsvg_location librsvg:location
+    key libuserhosts_location libuserhosts:location
     key libxcb_location libxcb:location
     key local_bt5_repository local-bt5-repository:list
     key logrotate_location logrotate:location
@@ -286,6 +294,7 @@ extra-context =
     key template_mariadb_initial_setup template-mariadb-initial-setup:target
     key template_my_cnf template-my-cnf:target
     key template_neo template-neo:target
+    key template_runzope_userhosts_preloaded template-runzope-userhosts-preloaded:target
     key template_zeo template-zeo:target
     key template_zope template-zope:target
     key template_zope_conf template-zope-conf:target
@@ -295,7 +304,7 @@ extra-context =
 [template-erp5]
 < = download-base
 filename = instance-erp5.cfg.in
-md5sum = 62d07372e84d237c6b7c4ef33f98a679
+md5sum = b628da98dbed2f7e46f0de207b4bd228
 
 [template-neo]
 < = download-base
@@ -310,7 +319,7 @@ md5sum = 7bbb690cb2ea38cd2aa84c8a79c50399
 [template-zope]
 < = download-base
 filename = instance-zope.cfg.in
-md5sum = 6a566960a2617a59fe0c8c77622dbac9
+md5sum = 0db31172368f14eeeb2ae8e22af30d9e
 link-binary =
   ${aspell:location}/bin/aspell
   ${coreutils:location}/bin/basename

stack/erp5/instance-create-erp5-site-real.cfg.in

@@ -7,8 +7,14 @@ promise = ${:etc}/promise
 [erp5-bootstrap]
 recipe = slapos.cookbook:erp5.bootstrap
 runner-path = ${directory:services}/erp5-bootstrap
+{# Note: a random domain name will be picked if several point to the same IP -#}
+{% set reverse_hosts = {} -%}
+{% for x, y in publish['hosts-dict'].iteritems() -%}
+{%   do reverse_hosts.__setitem__(y, x) -%}
+{% endfor -%}
 {# XXX: Expect the first database to be the one to use for catalog. -#}
-mysql-url = {{ publish['mariadb-database-list'][0] }}
+{% set mysql_parsed = urlparse.urlparse(publish['mariadb-database-list'][0]) -%}
+mysql-url = {{ urlparse.urlunparse(mysql_parsed[:1] + (reverse_hosts.get(mysql_parsed.hostname, mysql_parsed.hostname) ~ ':' ~ mysql_parsed.port, ) + mysql_parsed[2:]) }}
 {# Pick the first http[s] family found, they should be all equivalent anyway. -#}
 {% set family_list = [] -%}
 {% for key, value in publish.items() -%}

stack/erp5/instance-create-erp5-site.cfg.in

@@ -16,6 +16,7 @@ template = {{ parameter_dict['template-create-erp5-site-real'] }}
 rendered = ${buildout:directory}/instance-create-erp5-site-real.cfg
 extensions = jinja2.ext.do
 context =
+  import urlparse urlparse
   section publish publish
   section parameter_dict instance-create-erp5-site-real-parameters
   key eggs_directory buildout:eggs-directory

stack/erp5/instance-erp5.cfg.in

@@ -67,12 +67,14 @@ recipe = slapos.cookbook:generate.password
 < = request-common
 return =
   zope-address-list
+  hosts-dict
 extra-config =
   bt5
   bt5-repository-url
   cloudooo-url
   deadlock-debugger-password
   developer-list
+  hosts-dict
   inituser-login
   inituser-password
   instance-count
@@ -99,6 +101,7 @@ config-bt5-repository-url = {{ dumps(slapparameter_dict.get('bt5-repository-url'
 config-cloudooo-url = ${request-cloudooo:connection-url}
 config-deadlock-debugger-password = ${deadlock-debugger-password:passwd}
 config-developer-list = {{ dumps(slapparameter_dict.get('developer-list', [inituser_login])) }}
+config-hosts-dict = {{ dumps(slapparameter_dict.get('hosts-dict', {})) }}
 config-inituser-login = {{ dumps(inituser_login) }}
 config-inituser-password = ${inituser-password:passwd}
 config-kumofs-url = ${request-memcached-persistent:connection-url}
@@ -208,6 +211,11 @@ return = site_url
 recipe = slapos.cookbook:publish.serialised
 deadlock-debugger-password = ${deadlock-debugger-password:passwd}
 inituser-password = ${inituser-password:passwd}
+{#
+Pick any published hosts-dict, they are expected to be identical - and there is
+no way to check here.
+-#}
+hosts-dict = {{ '${' ~ zope_address_list_id_dict.keys()[0] ~ ':connection-hosts-dict}' }}
 {% for name, value in publish_dict.items() -%}
 {{   name }} = {{ value }}
 {% endfor -%}

stack/erp5/instance-zope.cfg.in

@@ -164,6 +164,53 @@ ipv4 =
 ipv4-port =
 {% endif -%}
 
+{% set hosts_dict = {} -%}
+{% for alias, url in (
+    ('erp5-memcached-volatile', slapparameter_dict['memcached-url']),
+    ('erp5-memcached-persistent', slapparameter_dict['kumofs-url']),
+    ('erp5-cloudooo', slapparameter_dict['cloudooo-url']),
+  ) -%}
+{%   do hosts_dict.__setitem__(
+       alias,
+       urlparse.urlparse(url).hostname,
+     )  -%}
+{%- endfor %}
+{# jinja2 does not support enumerate... -#}
+{% set catalog_counter = 0 %}
+{% for url in slapparameter_dict['mysql-url-list'] -%}
+{%   do hosts_dict.__setitem__(
+       'erp5-catalog-' ~ catalog_counter,
+       urlparse.urlparse(url).hostname,
+     ) -%}
+{%   set catalog_counter = catalog_counter + 1 -%}
+{%- endfor %}
+{% do hosts_dict.update(slapparameter_dict['hosts-dict']) -%}
+[hosts-parameter]
+host-dict = {{ dumps(hosts_dict) }}
+
+[hosts]
+recipe = slapos.recipe.template:jinja2
+template = inline: {{ '
+  {% for alias, aliased in host_dict.items() -%}
+  {{ aliased }} {{ alias }}
+  {% endfor %}
+' }}
+rendered = ${directory:etc}/hosts
+context = key host_dict hosts-parameter:host-dict
+
+[preload-userhosts-runzope-parameter]
+runzope-binary = {{ bin_directory }}/runzope
+libuserhosts = {{ parameter_dict['libuserhosts'] }}
+shell-path = {{ parameter_dict['dash'] }}/bin/dash
+hosts = ${hosts:rendered}
+
+[preload-userhosts-runzope]
+recipe = slapos.recipe.template:jinja2
+rendered = ${directory:bin}/runzope_userhosts_preloaded
+context = section parameter_dict preload-userhosts-runzope-parameter
+template = {{ parameter_dict['runzope-userhosts-preloaded-template'] }}
+mode = 755
+
 [zope-base]
 recipe = slapos.cookbook:generic.zope.zeo.client
 inituser = ${directory:instance}/inituser
@@ -173,7 +220,7 @@ timezone = {{ dumps(slapparameter_dict['timezone']) }}
 tmp-path = ${directory:tmp}
 bin-path = ${directory:bin}
 site-zcml = ${directory:instance-etc}/site.zcml
-runzope-binary = {{ bin_directory }}/runzope
+runzope-binary = ${preload-userhosts-runzope:rendered}
 bt5-repository =
 
 [zope-conf-parameter-base]
@@ -295,6 +342,14 @@ post = {{ bin_directory }}/killpidfromfile {{ '${' ~ conf_parameter_name ~ ':pid
 [publish-zope]
 recipe = slapos.cookbook:publish.serialised
 zope-address-list = {{ dumps(publish_list) }}
+{#
+Note: hosts_dist is generated at zope level rather than at erp5 (root partition)
+level, as it is easier: we can access urls as python values trivially here.
+This has the downside of making each zope partition publish the (hopefuly) same
+dict toward erp5 partition, violating the DRY principle and making the intent
+hard to guess.
+-#}
+hosts-dict = {{ dumps(hosts_dict) }}
 
 [erp5-promise]
 recipe = slapos.cookbook:erp5.promise

stack/erp5/instance.cfg.in

@@ -141,6 +141,8 @@ buildout-bin-directory = {{ buildout_bin_directory }}
 dash = {{ dash_location }}
 jsl = {{ jsl_location }}
 link-binary = {{ dumps(zope_link_binary) }}
+libuserhosts = {{ libuserhosts_location }}
+runzope-userhosts-preloaded-template = {{ template_runzope_userhosts_preloaded }}
 
 [dynamic-template-zope]
 < = jinja2-template-base
@@ -150,6 +152,7 @@ extensions = jinja2.ext.do
 extra-context =
     key buildout_directory buildout:directory
     section parameter_dict dynamic-template-zope-parameters
+    import urlparse urlparse
 # Must match the key id in [switch-softwaretype] which uses this section.
     key software_type :software-type
 software-type = zope

stack/erp5/runzope_userhosts_preloaded.in

+#!{{ parameter_dict['shell-path'] }}
+export HOSTS='{{ parameter_dict['hosts'] }}' LD_PRELOAD='{{ parameter_dict['libuserhosts'] }}'
+exec {{ parameter_dict['runzope-binary'] }} "$@"