From 761bed516445a7d93c2643dd6296043675cae233 Mon Sep 17 00:00:00 2001 From: Vincent Pelletier <vincent@nexedi.com> Date: Sat, 29 Sep 2012 05:31:46 +0200 Subject: [PATCH] Make ERP5 SR able to deploy an ERP5 cluster. --- component/findutils/buildout.cfg | 8 + slapos/recipe/generic_mysql/mysql.py | 6 +- software/erp5/README.txt | 408 +++++++++++++++- stack/erp5/buildout.cfg | 148 +++--- stack/erp5/instance-balancer.cfg.in | 159 +++++++ stack/erp5/instance-cloudoo.cfg.in | 35 +- stack/erp5/instance-cluster-zope.cfg.in | 294 ++++++++++++ stack/erp5/instance-erp5-cluster.cfg.in | 245 ++++++---- stack/erp5/instance-erp5-single.cfg.in | 13 +- stack/erp5/instance-kumofs.cfg.in | 53 ++- stack/erp5/instance-logrotate-base.cfg.in | 50 ++ stack/erp5/instance-mariadb.cfg.in | 321 +++++++------ stack/erp5/instance-memcached.cfg.in | 147 ------ stack/erp5/instance-tidstorage.cfg.in | 538 ---------------------- stack/erp5/instance-varnish.cfg.in | 2 +- stack/erp5/instance-zeo.cfg.in | 220 +++++++++ stack/erp5/instance-zope.cfg.in | 6 +- stack/erp5/instance.cfg.in | 191 +++++--- stack/erp5/mariadb_initial_setup.sql.in | 16 + stack/erp5/my.cnf.in | 79 ++++ stack/erp5/zope.conf.in | 4 +- 21 files changed, 1862 insertions(+), 1081 deletions(-) create mode 100644 component/findutils/buildout.cfg create mode 100644 stack/erp5/instance-balancer.cfg.in create mode 100644 stack/erp5/instance-cluster-zope.cfg.in create mode 100644 stack/erp5/instance-logrotate-base.cfg.in delete mode 100644 stack/erp5/instance-memcached.cfg.in delete mode 100644 stack/erp5/instance-tidstorage.cfg.in create mode 100644 stack/erp5/instance-zeo.cfg.in create mode 100644 stack/erp5/mariadb_initial_setup.sql.in create mode 100644 stack/erp5/my.cnf.in diff --git a/component/findutils/buildout.cfg b/component/findutils/buildout.cfg new file mode 100644 index 000000000..2b9f7e6a5 --- /dev/null +++ b/component/findutils/buildout.cfg @@ -0,0 +1,8 @@ +[buildout] +parts = + findutils + +[findutils] +recipe = slapos.recipe.cmmi +url = http://ftp.gnu.org/gnu/findutils/findutils-4.4.2.tar.gz +md5sum = 351cc4adb07d54877fa15f75fb77d39f diff --git a/slapos/recipe/generic_mysql/mysql.py b/slapos/recipe/generic_mysql/mysql.py index 8e0e0bf6c..8ba7f3c57 100644 --- a/slapos/recipe/generic_mysql/mysql.py +++ b/slapos/recipe/generic_mysql/mysql.py @@ -66,8 +66,9 @@ def updateMysql(args): with open(script_filename) as script_file: conf['mysql_script'] = script_file.read() while True: - mysql_upgrade_list = [conf['mysql_upgrade_binary'], '--no-defaults', '--user=root'] + mysql_upgrade_list = [conf['mysql_upgrade_binary'], '--user=root'] if 'socket' in conf: + mysql_upgrade_list.insert(1, '--no-defaults') mysql_upgrade_list.append('--socket=' + conf['socket']) mysql_upgrade = subprocess.Popen(mysql_upgrade_list, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) result = mysql_upgrade.communicate()[0] @@ -80,8 +81,9 @@ def updateMysql(args): print "MySQL database upgraded with result:\n%s" % result else: print "No need to upgrade MySQL database" - mysql_list = [conf['mysql_binary'].strip(), '--no-defaults', '-B', '--user=root'] + mysql_list = [conf['mysql_binary'].strip(), '-B', '--user=root'] if 'socket' in conf: + mysql_list.insert(1, '--no-defaults') mysql_list.append('--socket=' + conf['socket']) mysql = subprocess.Popen(mysql_list, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) diff --git a/software/erp5/README.txt b/software/erp5/README.txt index 6a1ac6885..a8f16bbf4 100644 --- a/software/erp5/README.txt +++ b/software/erp5/README.txt @@ -1,11 +1,131 @@ -TODO: improve +Software types +============== +Which software type is an entry point and can be used for root software +instance. -Instance Parameters -=================== +Parameters are expected to be passed as of *.serialised recipes expect them:: -Zope Parameters ---------------- -Needed by software-type development (default) and zope. +``` +<?xml version='1.0' encoding='utf-8'?> +<instance> + <parameter id="_">...</parameter> +</instance> +``` + +where `...` is a json expression (typically a dict). + +TCPv4 ports allocation +---------------------- +Service listening ports are allocated in the following pattern. +Base port of each software can be overridden, those are the default values. +- kumofs (persistent) + 2000: manager + 2001: server port (?) + 2002: server listen port (?) + 2003: gateway port (?) +- kumofs (volatile) + 2010: manager + 2011: server port (?) + 2012: server listen port (?) + 2013: gateway port (?) +- cloudooo + 2020: cloudooo + 2021: openoffice +- mariadb + 2099: mariadb +- zeo & tidstorage + 2100: tidstorage + 2101: first zeo + 2102: second zeo + (etc) +- haproxy + 2150: first haproxy + 2151: first apache + 2152: second haproxy + 2153: second apache + (etc) +- cluster-zope + 2200: first zope + 2201: second zope + (etc) +Note: these are not applicable when (yet unsupported) ipv6 mode is enabled, as +stunnel (used to tunnel ipv4-only services over ipv6) needs its own listening +ports. +This pattern was chosen to make it possible to deploy this software release in +a setup where all partitions would share the same IPv4 without having to +provide many parameters. +Some ports are unused in the overall range to allow software types to grow. + +common +------ +Parameters which are available to all software types. + +'mariadb-dict' (dict, optional) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +MariaDB (used for catalog, activity tables and id generator) instance +parameters. +Defaults to {}. +Possible keys and associated value types: +'tcpv4-port' (int, optional) + TCPv4 port to listen on. + Defaults to 2099. +'database-list' (list, optional) + Define the list of databases mariadb must provide, and the user having entire + access to each database. Each entry in the list is a dict, with these + possible keys and associated value types: + 'name' (str, mandatory) + Database name + 'user' (str, mandatory) + User login + 'password' (str, mandatory) + User password + Defaults to: [{'name': 'erp5', 'user': 'user', 'password': 'insecure'}] +'test-database-amount' (int, optional) + Number of test databases to generate in addition of requested databases. + A test database, if it were provided as a database-list entry, would look + like: + {'name': 'erp5_test_0', 'user': 'testuser_0', 'password': 'testpassword0'} + with '0' being all numbers from 0 to test-database-amount - 1. + Defaults to 30. + Note: the default is way too much for "normal" usage. You are encouraged to + provide this key to some lower value: 0 if you don't intend to run any unit + test (ex: production instance) 3..5 if you intend to run tests without + paralellism. +'full-retention-days' (int, optional) + The number of days full backups will be kept. + If -1, backups are disabled. + If 0, backup never expire. + Defaults to 7. +'incremental-retention-days' (int, optional) + The number of days binlogs will be kept. + If -1, binlogs are disabled. + If 0, binlogs are never expired. + Defaults to 'full-retention-days' value. +'innodb-buffer-pool-size' (str, optional) + See mariadb documentation for innodb_buffer_pool_size configuration + parameter. Value is used verbatim in configuration file. + Empty string means unconfigured (ie, bail out to mariadb's default). + Defaults to "". +'innodb-log-file-size' (str, optional) + See mariadb documentation for innodb_log_file_size configuration parameter. + Value is used verbatim in configuration file. + Empty string means unconfigured (ie, bail out to mariadb's default). + Defaults to "". +'innodb-log-buffer-size' (str, optional) + See mariadb documentation for innodb_log_buffer_size configuration parameter. + Value is used verbatim in configuration file. + Empty string means unconfigured (ie, bail out to mariadb's default). + Defaults to "". +'mariadb-relaxed-writes' (int, optional) + Controls relaxed writes, which improves performances at the cost of data + safety. DO NOT ENABLE THIS ON PRODUCTION. It's fine for unit tests and may be + acceptable for development instances. Set to 1 to enable. + Default: 0 + +single (default) +---------------- +This creates an ERP5 instance suited for small needs/resources, or local +development. A minimal ERP5 site is created when instance is started. frontend-software-url ~~~~~~~~~~~~~~~~~~~~~ @@ -31,3 +151,279 @@ frontend-domain (optional) ~~~~~~~~~~~~~~~~~~~~~~~~~~ Domain name frontend must recognise as belonging to this instance. +cluster +------- +This creates a massive ERP5 instance suited for high-demanding production +setups - which also have a lot of available resources (several machines, +several CPUs per machine, GBs of ram, several machines...). + +For each available key in the outmost dict are described below. +Publishes a dict in which each entry is the URL to a balancer entry point +(apache listening socket), witht the same keys as zope-partition-dict. + +'zodb-software-type' (str, optional) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Storage mechanism to use. To know the list of supported values, see all keys in +instance.cfg.in's section [switch-softwaretype] which start with "zodb-". +Defaults to 'zeo'. + +'zodb-tcpv4-port' (int, optional) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Base TCPv4 port for ZODB provider, if applicable (depends on chosen +software-type). +Defaults to 2100. + +'zodb-dict' (dict, optional) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Describes ZODBs to create and use in the instance. At least one entry is +required to achieve anything sensible with the instance. +key (str) + Zope internal name for this ZODB. Used to tell mountpoints apart in the ZMI + when populating them. +value (dict) + key (str) + Possible keys and associated value types: + 'storage-family' (str, optional) + Storage family. All zodbs requested with the same value are provided + by the same service (ex: same ZEO process). Might not be supported by + all zodb-software-type, silently ignored if so (ie, each ZODB gets its + own family). + Defaults to 'default'. + 'mount-point' (str, optional) + Storage mount point. + Defaults to '/'. + 'cache-size' (int, optional) + Storage ZODB cache size (aka 'Connection cache'), in objects. No value + is specified when negative is provided (ie, uses ZODB's default). + Defaults to -1. + 'storage-dict' (dict, optional) + Storage-type-specific parameters. For example, it can be used to tell where + a ZEO filestorage database is located. + When zodb-software-type is 'zeo', the following keys are supported: + 'path' (string, optional) + FileStorage file path. Occurrences of '%(zodb)s' are replaced with the + path to partition's srv/zodb directory. + Defaults to '%(zodb)s/', Zope's internal name for this ZODB, and '.fs'. + 'client' (dict, optional) + Client(=Zope)-side settings. 'server' and 'storage' keys will be + overwritten. Keys and values are expected to be strings, and map + directly to options in the resulting <zeoclient> section. + Defaults to {}. + +'tidstorage-dict' (dict, optional) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Backup parameters for tidstorage-related backup scripts. +TIDStorage is not deployed if not provided. +key (str) + Possible keys and associated value types: + 'zodb-dict' (dict, optional) + key (str) + (same as zodb-dict) + value (str, optional) + Path to store backups of this zodb into. Occurrences of '%(backup)s' are + replaced with the path to partition's srv/backup/zodb directory. + Defaults to {}. + If an item is missing compared to zodb-dict, value defaults to + '%(backup)s/' + key. + 'timestamp-path' (str, optional) + Path to backup timestamp file. + Occurrences of '%(backup)s' are replaced with the path to partition's + srv/backup/tidstorage directory. + Defaults to '%(backup)s/repozo_tidstorage_timestamp.log'. + +'zope-partition-dict' (dict, optional) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +key (str) + Instance name. +value (dict) + Possible keys and associated value types: + 'family' (str, optional) + The family this partition is part of. For example: 'public', 'admin', + 'backoffice', 'web-service'... Each family gets its own frontend + (=client-facing ip & port). It has no special meaning as far as + buildout is concerned. + Defaults to 'default'. + 'instance-count' (int, optional) + The number of Zopes to setup on this partition. + Defaults to 1. + 'thread-amount' (int, optional) + The number of worker threads for each created Zope process. + Defaults to 1. + 'timerserver-interval' (int, optional) + The timerserver tick perdiod. 0 to disable timerserver. + Defaults to 5. + 'computer-guid' (string, optional) + Computer on which partition should be allocated. + Defauts to the same computer as the one this .cfg is executed in. + 'longrequest-logger-interval' (int, optional) + The period, in seconds, with which LongRequestLogger polls worker thread + stack traces. -1 to disable. + Defaults to -1. + 'longrequest-logger-timeout' (int, optional) + The transaction duration after which LongRequestLogger will start logging + its stack trace, in seconds. Ignored if longrequest-logger-interval is -1. + Defaults to 1. + 'port-base' (int, optional) + Start allocating ports at this value. Useful if one needs to make several + partitions share the same port range (ie, several partitions bound to a + single address). + Defaults to 2200. + +'haproxy-maxconn' (int, optional) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +The number of connections haproxy accepts for a given backend. +See haproxy's "server maxconn" setting. +Defaults to 1 (correct for single-worker-threaded zopes). + +'haproxy-tcpv4-port' (int, optional) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Base TCPv4 port for load-balancer (haproxy + backend apache). +Defaults to 2150. + +'haproxy-server-check-path' (str, optional) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +The path haproxy accesses on a each backend to test their responsiveness. +Occurrences of '%(site-id)s' are replaced with site-id value (see below). +Defaults to '/'. + +'apache-backend-path' (str, optional) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Used as a rewriterule to strip components from site's URL. +Occurrences of '%(site-id)s' are replaced with site-id value (see below). +XXX: You may want to avoid using this when also requesting a frontend apache. +Defaults to '/' (ie, nothing is stripped). + +'apache-access-control-string' (str, optional) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +The list of hosts apache accepts connections from. +Defaults to 'all'. + +'apache-ssl-authentication' (booleanish, optional) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Controls certificate-based authentication. +Defaults to '0' (disabled). + +'site-id' (str, optional) +~~~~~~~~~~~~~~~~~~~~~~~~~ +Site object's id. Defaults to 'erp5'. + +'ca' (dict, optional) +~~~~~~~~~~~~~~~~~~~~~ +Certificate autority parameters. +Possible keys and associated value types: +'country-code' (str, optional) + ISO 3166-1 alpha-2 country code. Defaults to 'ZZ'. +'email' (str, optional) + E-mail address. Defaults to 'nobody@example.com' +'state' (str, optional) + State name. Defaults to 'Dummy State'. +'city' (str, optional) + City name. Default to 'Dummy City'. +'company' (str, optional) + Company name. Defaults to 'Dummy Company'. + +'mariadb-computer-guid' (str, optional) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Computer GUID identifying the partition mariadb is to be requested on. +Defaults to "cluster" software type's partition's effective computer GUID. + +'cloudooo-tcpv4-port' (int, optional) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Base TCPv4 port for cloudooo. +Defaults to 2020. + +'cloudooo-computer-guid' (str, optional) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Computer GUID identifying the partition cloudooo is to be requested on. +Defaults to "cluster" software type's partition's effective computer GUID. + +'memcached-size' (int, optional) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Megabytes of ram to allocate for volatile memcached use. +Defaults to 64. +Negative/zero values cause undefined behaviour which may change in the future. + +'memcached-tcpv4-port' +~~~~~~~~~~~~~~~~~~~~~~ +Base TCPv4 port for volatile memcached. +Defaults to 2010. + +'memcached-computer-guid' (str, optional) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Computer GUID identifying the partition memcached is to be requested on. +Defaults to "cluster" software type's partition's effective computer GUID. + +'kumofs-tcpv4-port' +~~~~~~~~~~~~~~~~~~~ +Base TCPv4 port for persistent memcached. +Defaults to 2000. + +'kumofs-computer-guid' (str, optional) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Computer GUID identifying the partition kumofs is to be requested on. +Defaults to "cluster" software type's partition's effective computer GUID. + +'zodb-computer-guid' (str, optional) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Computer GUID identifying the partition ZODB server is to be requested on. +Defaults to "cluster" software type's partition's effective computer GUID. + +'balancer-computer-guid' (str, optional) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Computer GUID identifying the partition balander (haproxy, apache, some HTTP +cache) is to be requested on. +Defaults to "cluster" software type's partition's effective computer GUID. + +'font-url-list' (list of strings, optional) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +List of extra fonts URLs to be used by cloudooo. +Defaults to []. + +'bt5' (str, optional) +~~~~~~~~~~~~~~~~~~~~~ +XXX: what is this ? +Defaults to 'erp5_full_text_myisam_catalog \ +erp5_configurator_standard \ +erp5_configurator_maxma_demo \ +erp5_configurator_ung \ +erp5_configurator_run_my_doc'. + +'bt5-repository-url' (str, optional) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +XXX: what is this ? +Defaults to SR's buildout['local-bt5-repository']['list']. + +'smtp-url' (str, optional) +~~~~~~~~~~~~~~~~~~~~~~~~~~ +XXX: what is this ? +Defaults to 'smtp://localhost:25/'. + +'timezone' (str, optional) +~~~~~~~~~~~~~~~~~~~~~~~~~~ +Timezone to put processes in (default timezone for DateTime instances). +Defaults to 'Europe/Paris'. + +'frontend-software-url' (str, optional) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Frontend's software url. +Defaults to ''. +If non-empty, the following options are used (otherwise, they are all +optional and ignored): + +'frontend-software-type' (str, optional) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Frontend's software type. +Defaults to 'RootSoftwareInstance' + +'frontend-instance-guid' (str, mandatory) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Instance GUID identifying the partition frontend is on. + +'frontend-domain' (str, optional) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +The domain name used to access this ERP5 cluster, ignored if empty. +Defaults to ''. + +For a better description of these parameters, see a frontend software +release documentation. + diff --git a/stack/erp5/buildout.cfg b/stack/erp5/buildout.cfg index 4dac1445c..ad14c8050 100644 --- a/stack/erp5/buildout.cfg +++ b/stack/erp5/buildout.cfg @@ -55,7 +55,6 @@ extends = ../../component/w3m/buildout.cfg ../../component/xorg/buildout.cfg ../../component/poppler/buildout.cfg - ../../component/xtrabackup/buildout.cfg ../../component/zabbix/buildout.cfg ../../component/sed/buildout.cfg ../../component/coreutils/buildout.cfg @@ -65,6 +64,8 @@ extends = ../../component/aspell/buildout.cfg ../../component/cloudooo/buildout.cfg ../../component/jsl/buildout.cfg + ../../component/6tunnel/buildout.cfg + ../../component/findutils/buildout.cfg parts = rdiff-backup @@ -73,6 +74,7 @@ parts = apache apache-antiloris file + findutils graphviz haproxy jsl @@ -137,6 +139,11 @@ parts = # Create instance template template +[download-base] +recipe = slapos.recipe.build:download +url = ${:_profile_base_location_}/${:filename} +mode = 644 + # Local development [slapos.cookbook-repository] recipe = slapos.recipe.build:gitclone @@ -164,63 +171,46 @@ context = ${:extra-context} [template-mariadb] -< = template-jinja2-base -filename = instance-mariadb.cfg -md5sum = 49ae6f1bf97d3755978fe08b8e29fb5a -extra-context = - key coreutils_location coreutils:location - key dcron_location dcron:location - key gettext_location gettext:location - key grep_location grep:location - key gzip_location gzip:location - key logrotate_location logrotate:location - key mariadb_location mariadb:location - key perl_location perl:location - key perl_siteprefix perl:siteprefix - key sed_location sed:location - key xtrabackup_location xtrabackup:location +< = download-base +filename = instance-mariadb.cfg.in +md5sum = defc8f21dd651dab65102f889740a4e5 [template-zope] -recipe = slapos.recipe.build:download -url = ${:_profile_base_location_}/instance-zope.cfg.in -md5sum = 69556ddc9773fb95896b7762e998bb9b -mode = 640 +< = download-base +filename = instance-zope.cfg.in +md5sum = 2b68521ebde309d6453336f1a2e46381 [template-kumofs] -< = template-jinja2-base -filename = instance-kumofs.cfg -md5sum = 90a321be12ee977800d590bf941021ef -extra-context = - key dash_location dash:location - key dcron_location dcron:location - key gzip_location gzip:location - key kumo_location kumo:location - key logrotate_location logrotate:location - -[template-tidstorage] -recipe = slapos.recipe.build:download -url = ${:_profile_base_location_}/instance-tidstorage.cfg.in -md5sum = 5572d10b343bd2de90deeaf55dd0fdc7 -mode = 640 +< = download-base +filename = instance-kumofs.cfg.in +md5sum = 0dd51fb85ced7e77d4a8544115f90f8e [template-cloudooo] -recipe = slapos.recipe.build:download -url = ${:_profile_base_location_}/instance-cloudoo.cfg.in -md5sum = 53492e520be57f4c6a9eacd107c8d446 -mode = 640 +< = download-base +filename = instance-cloudoo.cfg.in +md5sum = 6660382993d14e1d538e737e881219ba [template-zope-conf] -recipe = slapos.recipe.build:download -url = ${:_profile_base_location_}/zope.conf.in -md5sum = bc2154161a1d5baddc4ed4dfaaf94fbe -mode = 640 +< = download-base +filename = zope.conf.in +md5sum = c4c42d06c63d33de703fc03415e34d84 + +[template-my-cnf] +< = download-base +filename = my.cnf.in +md5sum = 1339485f80e049fd22d0270ebf044aa6 + +[template-mariadb-initial-setup] +< = download-base +filename = mariadb_initial_setup.sql.in +md5sum = 564006953b7d7a12d40a14b6648b32f0 [template] < = template-jinja2-base # XXX: "template.cfg" is hardcoded in instanciation recipe filename = template.cfg template = ${:_profile_base_location_}/instance.cfg.in -md5sum = a1a5a9983207e4a8128bab331cbd3cd5 +md5sum = d4b6e9bdaea4c11827dcb7a30c2d20c9 extra-context = key apache_location apache:location key aspell_location aspell:location @@ -232,9 +222,11 @@ extra-context = key dmtx_utils_location dmtx-utils:location key erp5_location erp5:location key file_location file:location + key findutils_location findutils:location key fontconfig_location fontconfig:location key fonts_location fonts:location key freetype_location freetype:location + key gettext_location gettext:location key git_location git:location key graphviz_location graphviz:location key grep_location grep:location @@ -242,6 +234,7 @@ extra-context = key haproxy_location haproxy:location key imagemagick_location imagemagick:location key jsl_location jsl:location + key kumo_location kumo:location key libICE_location libICE:location key libSM_location libSM:location key libX11_location libX11:location @@ -257,17 +250,24 @@ extra-context = key logrotate_location logrotate:location key mariadb_location mariadb:location key openssl_location openssl:location + key perl_location perl:location + key perl_siteprefix perl:siteprefix key poppler_location poppler:location key sed_location sed:location + key sixtunnel_location 6tunnel:location key stunnel_location stunnel:location + key template_balancer template-balancer:target key template_cloudooo template-cloudooo:target + key template_cluster_zope template-cluster-zope:target key template_erp5_single template-erp5-single:target key template_erp5_cluster template-erp5-cluster:target - key template_kumofs template-kumofs:rendered - key template_mariadb template-mariadb:rendered - key template_memcached template-memcached:rendered - key template_tidstorage template-tidstorage:target + key template_kumofs template-kumofs:target + key template_logrotate_base template-logrotate-base:rendered + key template_mariadb template-mariadb:target + key template_mariadb_initial_setup template-mariadb-initial-setup:target + key template_my_cnf template-my-cnf:target key template_varnish template-varnish:target + key template_zeo template-zeo:target key template_zope template-zope:target key template_zope_conf template-zope-conf:target key tesseract_location tesseract:location @@ -276,34 +276,44 @@ extra-context = key wget_location wget:location key zlib_location zlib:location -[template-memcached] +[template-erp5-single] +< = download-base +filename = instance-erp5-single.cfg.in +md5sum = 77ec4f095a22b5bd7dfea9dff63bade5 + +[template-erp5-cluster] +< = download-base +filename = instance-erp5-cluster.cfg.in +md5sum = c745d794b28cae64feba527f894d7340 + +[template-zeo] +< = download-base +filename = instance-zeo.cfg.in +md5sum = 6d0bdee21ac4837f07852b98b6fcea36 + +[template-cluster-zope] +< = download-base +filename = instance-cluster-zope.cfg.in +md5sum = 019f306860857e8536f26aba32101b30 + +[template-balancer] +< = download-base +filename = instance-balancer.cfg.in +md5sum = 9c9741c4db7e46f3cd13c0b2f392fd5d + +[template-logrotate-base] < = template-jinja2-base -filename = instance-memcached.cfg -md5sum = 346c864c1f119360eddb5e163f16d4f3 +filename = instance-logrotate-base.cfg +md5sum = af19ff0c7817df85987c69738fb083f2 extra-context = - key dash_location dash:location key dcron_location dcron:location key gzip_location gzip:location - key kumo_location kumo:location key logrotate_location logrotate:location -[template-erp5-single] -recipe = slapos.recipe.build:download -url = ${:_profile_base_location_}/instance-erp5-single.cfg.in -md5sum = 2a4312202a3160f874c4fa3ee9155a1d -mode = 640 - -[template-erp5-cluster] -recipe = slapos.recipe.build:download -url = ${:_profile_base_location_}/instance-erp5-cluster.cfg.in -md5sum = eecde68b9a266d8883042540ac077839 -mode = 640 - [template-varnish] -recipe = slapos.recipe.build:download -url = ${:_profile_base_location_}/instance-varnish.cfg.in -md5sum = ae0465591c22c0cb316c1706965c6b18 -mode = 640 +< = download-base +filename = instance-varnish.cfg.in +md5sum = b3020e1da069d43ece0de99b106cfb45 [bt5-repository] # Format: diff --git a/stack/erp5/instance-balancer.cfg.in b/stack/erp5/instance-balancer.cfg.in new file mode 100644 index 000000000..7f26f7179 --- /dev/null +++ b/stack/erp5/instance-balancer.cfg.in @@ -0,0 +1,159 @@ +{% if software_type == slap_software_type -%} +{% set part_list = [] -%} +{% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%} +{% set use_ipv6 = slapparameter_dict.get('use-ipv6', False) -%} +{# +XXX: This template only supports exactly one IPv4 and (if ipv6 is used) one IPv6 +per partition. No more (undefined result), no less (IndexError). +-#} +# TODO: insert varnish between apache & haproxy. +# And think of a way to specify which urls goe through varnish, which go +# directly to haproxy. (maybe just passing literal configuration file chunk) +{% set ipv4 = (ipv4_set | list)[0] -%} + +{% if use_ipv6 -%} +{% set ipv6 = (ipv6_set | list)[0] -%} +[zope-tunnel-base] +recipe = slapos.cookbook:ipv4toipv6 +runner-path = ${directory:services}/${:base-name} +6tunnel-path = {{ parameter_dict['6tunnel'] }}/bin/6tunnel +shell-path = {{ parameter_dict['dash'] }}/bin/dash +ipv4 = {{ ipv4 }} + +{% endif -%} +{% set haproxy_dict = {} -%} +{% set apache_dict = {} -%} +{% set next_port = slapparameter_dict['tcpv4-port'] -%} +{% for family_name, parameter_id_list in slapparameter_dict['zope-family-dict'].items() -%} +{% set zope_family_address_list = [] -%} +{% for parameter_id in parameter_id_list -%} +{% set zope_address_list = slapparameter_dict[parameter_id] -%} +{% for zope_address in zope_address_list -%} +{% if use_ipv6 -%} +[{{ section('zope-tunnel-' ~ next_port) }}] +< = zope-tunnel-base +base-name = {{ 'zeo-tunnel-' ~ next_port }} +ipv4-port = {{ next_port }} +ipv6-port = {{ zope_address.split(']:')[1] }} +ipv6 = {{ zope_address.split(']:')[0][1:] }} +{% set zope_effective_address = ipv4 ~ ":" ~ next_port -%} +{% set next_port = next_port + 1 -%} +{% else -%} +{% set zope_effective_address = zope_address -%} +{% endif -%} +{% do zope_family_address_list.append(zope_effective_address) -%} +{% endfor -%} +{% endfor -%} +{% set haproxy_port = next_port -%} +{% set next_port = next_port + 1 -%} +{% do haproxy_dict.__setitem__(family_name, (haproxy_port, zope_family_address_list)) -%} +{% do apache_dict.__setitem__(family_name, (next_port, 'http://' ~ ipv4 ~ ':' ~ haproxy_port)) -%} +{% set next_port = next_port + 1 -%} +{% endfor -%} + +[haproxy] +recipe = slapos.cookbook:haproxy +conf-path = ${directory:etc}/haproxy.cfg +ip = {{ ipv4 }} +maxconn = {{ slapparameter_dict['haproxy-maxconn'] }} +server-check-path = {{ slapparameter_dict['haproxy-server-check-path'] }} +wrapper-path = ${directory:services}/haproxy +ctl-path = ${directory:bin}/haproxy-ctl +socket-path = ${directory:run}/haproxy.sock +binary-path = {{ parameter_dict['haproxy'] }}/sbin/haproxy +backend-dict = {{ dumps(haproxy_dict) }} + +[apache] +recipe = slapos.cookbook:apache.zope.backend +backend-list = {{ dumps(apache_dict.values()) }} +ip = {% if use_ipv6 %}{{ ipv6 }}{% else %}{{ ipv4 }}{% endif %} +wrapper = ${directory:bin}/apache +scheme = https +key-file = ${directory:apache-conf}/apache.key +cert-file = ${directory:apache-conf}/apache.crt +configuration-file = ${directory:apache-conf}/apache.conf +access-control-string = {{ slapparameter_dict['apache-access-control-string'] }} +pid-file = ${directory:run}/apache.pid +lock-file = ${directory:run}/apache.lock +ssl-session-cache = $${directory:log}/apache-ssl-session-cache +error-log = ${directory:log}/apache-error.log +access-log = ${directory:log}/apache-access.log +apache-binary = {{ parameter_dict['apache'] }}/bin/httpd +ssl-authentication = {{ slapparameter_dict['apache-ssl-authentication'] }} +backend-path = {{ slapparameter_dict['apache-backend-path'] }} +# Note: Without certificate-authority main certificate have to be hardcoded +ssl-authentication-certificate = ${certificate-authority:ca-dir}/cacert.pem +ssl-authentication-crl = ${certificate-authority:ca-crl} + +[publish] +recipe = slapos.cookbook:publish.serialised +{% for family_name, (apache_port, _) in apache_dict.items() -%} +{% if use_ipv6 -%} +{{ family_name }} = ${apache:scheme}://[${apache:ip}]:{{ apache_port }} +{% else -%} +{{ family_name }} = ${apache:scheme}://${apache:ip}:{{ apache_port }} +{% endif -%} +{% endfor -%} + +[certificate-authority] +recipe = slapos.cookbook:certificate_authority +openssl-binary = {{ parameter_dict['openssl'] }}/bin/openssl +ca-dir = ${directory:ca-dir} +requests-directory = ${directory:requests} +wrapper = ${directory:services}/ca +ca-private = ${directory:private} +ca-certs = ${directory:certs} +ca-newcerts = ${directory:newcerts} +ca-crl = ${directory:crl} +{% set ca = slapparameter_dict['ca'] -%} +country-code = {{ ca.get('country-code', 'ZZ') }} +email = {{ ca.get('email', 'nobody@example.com') }} +state = {{ ca.get('state', 'Dummy State') }} +city = {{ ca.get('city', 'Dummy City') }} +company = {{ ca.get('company', 'Dummy Company') }} + +[ca-apache] +< = certificate-authority +recipe = slapos.cookbook:certificate_authority.request +key-file = ${apache:key-file} +cert-file = ${apache:cert-file} +executable = ${apache:wrapper} +wrapper = ${directory:services}/apache + +[logrotate-apache] +recipe = slapos.cookbook:logrotate.d +logrotate-entries = ${logrotate:logrotate-entries} +backup = ${logrotate:backup} +name = apache +log = ${apache:error-log} ${apache:access-log} +post = {{ parameter_dict['bin-directory'] }}/killpidfromfile ${apache:pid-file} SIGUSR1 + +[directory] +recipe = slapos.cookbook:mkdirectory +apache-conf = ${:etc}/apache +bin = ${buildout:directory}/bin +etc = ${buildout:directory}/etc +services = ${:etc}/run +var = ${buildout:directory}/var +run = ${:var}/run +log = ${:var}/log +ca-dir = ${buildout:directory}/srv/ssl +requests = ${:ca-dir}/requests +private = ${:ca-dir}/private +certs = ${:ca-dir}/certs +newcerts = ${:ca-dir}/newcerts +crl = ${:ca-dir}/crl + +[buildout] +extends = {{ parameter_dict['instance-logrotate-cfg'] }} +parts += + publish + logrotate-apache + haproxy + ca-apache + {{ part_list | join('\n ') }} +eggs-directory = {{ eggs_directory }} +develop-eggs-directory = {{ develop_eggs_directory }} +offline = true + +{% endif %} diff --git a/stack/erp5/instance-cloudoo.cfg.in b/stack/erp5/instance-cloudoo.cfg.in index 692c2e06f..952f2f49f 100644 --- a/stack/erp5/instance-cloudoo.cfg.in +++ b/stack/erp5/instance-cloudoo.cfg.in @@ -1,27 +1,33 @@ {% if software_type == slap_software_type -%} -{% set json = json_module.loads(parameter_dict.get('cloudooo-json', '{}')) -%} {% set bin_directory = parameter_dict['buildout-bin-directory'] -%} +{% set use_ipv6 = slapparameter_dict.get('use-ipv6', False) -%} [buildout] parts = publish-cloudooo-connection-information cloudooo-instance promise promise-openoffice + {% if use_ipv6 %}promise-tunnel{% endif %} eggs-directory = {{ eggs_directory }} develop-eggs-directory = {{ develop_eggs_directory }} offline = true [publish-cloudooo-connection-information] -recipe = slapos.cookbook:publishurl +recipe = slapos.cookbook:publish.serialised +{% if use_ipv6 -%} +url = cloudooo://[${ipv6toipv4:ipv6}]:${ipv6toipv4:ipv6-port}/ +{% else -%} url = cloudooo://${cloudooo-instance:ip}:${cloudooo-instance:port}/ +{% endif -%} [cloudooo-instance] recipe = slapos.cookbook:generic.cloudooo # Network options ip = ${slap-network-information:local-ipv4} -port = 23000 -openoffice-port = 23060 +{% set tcpv4_port = slapparameter_dict['tcpv4-port'] -%} +port = {{ tcpv4_port }} +openoffice-port = {{ tcpv4_port + 1 }} # Paths configuration-file = ${directory:etc}/cloudooo.cfg @@ -45,7 +51,7 @@ recipe = slapos.cookbook:fontconfig conf-path = ${directory:etc}/font.conf font-system-folder = {{ parameter_dict['fonts'] }} font-folder = ${directory:font} -url-list = {{ json.get('font_url_list', []) | join(' ') }} +url-list = {{ slapparameter_dict.get('font-url-list', []) | join(' ') }} service-folder = ${directory:service} onetimedownload_path = {{ bin_directory }}/onetimedownload @@ -61,6 +67,25 @@ path = ${directory:promise}/openoffice hostname = ${cloudooo-instance:ip} port = ${cloudooo-instance:openoffice-port} +{% if use_ipv6 -%} +[promise-tunnel] +recipe = slapos.cookbook:check_port_listening +path = ${directory:promise}/tunnel +hostname = ${ipv6toipv4:ipv6} +port = ${ipv6toipv4:ipv6-port} + +[ipv6toipv4] +recipe = slapos.cookbook:ipv6toipv4 +runner-path = ${directory:service}/${:base-name} +6tunnel-path = {{ parameter_dict['6tunnel'] }}/bin/6tunnel +shell-path = {{ parameter_dict['dash'] }}/bin/dash +ipv4 = ${cloudooo-instance:ip} +ipv6 = {{ (ipv6_set | list)[0] }} +ipv6-port = ${cloudooo-instance:port} +ipv4-port = ${cloudooo-instance:port} +base-name = cloudooo-tunnel + +{% endif -%} # rest of parts are candidates for some generic stuff [directory] recipe = slapos.cookbook:mkdirectory diff --git a/stack/erp5/instance-cluster-zope.cfg.in b/stack/erp5/instance-cluster-zope.cfg.in new file mode 100644 index 000000000..0ec3cb40a --- /dev/null +++ b/stack/erp5/instance-cluster-zope.cfg.in @@ -0,0 +1,294 @@ +{% if slap_software_type == software_type -%} +{% set use_ipv6 = parameter_dict.get('use-ipv6', False) -%} +{% set next_port = slapparameter_dict['port-base'] -%} +{% set site_id = slapparameter_dict['site-id'] -%} +{% set storage_type = slapparameter_dict['zodb-storage-type'] -%} +{% set part_list = [] -%} +{% set publish_list = [] -%} +{% set zodb_dict = slapparameter_dict['zodb-dict'] -%} +{% set longrequest_logger_base_path = buildout_directory ~ '/var/log/longrequest_logger_' -%} +{% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%} +{% set bin_directory = parameter_dict['buildout-bin-directory'] -%} +{# +XXX: This template only supports exactly one IPv4 and one IPv6 per +partition. No more (undefined result), no less (IndexError). +-#} +{% set ipv4 = (ipv4_set | list)[0] -%} + +[directory] +recipe = slapos.cookbook:mkdirectory +bin = ${buildout:directory}/bin +etc = ${buildout:directory}/etc +instance = ${:srv}/erp5shared +instance-constraint = ${:instance}/Constraint +instance-document = ${:instance}/Document +instance-etc = ${:instance}/etc +instance-etc-package-include = ${:instance}/etc/package-include +instance-extensions = ${:instance}/Extensions +instance-import = ${:instance}/import +instance-lib = ${:instance}/lib +instance-products = ${:instance}/Products +instance-propertysheet = ${:instance}/PropertySheet +instance-tests = ${:instance}/tests +log = ${:var}/log +run = ${:var}/run +services = ${:etc}/run +srv = ${buildout:directory}/srv +tmp = ${buildout:directory}/tmp +var = ${buildout:directory}/var +promises = ${:etc}/promise + +[binary-link] +recipe = slapos.cookbook:symbolic.link +target-directory = ${directory:bin} +link-binary = + {{ parameter_dict['coreutils'] }}/bin/basename + {{ parameter_dict['coreutils'] }}/bin/cat + {{ parameter_dict['coreutils'] }}/bin/cp + {{ parameter_dict['coreutils'] }}/bin/ls + {{ parameter_dict['coreutils'] }}/bin/tr + {{ parameter_dict['coreutils'] }}/bin/uname + {{ parameter_dict['git'] }}/bin/git + {{ parameter_dict['graphviz'] }}/bin/dot + {{ parameter_dict['grep'] }}/bin/grep + {{ parameter_dict['imagemagick'] }}/bin/convert + {{ parameter_dict['imagemagick'] }}/bin/identify + {{ parameter_dict['sed'] }}/bin/sed + {{ parameter_dict['tesseract'] }}/bin/tesseract + {{ parameter_dict['w3m'] }}/bin/w3m + {{ parameter_dict['openssl'] }}/bin/openssl + {{ parameter_dict['poppler'] }}/bin/pdfinfo + {{ parameter_dict['poppler'] }}/bin/pdftotext + {{ parameter_dict['poppler'] }}/bin/pdftohtml + {{ parameter_dict['dmtx-utils'] }}/bin/dmtxwrite + +[certificate-authority-common] +requests-directory = ${directory:requests} +ca-dir = ${directory:ca-dir} +ca-private = ${directory:private} +ca-certs = ${directory:certs} +ca-newcerts = ${directory:newcerts} +ca-crl = ${directory:crl} + +[certificate-authority] +< = certificate-authority-common +recipe = slapos.cookbook:certificate_authority +openssl-binary = {{ parameter_dict['openssl'] }}/bin/openssl +wrapper = ${directory:services}/ca + +{% if use_ipv6 -%} +{% set ipv6 = (ipv6_set | list)[0] -%} +[zeo-tunnel-base] +recipe = slapos.cookbook:ipv4toipv6 +runner-path = ${directory:services}/${:base-name} +6tunnel-path = {{ parameter_dict['6tunnel'] }}/bin/6tunnel +shell-path = {{ parameter_dict['dash'] }}/bin/dash +ipv4 = {{ ipv4 }} + +{% if storage_type == 'zeoclient' -%} +{# ZEO needs tunelling for IPv6 (...until next version becomes current in Zope) -#} +{% set zeo_tunneling_dict = {} -%} +{% for _, _, storage_dict in zodb_dict.values() -%} +{% set storage_server = storage_dict['server'] -%} +{% if storage_server not in zeo_tunneling_dict -%} +{% set current_port = next_port + (zeo_tunneling_dict | length) -%} +{% do zeo_tunneling_dict.__setitem__(storage_server, current_port) -%} +[{{ section('zeo-tunnel-' ~ current_port) }}] +< = zeo-tunnel-base +base-name = {{ 'zeo-tunnel-' ~ current_port }} +ipv4-port = {{ current_port }} +ipv6-port = {{ storage_server.split(']:')[1] }} +ipv6 = {{ storage_server.split(']:')[0][1:] }} +{% endif -%} +{% do storage_dict.__setitem__('server', '' ~ ipv4 ~ ':' ~ zeo_tunneling_dict[storage_server]) -%} +{% endfor -%} +{% set next_port = next_port + (zeo_tunneling_dict | length) -%} +{% endif -%} +{% if slapparameter_dict.get('tidstorage-ip') -%} +[tidstorage-tunnel] +< = zeo-tunnel-base +base-name = {{ 'tidstorage-tunnel' }} +ipv4-port = {{ next_port }} +ipv6 = {{ slapparameter_dict.get('tidstorage-ip') }} +ipv6-port = {{ slapparameter_dict.get('tidstorage-port') }} +{% do slapparameter_dict.__setitem__('tidstorage-ip', ipv4) -%} +{% do slapparameter_dict.__setitem__('tidstorage-port', next_port) -%} +{% set next_port = next_port + 1 -%} + +[{{ section("promise-tidstorage-tunnel") }}] +recipe = slapos.cookbook:check_port_listening +hostname = ${tidstorage-tunnel:ipv4} +port = ${tidstorage-tunnel:ipv4-port} +path = ${directory:promises}/tidstorage +{% endif -%} + +[ipv6toipv4-base] +recipe = slapos.cookbook:ipv6toipv4 +runner-path = ${directory:services}/${:base-name} +6tunnel-path = {{ parameter_dict['6tunnel'] }}/bin/6tunnel +shell-path = {{ parameter_dict['dash'] }}/bin/dash +ipv4 = {{ ipv4 }} +ipv6 = {{ ipv6 }} +{% endif -%} + +{% if slapparameter_dict.get('tidstorage-ip') -%} +[tidstorage] +ipv4 = {{ slapparameter_dict['tidstorage-ip'] }} +ipv4-port = {{ slapparameter_dict['tidstorage-port'] }} +{% else -%} +[tidstorage] +ipv4 = +ipv4-port = +{% endif -%} + +[zope-base] +recipe = slapos.cookbook:generic.zope.zeo.client +user = zope +ip = {{ ipv4 }} +timezone = {{ slapparameter_dict['timezone'] }} +tidstorage-ip = ${tidstorage:ipv4} +tidstorage-port = ${tidstorage:ipv4-port} +instance-etc = ${directory:instance-etc} +bt5-repository = ${directory:var}/bt5_repository +tmp-path = ${directory:tmp} +bin-path = ${directory:bin} +site-zcml = ${:instance-etc}/site.zcml +inituser = ${directory:instance}/inituser +runzope-binary = {{ bin_directory }}/runzope +bt5-repository-list = + +[deadlock-debugger-password] +recipe = slapos.cookbook:pwgen.stable + +[zope-conf-parameter-base] +ip = {{ ipv4 }} +site-id = {{ site_id }} +{% set zodb_list = [] -%} +{% for key, (mount_point, cache_size, storage_dict) in zodb_dict.items() -%} +{% do zodb_list.append([key, mount_point, cache_size, storage_type, storage_dict]) -%} +{% endfor -%} +zodb-list = {{ dumps(zodb_list) }} + +[zope-conf-base] +recipe = slapos.recipe.template:jinja2 +template = {{ parameter_dict['zope-conf-template'] }} +extra-context = +context = + key instance directory:instance + key instance_products directory:instance-products + raw deadlock_path /manage_debug_threads + key deadlock_debugger_password deadlock-debugger-password:password + key tidstorage_ip tidstorage:ipv4 + key tidstorage_port tidstorage:ipv4-port + key promise_path erp5-promise:promise-path + ${:extra-context} + +[logrotate-entry-base] +recipe = slapos.cookbook:logrotate.d +logrotate-entries = ${logrotate:logrotate-entries} +backup = ${logrotate:backup} + +{% macro zope( + name, + port, + thread_amount, + timerserver_interval, + longrequest_logger_timeout, + longrequest_logger_interval +) -%} +{% set conf_name = name ~ '-conf' -%} +{% set conf_parameter_name = conf_name ~ '-param' -%} +{% set zope_tunnel_section_name = name ~ '-ipv6toipv4' -%} +{% set zope_tunnel_base_name = zope_tunnel_section_name -%} +[{{ conf_parameter_name }}] +< = zope-conf-parameter-base +pid-file = ${directory:run}/{{ name }}.pid +lock-file = ${directory:run}/{{ name }}.lock +port = {{ port }} +thread-amount = {{ thread_amount }} +timerserver-interval = {{ dumps(timerserver_interval) }} +event-log = ${directory:log}/{{ name }}-event.log +z2-log = ${directory:log}/{{ name }}-Z2.log + +[{{ conf_name }}] +< = zope-conf-base +rendered = ${directory:etc}/{{ name }}.conf +extra-context = + section parameter_dict {{ conf_parameter_name }} + +[{{ name }}] +< = zope-base +{% if longrequest_logger_interval < 0 -%} +longrequest-logger-file = +longrequest-logger-timeout = +longrequest-logger-interval = +{% else -%} +longrequest-logger-file = {{ longrequest_logger_base_path ~ name ~ ".log" }} +longrequest-logger-timeout = {{ dumps(longrequest_logger_timeout) }} +longrequest-logger-interval = {{ dumps(longrequest_logger_interval) }} +{% endif -%} +wrapper = ${directory:services}/{{ name }} +configuration-file = {{ '${' ~ conf_name ~ ':rendered}' }} +port = {{ '${' ~ conf_parameter_name ~ ':port}' }} + +[{{ section("promise-" ~ name) }}] +recipe = slapos.cookbook:check_port_listening +hostname = {{ '${' ~ name ~ ':ip}' }} +port = {{ '${' ~ name ~ ':port}' }} +path = ${directory:promises}/{{ name }} + +{% if use_ipv6 -%} +[{{ zope_tunnel_section_name }}] +< = ipv6toipv4-base +base-name = {{ zope_tunnel_base_name }} +ipv6-port = {{ port }} +ipv4-port = {{ port }} +{% do publish_list.append("[" ~ ipv6 ~ "]:" ~ port) -%} + +[{{ section("promise-tunnel-" ~ name) }}] +recipe = slapos.cookbook:check_port_listening +hostname = {{ '${' ~ zope_tunnel_section_name ~ ':ipv6}' }} +port = {{ '${' ~ zope_tunnel_section_name ~ ':ipv6-port}' }} +path = ${directory:promises}/{{ zope_tunnel_base_name }} +{% else -%} +{% do publish_list.append(ipv4 ~ ":" ~ port) -%} +{% endif -%} + +[{{ section('logrotate-entry-' ~ name) }}] +< = logrotate-entry-base +name = {{ name }} +log = {{ '${' ~ conf_parameter_name ~ ':event-log}' }} {{ '${' ~ conf_parameter_name ~ ':z2-log}' }} {{ '${' ~ name ~ ':longrequest-logger-file}' }} +post = {{ bin_directory }}/killpidfromfile {{ '${' ~ conf_parameter_name ~ ':pid-file}' }} SIGUSR2 +{% endmacro -%} + +{% for i in range(slapparameter_dict.get('instance-count', 1)) -%} +{{ zope("zope-" ~ i, next_port, slapparameter_dict['thread-amount'], slapparameter_dict['timerserver-interval'], slapparameter_dict['longrequest-logger-timeout'], slapparameter_dict['longrequest-logger-interval']) }} +{% set next_port = next_port + 1 -%} +{% endfor -%} + +[publish-zope] +recipe = slapos.cookbook:publish.serialised +zope-address-list = {{ dumps(publish_list) }} + +[erp5-promise] +recipe = slapos.cookbook:erp5.promise +promise-path = ${directory:etc}/erp5promise.cfg +kumofs-url = {{ slapparameter_dict['kumofs-url'] }} +memcached-url = {{ slapparameter_dict['memcached-url'] }} +cloudooo-url = {{ slapparameter_dict['cloudooo-url'] }} +smtp-url = {{ slapparameter_dict['smtp-url'] }} +bt5 = {{ slapparameter_dict['bt5'] }} +bt5-repository-url = {{ slapparameter_dict['bt5-repository-url'] }} + +[buildout] +eggs-directory = {{ eggs_directory }} +develop-eggs-directory = {{ develop_eggs_directory }} +offline = true +extends = + {{ parameter_dict['instance-logrotate-cfg'] }} +parts += + binary-link + erp5-promise + {{ part_list | join('\n ') }} + publish-zope +{% endif %} diff --git a/stack/erp5/instance-erp5-cluster.cfg.in b/stack/erp5/instance-erp5-cluster.cfg.in index 531852e49..9d2a2a289 100644 --- a/stack/erp5/instance-erp5-cluster.cfg.in +++ b/stack/erp5/instance-erp5-cluster.cfg.in @@ -1,62 +1,54 @@ {% if slap_software_type == software_type -%} -############################# -# -# Request erp5 production environnment -# -############################# -[buildout] - -parts = - request-tidstorage - basedirectory - -eggs-directory = {{ eggs_directory }} -develop-eggs-directory = {{ develop_eggs_directory }} -offline = true - +{% set publish_dict = {} -%} +{% set frontend_dict = slapparameter_dict.get('frontend', {}) %} +{% set has_frontend = frontend_dict.get('software-url', '') != '' -%} +{% set site_id = slapparameter_dict.get('site-id', 'erp5') -%} [request-common] -recipe = slapos.cookbook:request +recipe = slapos.cookbook:request.serialised software-url = ${slap-connection:software-release-url} sla = computer_guid -return = url server-url = ${slap-connection:server-url} key-file = ${slap-connection:key-file} cert-file = ${slap-connection:cert-file} computer-id = ${slap-connection:computer-id} partition-id = ${slap-connection:partition-id} +config = + use-ipv6 + ${:extra-config} +extra-config = +config-use-ipv6 = {{ dumps(slapparameter_dict.get('use-ipv6', False)) }} -[request-mariadb] -<=request-common -name = MariaDB DataBase -software-type = mariadb -sla-computer_guid = ${slap-parameter:mariadb-computer-guid} - -[request-cloudooo] -<=request-common -name = Cloudooo -config = cloudooo-json -config-cloudooo-json = ${slap-parameter:cloudooo-json} -software-type = cloudooo -sla-computer_guid = ${slap-parameter:cloudooo-computer-guid} - -[request-memcached] -<=request-common -name = Memcached -software-type = memcached -sla-computer_guid = ${slap-parameter:memcached-computer-guid} +{% macro request(name, software_type, config_key, config={}, ret={'url': True}) -%} +{% do config.update(slapparameter_dict.get(config_key, {})) -%} +{% set section = 'request-' ~ name -%} +[{{ section }}] +< = request-common +name = {{ name }} +software-type = {{ software_type }} +return = {{ ret.keys() | join(' ') }} +{% for ret, publish in ret.items() -%} +{% if publish -%} +{% do publish_dict.__setitem__(name ~ '-' ~ ret, '${' ~ section ~ ':connection-' ~ ret ~ '}')%} +{% endif -%} +{% endfor -%} +sla-computer_guid = {{ slapparameter_dict.get(config_key + '-computer-guid', computer_id) }} +extra-config = {{ ' '.join(config) }} +{% for option, value in config.items() -%} +config-{{ option }} = {{ dumps(value) }} +{% endfor -%} +{% endmacro -%} -[request-kumofs] -<=request-common -name = KumoFS -software-type = kumofs -sla-computer_guid = ${slap-parameter:kumofs-computer-guid} +{{ request('memcached-persistent', 'kumofs', 'kumofs', {'tcpv4-port': 2000}) }} +{{ request('memcached-volatile', 'kumofs', 'memcached', {'tcpv4-port': 2010, 'ram-storage-size': 64}) }} +{{ request('cloudooo', 'cloudooo', 'cloudooo', {'tcpv4-port': 2020}) }} +{{ request('mariadb', 'mariadb', 'mariadb', {'tcpv4-port': 2099}) }} +{{ request('zodb', 'zodb-' ~ slapparameter_dict.get('zodb-software-type', 'zeo'), 'zodb', {'tcpv4-port': 2100, 'zodb-dict': {'root': {}}}, {'zodb-storage-type': False, 'zodb-dict': False, 'tidstorage-ip': False, 'tidstorage-port': False}) }} -[request-tidstorage] -<=request-common -name = TidStorage -return = url-login -config = - json +[request-zope-base] +< = request-common +return = + zope-address-list +extra-config = mysql-url memcached-url cloudooo-url @@ -64,45 +56,132 @@ config = smtp-url bt5 bt5-repository-url -config-json = ${slap-parameter:json} + zodb-dict + zodb-storage-type + tidstorage-ip + tidstorage-port + instance-count + thread-amount + timerserver-interval + timezone + site-id + longrequest-logger-interval + longrequest-logger-timeout + port-base config-mysql-url = ${request-mariadb:connection-url} -config-memcached-url = ${request-memcached:connection-url} +config-memcached-url = ${request-memcached-volatile:connection-url} config-cloudooo-url = ${request-cloudooo:connection-url} -config-kumofs-url = ${request-kumofs:connection-url} -config-bt5 = ${slap-parameter:bt5} -config-bt5-repository-url = ${slap-parameter:bt5-repository-url} -config-smtp-url = ${slap-parameter:smtp-url} -software-type = tidstorage -sla-computer_guid = ${slap-parameter:tidstorage-computer-guid} +config-kumofs-url = ${request-memcached-persistent:connection-url} +config-bt5 = {{ slapparameter_dict.get('bt5', 'erp5_full_text_myisam_catalog erp5_configurator_standard erp5_configurator_maxma_demo erp5_configurator_ung erp5_configurator_run_my_doc') }} +config-bt5-repository-url = {{ slapparameter_dict.get('bt5-repository-url', local_bt5_repository) }} +config-smtp-url = {{ slapparameter_dict.get('smtp-url', 'smtp://localhost:25/') }} +config-zodb-dict = ${request-zodb:connection-zodb-dict} +config-zodb-storage-type = ${request-zodb:connection-zodb-storage-type} +config-tidstorage-ip = ${request-zodb:connection-tidstorage-ip} +config-tidstorage-port = ${request-zodb:connection-tidstorage-port} +config-timezone = {{ slapparameter_dict.get('timezone', 'UTC') }} +config-site-id = {{ site_id }} +software-type = cluster-zope + +{% set zope_family_dict = {} -%} +{% for custom_name, zope_parameter_dict in slapparameter_dict.get('zope-partition-dict', {'1': {}}).items() -%} +{% set base_name = 'zope-' ~ custom_name -%} +{% set name = 'request-' ~ base_name -%} +{% do zope_family_dict.setdefault(zope_parameter_dict.get('family', 'default'), []).append(name) -%} +[{{ name }}] +< = request-zope-base +name = {{ base_name }} +config-instance-count = {{ dumps(zope_parameter_dict.get('instance-count', 1)) }} +config-thread-amount = {{ dumps(zope_parameter_dict.get('thread-amount', 1)) }} +config-timerserver-interval = {{ dumps(zope_parameter_dict.get('timerserver-interval', 5)) }} +config-longrequest-logger-interval = {{ dumps(zope_parameter_dict.get('longrequest-logger-interval', -1)) }} +config-longrequest-logger-timeout = {{ dumps(zope_parameter_dict.get('longrequest-logger-timeout', 1)) }} +config-port-base = {{ dumps(zope_parameter_dict.get('port-base', 2200)) }} +sla-computer_guid = {{ zope_parameter_dict.get('computer-guid', computer_id) }} +{% endfor -%} + +{# We need to concatenate lists that we cannot read as lists, so this gets hairy. -#} +{% set zope_address_list_id_dict = {} -%} +{% set zope_family_parameter_dict = {} -%} +{% for family_name, zope_section_id_list in zope_family_dict.items() -%} +{% for zope_section_id in zope_section_id_list -%} +{% set parameter_name = 'zope-family-entry-' ~ zope_section_id -%} +{% do zope_address_list_id_dict.__setitem__(zope_section_id, parameter_name) -%} +{% do zope_family_parameter_dict.setdefault(family_name, []).append(parameter_name) -%} +{% endfor -%} +{% if has_frontend -%} +{% set frontend_name = 'frontend-' ~ family_name -%} +{% set publishable = frontend_name ~ ':connection-site_url' -%} +[{{ frontend_name }}] +< = request-frontend-base +name = {{ frontend_name }} +config-url = {{ backend_url }} +{% else -%} +{% set publishable = 'request-balancer:connection-' ~ family_name -%} +{% endif -%} +{% do publish_dict.__setitem__('family-' ~ family_name, '${' ~ publishable ~ '}' ) -%} +{% endfor -%} + +[request-balancer] +< = request-common +name = balancer +software-type = balancer +sla-computer_guid = {{ slapparameter_dict.get('balancer-computer-guid', computer_id) }} +extra-config = + tcpv4-port + haproxy-maxconn + haproxy-server-check-path + apache-access-control-string + apache-ssl-authentication + apache-backend-path + ca + zope-family-dict + {{ zope_address_list_id_dict.values() | join(' ') }} +return = + {{ zope_family_dict.keys() | join(' ') }} +config-zope-family-dict = {{ dumps(zope_family_parameter_dict) }} +config-tcpv4-port = {{ dumps(slapparameter_dict.get('', 2150)) }} +{% for zope_section_id, name in zope_address_list_id_dict.items() -%} +config-{{ name }} = {{ ' ${' ~ zope_section_id ~ ':connection-zope-address-list}' }} +{% endfor -%} +# XXX: should those really be same for all families ? +config-haproxy-maxconn = {{ slapparameter_dict.get('haproxy-maxconn', 1) }} +config-haproxy-server-check-path = {{ slapparameter_dict.get('haproxy-server-check-path', '/') % {'site-id': site_id} }} +config-apache-access-control-string = {{ slapparameter_dict.get('apache-access-control-string', 'all') }} +config-apache-ssl-authentication = {{ slapparameter_dict.get('apache-ssl-authentication', '0') }} +config-apache-backend-path = {{ slapparameter_dict.get('apache-backend-path', '/') % {'site-id': site_id} }} +config-ca = {{ dumps(slapparameter_dict.get('ca', {})) }} -[request-varnish] -<=request-common -name = Varnish -config = tidstorage-url -config-tidstorage-url = ${request-tidstorage:connection-url-login} -config-web-checker-mail-address = ${slap-parameter:web-checker-mail-address} -config-web-checker-smtp-host = ${slap-parameter:web-checker-smtp-host} -software-type = varnish -sla-computer_guid = ${slap-parameter:varnish-computer-guid} +[request-frontend-base] +{% if has_frontend -%} +< = request-common +software-url = {{ frontend_dict['software-url'] }} +software-type = {{ frontend_dict.get('software-type', 'RootSoftwareInstance') }} +sla = instance_guid +sla-instance_guid = {{ frontend_dict['instance-guid'] }} +slave = true +{% set config_dict = { + 'type': 'zope', +} -%} +{% if frontend_dict.get('domain') -%} +{% do config_dict.__setitem__('custom_domain', frontend_dict['domain']) -%} +{% endif -%} +extra-config = url {{ config_dict.keys() | join(' ') }} +{% for name, value in config_dict.items() -%} +config-{{ name }} = {{ value }} +{% endfor -%} +return = site_url +{% endif -%} -[slap-parameter] -# Default value if no computer_guid is specified for each type -mariadb-computer-guid = ${slap-connection:computer-id} -cloudooo-computer-guid = ${slap-connection:computer-id} -memcached-computer-guid = ${slap-connection:computer-id} -kumofs-computer-guid = ${slap-connection:computer-id} -tidstorage-computer-guid = ${slap-connection:computer-id} -varnish-computer-guid = ${slap-connection:computer-id} -cloudooo-json = -bt5 = erp5_full_text_myisam_catalog - erp5_configurator_standard - erp5_configurator_maxma_demo - erp5_configurator_ung - erp5_configurator_run_my_doc -bt5-repository-url = {{ local_bt5_repository }} -smtp-url = smtp://localhost:25/ +[publish] +recipe = slapos.cookbook:publish.serialised +{% for name, value in publish_dict.items() -%} +{{ name }} = {{ value }} +{% endfor -%} -[basedirectory] -recipe = slapos.cookbook:mkdirectory -etc = ${buildout:directory}/etc/run +[buildout] +parts = publish +eggs-directory = {{ eggs_directory }} +develop-eggs-directory = {{ develop_eggs_directory }} +offline = true {% endif %} diff --git a/stack/erp5/instance-erp5-single.cfg.in b/stack/erp5/instance-erp5-single.cfg.in index 15aa6ebd9..3902b8758 100644 --- a/stack/erp5/instance-erp5-single.cfg.in +++ b/stack/erp5/instance-erp5-single.cfg.in @@ -87,7 +87,7 @@ bt5 = ${slap-parameter:bt5} bt5-repository-url = ${slap-parameter:bt5-repository-url} [request-common] -recipe = slapos.cookbook:request +recipe = slapos.cookbook:request.serialised software-url = ${slap-connection:software-release-url} sla = computer_guid sla-computer_guid = ${slap-connection:computer-id} @@ -102,6 +102,11 @@ partition-id = ${slap-connection:partition-id} <=request-common name = MariaDB DataBase software-type = mariadb +{% set mariadb_dict = slapparameter_dict.get('mariadb-dict', {}) -%} +{% for option, value in mariadb_dict.items() -%} +config-{{ option }} = {{ dumps(value) }} +{% endfor -%} +config = {{ ' '.join(mariadb_dict) }} [request-cloudooo] <=request-common @@ -112,8 +117,10 @@ software-type = cloudooo [request-memcached] <=request-common -name = Memcached -software-type = memcached +name = KumoFS-ram +software-type = kumofs +config = ram-storage-size +config-ram-storage-size = 64m [request-kumofs] <=request-common diff --git a/stack/erp5/instance-kumofs.cfg.in b/stack/erp5/instance-kumofs.cfg.in index 5609b0248..2fe5460e9 100644 --- a/stack/erp5/instance-kumofs.cfg.in +++ b/stack/erp5/instance-kumofs.cfg.in @@ -1,3 +1,5 @@ +{% if software_type == slap_software_type -%} +{% set use_ipv6 = slapparameter_dict.get('use-ipv6', False) -%} [buildout] parts = publish-kumofs-connection-information @@ -16,53 +18,67 @@ develop-eggs-directory = {{ develop_eggs_directory }} offline = true [publish-kumofs-connection-information] -recipe = slapos.cookbook:publishurl +recipe = slapos.cookbook:publish.serialised +{% if use_ipv6 -%} +url = memcached://[${kumofs-instance:ip}]:${kumofs-instance:gateway-port}/ +{% else -%} url = memcached://${kumofs-instance:ip}:${kumofs-instance:gateway-port}/ +{% endif -%} [kumofs-instance] recipe = slapos.cookbook:generic.kumofs # Network options +{% if use_ipv6 -%} +ip = ${slap-network-information:global-ipv6} +address-family = inet6 +{% else -%} ip = ${slap-network-information:local-ipv4} -manager-port = 13101 -server-port = 13201 -server-listen-port = 13202 -gateway-port = 13301 +address-family = inet4 +{% endif -%} +{% set tcpv4_port = slapparameter_dict['tcpv4-port'] -%} +manager-port = {{ tcpv4_port }} +server-port = {{ tcpv4_port + 1 }} +server-listen-port = {{ tcpv4_port + 2 }} +gateway-port = {{ tcpv4_port + 3 }} +# Paths: Data +{% set ram_storage_size = slapparameter_dict.get('ram-storage-size') -%} +{% if ram_storage_size -%} +data-path = *#capsiz={{ ram_storage_size }}m +{% else -%} +data-directory = ${directory:kumofs-data} +{% endif -%} # Paths: Running wrappers gateway-wrapper = ${basedirectory:services}/kumofs_gateway manager-wrapper = ${basedirectory:services}/kumofs_manager server-wrapper = ${basedirectory:services}/kumofs_server -# Paths: Data -data-directory = ${directory:kumofs-data} - # Paths: Logs kumo-gateway-log = ${basedirectory:log}/kumo-gateway.log kumo-manager-log = ${basedirectory:log}/kumo-manager.log kumo-server-log = ${basedirectory:log}/kumo-server.log # Binary information -kumo-gateway-binary = {{ kumo_location }}/bin/kumo-gateway -kumo-manager-binary = {{ kumo_location }}/bin/kumo-manager -kumo-server-binary = {{ kumo_location }}/bin/kumo-server -shell-path = {{ dash_location }}/bin/dash +kumo-gateway-binary = {{ parameter_dict['kumo-location'] }}/bin/kumo-gateway +kumo-manager-binary = {{ parameter_dict['kumo-location'] }}/bin/kumo-manager +kumo-server-binary = {{ parameter_dict['kumo-location'] }}/bin/kumo-server +shell-path = {{ parameter_dict['dash-location'] }}/bin/dash [logrotate-entry-kumofs] <= logrotate recipe = slapos.cookbook:logrotate.d name = kumofs -log = ${kumofs-instance:kumo-gateway-log} ${kumofs-instance:kumo-manager-log} - ${kumofs-instance:kumo-server-log} +log = ${kumofs-instance:kumo-gateway-log} ${kumofs-instance:kumo-manager-log} ${kumofs-instance:kumo-server-log} # rest of parts are candidates for some generic stuff [logrotate] recipe = slapos.cookbook:logrotate # Binaries -logrotate-binary = {{ logrotate_location }}/usr/sbin/logrotate -gzip-binary = {{ gzip_location }}/bin/gzip -gunzip-binary = {{ gzip_location }}/bin/gunzip +logrotate-binary = {{ parameter_dict['logrotate-location'] }}/usr/sbin/logrotate +gzip-binary = {{ parameter_dict['gzip-location'] }}/bin/gzip +gunzip-binary = {{ parameter_dict['gzip-location'] }}/bin/gunzip # Directories wrapper = ${rootdirectory:bin}/logrotate conf = ${rootdirectory:etc}/logrotate.conf @@ -96,7 +112,7 @@ bin = ${buildout:directory}/bin [cron] recipe = slapos.cookbook:cron -dcrond-binary = {{ dcron_location }}/sbin/crond +dcrond-binary = {{ parameter_dict['dcron-location'] }}/sbin/crond cron-entries = ${directory:cron-entries} crontabs = ${directory:crontabs} cronstamps = ${directory:cronstamps} @@ -140,3 +156,4 @@ port = ${kumofs-instance:gateway-port} <= promise-template path = ${basedirectory:promise}/kumofs-manager port = ${kumofs-instance:manager-port} +{% endif %} diff --git a/stack/erp5/instance-logrotate-base.cfg.in b/stack/erp5/instance-logrotate-base.cfg.in new file mode 100644 index 000000000..cff8e4c20 --- /dev/null +++ b/stack/erp5/instance-logrotate-base.cfg.in @@ -0,0 +1,50 @@ +[buildout] +parts = + cron-entry-logrotate + +[cron] +recipe = slapos.cookbook:cron +cron-entries = ${logrotate-directory:cron-entries} +dcrond-binary = {{ dcron_location }}/sbin/crond +crontabs = ${logrotate-directory:crontabs} +cronstamps = ${logrotate-directory:cronstamps} +catcher = ${cron-simplelogger:wrapper} +binary = ${logrotate-directory:services}/crond + +[cron-simplelogger] +recipe = slapos.cookbook:simplelogger +wrapper = ${logrotate-directory:bin}/cron_simplelogger +log = ${logrotate-directory:log}/cron.log + +[logrotate] +recipe = slapos.cookbook:logrotate +logrotate-entries = ${logrotate-directory:logrotate-entries} +backup = ${logrotate-directory:logrotate-backup} +logrotate-binary = {{ logrotate_location }}/usr/sbin/logrotate +gzip-binary = {{ gzip_location }}/bin/gzip +gunzip-binary = {{ gzip_location }}/bin/gunzip +wrapper = ${logrotate-directory:bin}/logrotate +conf = ${logrotate-directory:etc}/logrotate.conf +state-file = ${logrotate-directory:srv}/logrotate.status + +[cron-entry-logrotate] +recipe = slapos.cookbook:cron.d +cron-entries = ${cron:cron-entries} +name = logrotate +frequency = 0 0 * * * +command = ${logrotate:wrapper} + +[logrotate-directory] +recipe = slapos.cookbook:mkdirectory +cron-entries = ${:etc}/cron.d +cronstamps = ${:etc}/cronstamps +crontabs = ${:etc}/crontabs +logrotate-backup = ${:backup}/logrotate +logrotate-entries = ${:etc}/logrotate.d +bin = ${buildout:directory}/bin +srv = ${buildout:directory}/srv +backup = ${:srv}/backup +etc = ${buildout:directory}/etc +services = ${:etc}/run +log = ${buildout:directory}/var/log + diff --git a/stack/erp5/instance-mariadb.cfg.in b/stack/erp5/instance-mariadb.cfg.in index 6acf65681..4558ef970 100644 --- a/stack/erp5/instance-mariadb.cfg.in +++ b/stack/erp5/instance-mariadb.cfg.in @@ -1,140 +1,150 @@ -[buildout] -parts = - publish-mariadb-url - mariadb-instance - logrotate - logrotate-entry-mariadb - cron - cron-entry-logrotate - cron-entry-mariadb-backup - binary-link - promise - -eggs-directory = {{ eggs_directory }} -develop-eggs-directory = {{ develop_eggs_directory }} -offline = true +{% if software_type == slap_software_type -%} +{% set part_list = [] -%} +{% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%} +{% set use_ipv6 = slapparameter_dict.get('use-ipv6', False) -%} +{% set database_list = slapparameter_dict.get('database-list', [{'name': 'erp5', 'user': 'user', 'password': 'insecure'}]) -%} +{% for database_count in range(slapparameter_dict.get('test-database-amount', 0)) -%} +{% do database_list.append({'name': 'erp5_test_' ~ database_count, 'user': 'testuser_' ~ database_count, 'password': 'testpassword' ~ database_count}) -%} +{% endfor -%} +{% set catalog_backup = slapparameter_dict.get('catalog-backup', {}) -%} +{% set full_backup_retention_days = catalog_backup.get('full-retention-days', 7) -%} +{% set incremental_backup_retention_days = catalog_backup.get('incremental-retention-days', full_backup_retention_days) -%} [publish-mariadb-url] -recipe = slapos.cookbook:publishurl -url = mysql://${mariadb-instance:user}:${mariadb-instance:password}@${mariadb-instance:ip}:${mariadb-instance:port}/${mariadb-instance:database} - -[cron-entry-mariadb-backup] +recipe = slapos.cookbook:publish.serialised +{% if use_ipv6 -%} +{% set address = "[${my-cnf-parameters:ip}]" -%} +{% else -%} +{% set address = "${my-cnf-parameters:ip}" -%} +{% endif -%} +{# TODO: support any number of databases -#} +{% set database = database_list[0] %} +url = mysql://{{ database['user'] }}:{{ database['password'] }}@{{ address }}:${my-cnf-parameters:port}/{{ database['name'] }} + +{% if full_backup_retention_days > -1 -%} +[{{ section('cron-entry-mariadb-backup') }}] <= cron recipe = slapos.cookbook:cron.d name = mariadb-backup -frequency = 0 0 * * * -command = ${mariadb-instance:backup-script} +frequency = 0 22 * * * +{# When binlogs are enabled: +# flush-logs: used so no manipulation on binlogs is needed to restore from +# full + binlogs. The first binlog after a dump starts from dump snapshot and +# can be fully restored. +# master-data: use value "2" as we are not in a replication case +#} +command = "${binary-wrap-mysqldump:wrapper-path}" -u root --all-databases {% if incremental_backup_retention_days > -1 %}--flush-logs --master-data=2 {% endif %}| {{ parameter_dict['gzip-location'] }}/bin/gzip > "${directory:mariadb-backup-full}/$({{ parameter_dict['coreutils-location'] }}/bin/date "+%Y%m%d%H%M%S").sql.gz" +{# KEEP GLOB PATTERN IN SYNC with generated filenames above +# YYYYmmddHHMMSS -#} +file-glob = ??????????????.sql.gz + +{% if full_backup_retention_days > 0 -%} +[{{ section("cron-entry-mariadb-backup-expire") }}] +<= cron +recipe = slapos.cookbook:cron.d +name = mariadb-backup-expire +frequency = 0 22 * * * +command = {{ parameter_dict['findutils-location'] }}/bin/find "${directory:mariadb-backup-full}" -maxdepth 1 -name "${cron-entry-mariadb-backup:file-glob}" -daystart -mtime +{{ full_backup_retention_days }} -delete +{%- endif %} +{%- endif %} + +[my-cnf-parameters] +{% if use_ipv6 -%} +ip = {{ (ipv6_set | list)[0] }} +{% else -%} +ip = {{ (ipv4_set | list)[0] }} +{% endif -%} +port = {{ slapparameter_dict['tcpv4-port'] }} +socket = ${directory:run}/mariadb.sock +data-directory = ${directory:mariadb-data} +pid-file = ${directory:run}/mariadb.pid +error-log = ${directory:log}/mariadb_error.log +slow-query-log = ${directory:log}/mariadb_slowquery.log +innodb-buffer-size = {{ slapparameter_dict.get('innodb-buffer-size', '') }} +innodb-log-file-size = {{ slapparameter_dict.get('innodb-log-file-size', '') }} +innodb-log-buffer-size = {{ slapparameter_dict.get('innodb-log-buffer-size', '') }} +relaxed_writes = {{ dumps(slapparameter_dict.get('mariadb-relaxed-writes', 0)) }} +{% if incremental_backup_retention_days > -1 -%} +binlog-path = ${directory:mariadb-backup-incremental}/binlog +# XXX: binlog rotation happens along with other log's rotation +binlog-expire-days = {{ incremental_backup_retention_days }} +{% endif -%} + +[my-cnf] +recipe = slapos.recipe.template:jinja2 +rendered = ${directory:etc}/mariadb.cnf +template = {{ parameter_dict['template-my-cnf'] }} +context = section parameter_dict my-cnf-parameters + +[init-script-parameters] +database-list = {{ dumps(database_list) }} + +[init-script] +recipe = slapos.recipe.template:jinja2 +# XXX: is there a better location ? +rendered = ${directory:etc}/mariadb_initial_setup.sql +template = {{ parameter_dict['template-mariadb-initial-setup'] }} +context = section parameter_dict init-script-parameters + +[update-mysql] +recipe = slapos.cookbook:generic.mysql.wrap_update_mysql +output = ${directory:services}/mariadb_update +binary = ${binary-wrap-mysql_upgrade:wrapper-path} +mysql = ${binary-wrap-mysql:wrapper-path} +init-script = ${init-script:rendered} + +[mysqld] +recipe = slapos.cookbook:generic.mysql.wrap_mysqld +output = ${directory:services}/mariadb +binary = {{ parameter_dict['mariadb-location'] }}/bin/mysqld +configuration-file = ${my-cnf:rendered} +data-directory = ${my-cnf-parameters:data-directory} +mysql-install-binary = {{ parameter_dict['mariadb-location'] }}/scripts/mysql_install_db +mysql-base-directory = {{ parameter_dict['mariadb-location'] }} [mariadb-instance] -recipe = slapos.cookbook:generic.mysql - # Options user = user -parallel-test-database-amount = ${slap-parameter:test-database-amount} -port = 45678 -ip = ${slap-network-information:local-ipv4} +parallel-test-database-amount = {{ slapparameter_dict.get('test-database-amount', 30) }} database = erp5 test-user = erp5_test test-database = erp5_test mysql-test-database-base = testdb mysql-test-user-base = testuser -# Paths -wrapper = ${basedirectory:services}/mariadb -update-wrapper = ${basedirectory:services}/mariadb_update -backup-script = ${rootdirectory:bin}/innobackupex-controller -full-backup-directory = ${directory:mariadb-backup-full} -incremental-backup-directory = ${directory:mariadb-backup-incremental} -data-directory = ${directory:mariadb-data} -pid-file = ${basedirectory:run}/mariadb.pid -socket = ${basedirectory:run}/mariadb.sock -error-log = ${basedirectory:log}/mariadb_error.log -slow-query-log = ${basedirectory:log}/mariadb_slowquery.log -conf-file = ${rootdirectory:etc}/mariadb.cnf -bin-directory = ${rootdirectory:bin} -innobackupex-incremental = ${rootdirectory:bin}/innobackupex-incremental -innobackupex-full = ${rootdirectory:bin}/innobackupex-full - -# Binary information -innobackupex-binary = {{ xtrabackup_location }}/bin/innobackupex -mysql-base-directory = {{ mariadb_location }} -mysql-binary = {{ mariadb_location }}/bin/mysql -mysql-install-binary = {{ mariadb_location }}/scripts/mysql_install_db -mysql-upgrade-binary = {{ mariadb_location }}/bin/mysql_upgrade -mysqld-binary = {{ mariadb_location }}/bin/mysqld -pt-align-binary = {{ perl_siteprefix }}/bin/pt-align -pt-archiver-binary = {{ perl_siteprefix }}/bin/pt-archiver -pt-collect-binary = {{ perl_siteprefix }}/bin/pt-collect -pt-config-diff-binary = {{ perl_siteprefix }}/bin/pt-config-diff -pt-deadlock-logger-binary = {{ perl_siteprefix }}/bin/pt-deadlock-logger -pt-diskstats-binary = {{ perl_siteprefix }}/bin/pt-diskstats -pt-duplicate-key-checker-binary = {{ perl_siteprefix }}/bin/pt-duplicate-key-checker -pt-fifo-split-binary = {{ perl_siteprefix }}/bin/pt-fifo-split -pt-find-binary = {{ perl_siteprefix }}/bin/pt-find -pt-fingerprint-binary = {{ perl_siteprefix }}/bin/pt-fingerprint -pt-fk-error-logger-binary = {{ perl_siteprefix }}/bin/pt-fk-error-logger -pt-heartbeat-binary = {{ perl_siteprefix }}/bin/pt-heartbeat -pt-index-usage-binary = {{ perl_siteprefix }}/bin/pt-index-usage -pt-ioprofile-binary = {{ perl_siteprefix }}/bin/pt-ioprofile -pt-kill-binary = {{ perl_siteprefix }}/bin/pt-kill -pt-log-player-binary = {{ perl_siteprefix }}/bin/pt-log-player -pt-mext-binary = {{ perl_siteprefix }}/bin/pt-mext -pt-mysql-summary-binary = {{ perl_siteprefix }}/bin/pt-mysql-summary -pt-online-schema-change-binary = {{ perl_siteprefix }}/bin/pt-online-schema-change -pt-pmp-binary = {{ perl_siteprefix }}/bin/pt-pmp -pt-query-advisor-binary = {{ perl_siteprefix }}/bin/pt-query-advisor -pt-query-digest-binary = {{ perl_siteprefix }}/bin/pt-query-digest -pt-show-grants-binary = {{ perl_siteprefix }}/bin/pt-show-grants -pt-sift-binary = {{ perl_siteprefix }}/bin/pt-sift -pt-slave-delay-binary = {{ perl_siteprefix }}/bin/pt-slave-delay -pt-slave-find-binary = {{ perl_siteprefix }}/bin/pt-slave-find -pt-slave-restart-binary = {{ perl_siteprefix }}/bin/pt-slave-restart -pt-stalk-binary = {{ perl_siteprefix }}/bin/pt-stalk -pt-summary-binary = {{ perl_siteprefix }}/bin/pt-summary -pt-table-checksum-binary = {{ perl_siteprefix }}/bin/pt-table-checksum -pt-table-sync-binary = {{ perl_siteprefix }}/bin/pt-table-sync -pt-table-usage-binary = {{ perl_siteprefix }}/bin/pt-table-usage -pt-tcp-model-binary = {{ perl_siteprefix }}/bin/pt-tcp-model -pt-trend-binary = {{ perl_siteprefix }}/bin/pt-trend -pt-upgrade-binary = {{ perl_siteprefix }}/bin/pt-upgrade -pt-variable-advisor-binary = {{ perl_siteprefix }}/bin/pt-variable-advisor -pt-visual-explain-binary = {{ perl_siteprefix }}/bin/pt-visual-explain -xtrabackup-binary = {{ xtrabackup_location }}/bin/xtrabackup_51 -perl-binary = {{ perl_location }}/bin/perl - [logrotate] recipe = slapos.cookbook:logrotate # Binaries -logrotate-binary = {{ logrotate_location }}/usr/sbin/logrotate -gzip-binary = {{ gzip_location }}/bin/gzip -gunzip-binary = {{ gzip_location }}/bin/gunzip +logrotate-binary = {{ parameter_dict['logrotate-location'] }}/usr/sbin/logrotate +gzip-binary = {{ parameter_dict['gzip-location'] }}/bin/gzip +gunzip-binary = {{ parameter_dict['gzip-location'] }}/bin/gunzip # Directories -wrapper = ${rootdirectory:bin}/logrotate -conf = ${rootdirectory:etc}/logrotate.conf +wrapper = ${directory:bin}/logrotate +conf = ${directory:etc}/logrotate.conf logrotate-entries = ${directory:logrotate-entries} backup = ${directory:logrotate-backup} -state-file = ${rootdirectory:srv}/logrotate.status +state-file = ${directory:srv}/logrotate.status [logrotate-entry-mariadb] <= logrotate recipe = slapos.cookbook:logrotate.d name = mariadb -log = ${mariadb-instance:error-log} ${mariadb-instance:slow-query-log} -post = ${mariadb-instance:mysql-binary} --no-defaults -B --socket=${mariadb-instance:socket} -e "FLUSH LOGS" +log = ${my-cnf-parameters:error-log} ${my-cnf-parameters:slow-query-log} +post = "${binary-wrap-mysql:wrapper-path}" --no-defaults -B -u root -e "FLUSH LOGS" [cron] recipe = slapos.cookbook:cron -dcrond-binary = {{ dcron_location }}/sbin/crond +dcrond-binary = {{ parameter_dict['dcron-location'] }}/sbin/crond cron-entries = ${directory:cron-entries} crontabs = ${directory:crontabs} cronstamps = ${directory:cronstamps} catcher = ${cron-simplelogger:wrapper} -binary = ${basedirectory:services}/crond +binary = ${directory:services}/crond [cron-simplelogger] recipe = slapos.cookbook:simplelogger -wrapper = ${rootdirectory:bin}/cron_simplelogger -log = ${basedirectory:log}/cron.log +wrapper = ${directory:bin}/cron_simplelogger +log = ${directory:log}/cron.log [cron-entry-logrotate] <= cron @@ -145,51 +155,80 @@ command = ${logrotate:wrapper} [binary-link] recipe = slapos.cookbook:symbolic.link -target-directory = ${rootdirectory:bin} +target-directory = ${directory:bin} link-binary = - {{ coreutils_location }}/bin/basename - {{ coreutils_location }}/bin/cat - {{ coreutils_location }}/bin/cp - {{ coreutils_location }}/bin/ls - {{ coreutils_location }}/bin/tr - {{ coreutils_location }}/bin/uname - {{ gettext_location }}/lib/gettext/hostname - {{ grep_location }}/bin/grep - {{ sed_location }}/bin/sed - {{ mariadb_location }}/bin/mysql - -[rootdirectory] -recipe = slapos.cookbook:mkdirectory -etc = ${buildout:directory}/etc -var = ${buildout:directory}/var -srv = ${buildout:directory}/srv -bin = ${buildout:directory}/bin - -[basedirectory] -recipe = slapos.cookbook:mkdirectory -log = ${rootdirectory:var}/log -services = ${rootdirectory:etc}/run -promise = ${rootdirectory:etc}/promise -run = ${rootdirectory:var}/run -backup = ${rootdirectory:srv}/backup + {{ parameter_dict['coreutils-location'] }}/bin/basename + {{ parameter_dict['coreutils-location'] }}/bin/cat + {{ parameter_dict['coreutils-location'] }}/bin/cp + {{ parameter_dict['coreutils-location'] }}/bin/ls + {{ parameter_dict['coreutils-location'] }}/bin/tr + {{ parameter_dict['coreutils-location'] }}/bin/uname + {{ parameter_dict['gettext-location'] }}/lib/gettext/hostname + {{ parameter_dict['grep-location'] }}/bin/grep + {{ parameter_dict['sed-location'] }}/bin/sed + +[binary-wrap-base] +recipe = slapos.cookbook:wrapper +# Note: --defaults-file must be the first argument, otherwise wrapped binary +# will reject it. +command-line = "{{ parameter_dict['mariadb-location'] }}/bin/${:command}" --defaults-file="${my-cnf:rendered}" +wrapper-path = ${directory:bin}/${:command} +parameters-extra = true + +[binary-wrap-mysql] +< = binary-wrap-base +command = mysql + +[binary-wrap-mysqldump] +< = binary-wrap-base +command = mysqldump + +[binary-wap-mysql_upgrade] +< = binary-wrap-base +command-line = mysql_upgrade [directory] recipe = slapos.cookbook:mkdirectory -cron-entries = ${rootdirectory:etc}/cron.d -crontabs = ${rootdirectory:etc}/crontabs -cronstamps = ${rootdirectory:etc}/cronstamps -ca-dir = ${rootdirectory:srv}/ssl -mariadb-backup-full = ${basedirectory:backup}/mariadb-full -mariadb-backup-incremental = ${basedirectory:backup}/mariadb-incremental -mariadb-data = ${rootdirectory:srv}/mariadb -logrotate-backup = ${basedirectory:backup}/logrotate -logrotate-entries = ${rootdirectory:etc}/logrotate.d +bin = ${buildout:directory}/bin +etc = ${buildout:directory}/etc +services = ${:etc}/run +promise = ${:etc}/promise +cron-entries = ${:etc}/cron.d +crontabs = ${:etc}/crontabs +cronstamps = ${:etc}/cronstamps +logrotate-entries = ${:etc}/logrotate.d +srv = ${buildout:directory}/srv +backup = ${:srv}/backup +mariadb-backup-full = ${:backup}/mariadb-full +mariadb-backup-incremental = ${:backup}/mariadb-incremental +logrotate-backup = ${:backup}/logrotate +mariadb-data = ${:srv}/mariadb +var = ${buildout:directory}/var +log = ${:var}/log +run = ${:var}/run [promise] recipe = slapos.cookbook:check_port_listening -path = ${basedirectory:promise}/mariadb -hostname = ${mariadb-instance:ip} -port = ${mariadb-instance:port} +path = ${directory:promise}/mariadb +hostname = ${my-cnf-parameters:ip} +port = ${my-cnf-parameters:port} -[slap-parameter] -test-database-amount = 100 +[buildout] +parts = + publish-mariadb-url + logrotate + logrotate-entry-mariadb + cron + cron-entry-logrotate + cron-entry-mariadb-backup + cron-entry-mariadb-backup-expire + binary-link + update-mysql + mysqld + promise + {{ part_list | join('\n ') }} + +eggs-directory = {{ eggs_directory }} +develop-eggs-directory = {{ develop_eggs_directory }} +offline = true +{% endif %} diff --git a/stack/erp5/instance-memcached.cfg.in b/stack/erp5/instance-memcached.cfg.in deleted file mode 100644 index 7fc8f81c0..000000000 --- a/stack/erp5/instance-memcached.cfg.in +++ /dev/null @@ -1,147 +0,0 @@ -# memcached-compatible volatile cache using kumofs -# that has no limitation for key length and data size - -[buildout] -parts = - publish-kumofs-connection-information - kumofs-instance - logrotate - logrotate-entry-kumofs - cron - cron-entry-logrotate - promise-kumofs-server - promise-kumofs-server-listen - promise-kumofs-gateway - promise-kumofs-manager - -eggs-directory = {{ eggs_directory }} -develop-eggs-directory = {{ develop_eggs_directory }} -offline = true - -[publish-kumofs-connection-information] -recipe = slapos.cookbook:publishurl -url = memcached://${kumofs-instance:ip}:${kumofs-instance:gateway-port}/ - -[kumofs-instance] -recipe = slapos.cookbook:generic.kumofs - -# Network options -ip = ${slap-network-information:local-ipv4} -manager-port = 13401 -server-port = 13501 -server-listen-port = 13502 -# previous memcached configuration -gateway-port = 11000 -# previous memcached configuration -storage-size = 64m - -# Paths: Running wrappers -gateway-wrapper = ${basedirectory:services}/volatile_kumofs_gateway -manager-wrapper = ${basedirectory:services}/volatile_kumofs_manager -server-wrapper = ${basedirectory:services}/volatile_kumofs_server - -# Paths: Data -data-path = *#capsiz=${:storage-size} - -# Paths: Logs -kumo-gateway-log = ${basedirectory:log}/kumo-gateway.log -kumo-manager-log = ${basedirectory:log}/kumo-manager.log -kumo-server-log = ${basedirectory:log}/kumo-server.log - -# Binary information -kumo-gateway-binary = {{ kumo_location }}/bin/kumo-gateway -kumo-manager-binary = {{ kumo_location }}/bin/kumo-manager -kumo-server-binary = {{ kumo_location }}/bin/kumo-server -shell-path = {{ dash_location }}/bin/dash - -[logrotate-entry-kumofs] -<= logrotate -recipe = slapos.cookbook:logrotate.d -name = kumofs -log = ${kumofs-instance:kumo-gateway-log} ${kumofs-instance:kumo-manager-log} - ${kumofs-instance:kumo-server-log} - -# rest of parts are candidates for some generic stuff - -[logrotate] -recipe = slapos.cookbook:logrotate -# Binaries -logrotate-binary = {{ logrotate_location }}/usr/sbin/logrotate -gzip-binary = {{ gzip_location }}/bin/gzip -gunzip-binary = {{ gzip_location }}/bin/gunzip -# Directories -wrapper = ${rootdirectory:bin}/logrotate -conf = ${rootdirectory:etc}/logrotate.conf -logrotate-entries = ${directory:logrotate-entries} -backup = ${directory:logrotate-backup} -state-file = ${rootdirectory:srv}/logrotate.status - -[basedirectory] -recipe = slapos.cookbook:mkdirectory -log = ${rootdirectory:var}/log -services = ${rootdirectory:etc}/run -promise = ${rootdirectory:etc}/promise -run = ${rootdirectory:var}/run -backup = ${rootdirectory:srv}/backup - -[directory] -recipe = slapos.cookbook:mkdirectory -cron-entries = ${rootdirectory:etc}/cron.d -crontabs = ${rootdirectory:etc}/crontabs -cronstamps = ${rootdirectory:etc}/cronstamps -logrotate-backup = ${basedirectory:backup}/logrotate -logrotate-entries = ${rootdirectory:etc}/logrotate.d - -[rootdirectory] -recipe = slapos.cookbook:mkdirectory -etc = ${buildout:directory}/etc -var = ${buildout:directory}/var -srv = ${buildout:directory}/srv -bin = ${buildout:directory}/bin - -[cron] -recipe = slapos.cookbook:cron -dcrond-binary = {{ dcron_location }}/sbin/crond -cron-entries = ${directory:cron-entries} -crontabs = ${directory:crontabs} -cronstamps = ${directory:cronstamps} -catcher = ${cron-simplelogger:wrapper} -binary = ${basedirectory:services}/crond - -[cron-simplelogger] -recipe = slapos.cookbook:simplelogger -wrapper = ${rootdirectory:bin}/cron_simplelogger -log = ${basedirectory:log}/cron.log - -[cron-entry-logrotate] -<= cron -recipe = slapos.cookbook:cron.d -name = logrotate -frequency = 0 0 * * * -command = ${logrotate:wrapper} - -# Deploy zope promises scripts -[promise-template] -recipe = slapos.cookbook:check_port_listening -hostname = ${kumofs-instance:ip} -port = ${kumofs-instance:server-listen-port} - -[promise-kumofs-server] -<= promise-template -path = ${basedirectory:promise}/kumofs-server -port = ${kumofs-instance:server-port} - -[promise-kumofs-server-listen] -<= promise-template -path = ${basedirectory:promise}/kumofs-server-listen -port = ${kumofs-instance:server-listen-port} - -[promise-kumofs-gateway] -<= promise-template -path = ${basedirectory:promise}/kumofs-gateway -port = ${kumofs-instance:gateway-port} - -[promise-kumofs-manager] -<= promise-template -path = ${basedirectory:promise}/kumofs-manager -port = ${kumofs-instance:manager-port} diff --git a/stack/erp5/instance-tidstorage.cfg.in b/stack/erp5/instance-tidstorage.cfg.in deleted file mode 100644 index bb930ae6f..000000000 --- a/stack/erp5/instance-tidstorage.cfg.in +++ /dev/null @@ -1,538 +0,0 @@ -{% if software_type == slap_software_type -%} -{# -Note: all port counters are pre-incremented. No idea why base port is skipped. --#} -{% set current_zeo_port = zeo_port_base | int -%} -{% set zope_port_base = zope_port_base | int -%} -{% set zope_dummy_list = [] -%} -{% set current_apache_port = apache_port_base | int -%} -{% set current_haproxy_port = haproxy_port_base | int -%} -{% set json = json_module.loads(slapparameter_dict['json']) -%} -{% set bin_directory = parameter_dict['buildout-bin-directory'] -%} -{# -XXX: This template only supports exactly one IPv4 and one IPv6 per -partition. No more (undefined result), no less (IndexError). --#} -{% set ipv4 = (ipv4_set | list)[0] -%} -{% set ipv6 = (ipv6_set | list)[0] -%} -{# -BBB: erp5-ca['state'] has been configured as string by mistake. Keep this for -backward compatibility with existing automatically setup CAs. --#} -{% set erp5_ca = json.get('erp5-ca', { - 'country-code': 'ZZ', - 'email': 'nobody@example.com', - 'state': "('State',)", - 'city': 'City', - 'company': 'Company', -}) -%} -{% set site_id = json['site-id'] -%} -{% set part_list = [] -%} -{% set known_tid_storage_identifier_dict = {} -%} -{% set zodb_connection_list = [] -%} -{% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%} -{% macro zope( - name, - thread_amount=1, - timerserver_interval=0, - longrequest_logger_file='', - longrequest_logger_timeout='', - longrequest_logger_interval='' -) -%} -{% set conf_name = name ~ '-conf' -%} -{% set conf_parameter_name = conf_name ~ '-param' -%} -[{{ conf_parameter_name }}] -< = zope-conf-parameter-base -pid-file = ${directory:run}/{{ name }}.pid -lock-file = ${directory:run}/{{ name }}.lock -{% do zope_dummy_list.append(None) -%} -{% set offset = zope_dummy_list | length -%} -port = {{ zope_port_base + offset }} -thread-amount = {{ thread_amount }} -{% if timerserver_interval -%} -timerserver-interval = {{ timerserver_interval }} -{% endif -%} -event-log = ${directory:log}/{{ name }}-event.log -z2-log = ${directory:log}/{{ name }}-Z2.log - -[{{ conf_name }}] -< = zope-conf-base -rendered = ${directory:etc}/{{ name }}.conf -extra-context = - section parameter_dict {{ conf_parameter_name }} - -[{{ section(name) }}] -< = zope-base -longrequest-logger-file = {{ longrequest_logger_file }} -longrequest-logger-timeout = {{ longrequest_logger_timeout }} -longrequest-logger-interval = {{ longrequest_logger_interval }} -wrapper = ${directory:services}/{{ name }} -configuration-file = {{ '${' ~ conf_name ~ ':rendered}' }} -port = {{ '${' ~ conf_parameter_name ~ ':port}' }} - -[{{ section('logrotate-entry-' ~ name) }}] -< = logrotate-base -recipe = slapos.cookbook:logrotate.d -name = {{ name }} -log = {{ '${' ~ conf_parameter_name ~ ':event-log}' }} {{ '${' ~ conf_parameter_name ~ ':z2-log}' }} -post = {{ bin_directory }}/killpidfromfile {{ '${' ~ conf_parameter_name ~ ':pid-file}' }} SIGUSR2 -{% endmacro -%} -############################# -# Directory creation -############################# -[directory] -recipe = slapos.cookbook:mkdirectory -apache-conf = ${:etc}/apache -backup = ${:srv}/backup -bin = ${buildout:directory}/bin -ca-dir = ${:srv}/ssl -cron-entries = ${:etc}/cron.d -cronstamps = ${:etc}/cronstamps -crontabs = ${:etc}/crontabs -erp5-ca-dir = ${:srv}/erp5-ssl -etc = ${buildout:directory}/etc -instance = ${:srv}/erp5shared -instance-constraint = ${:instance}/Constraint -instance-document = ${:instance}/Document -instance-etc = ${:instance}/etc -instance-etc-package-include = ${:instance}/etc/package-include -instance-extensions = ${:instance}/Extensions -instance-import = ${:instance}/import -instance-lib = ${:instance}/lib -instance-products = ${:instance}/Products -instance-propertysheet = ${:instance}/PropertySheet -instance-tests = ${:instance}/tests -log = ${:var}/log -logrotate-backup = ${:backup}/logrotate -logrotate-entries = ${:etc}/logrotate.d -run = ${:var}/run -services = ${:etc}/run -srv = ${buildout:directory}/srv -tidstorage = ${:srv}/tidstorage -tmp = ${buildout:directory}/tmp -var = ${buildout:directory}/var -zodb = ${:srv}/zodb -zodb-backup = ${:backup}/zodb - -############################# -# Binary symlinking -############################# -[binary-link] -recipe = slapos.cookbook:symbolic.link -target-directory = ${directory:bin} -link-binary = - {{ parameter_dict['coreutils'] }}/bin/basename - {{ parameter_dict['coreutils'] }}/bin/cat - {{ parameter_dict['coreutils'] }}/bin/cp - {{ parameter_dict['coreutils'] }}/bin/ls - {{ parameter_dict['coreutils'] }}/bin/tr - {{ parameter_dict['coreutils'] }}/bin/uname - {{ parameter_dict['git'] }}/bin/git - {{ parameter_dict['graphviz'] }}/bin/dot - {{ parameter_dict['grep'] }}/bin/grep - {{ parameter_dict['imagemagick'] }}/bin/convert - {{ parameter_dict['imagemagick'] }}/bin/identify - {{ parameter_dict['mariadb'] }}/bin/mysql - {{ parameter_dict['mariadb'] }}/bin/mysqldump - {{ parameter_dict['sed'] }}/bin/sed - {{ parameter_dict['tesseract'] }}/bin/tesseract - {{ parameter_dict['w3m'] }}/bin/w3m - {{ parameter_dict['openssl'] }}/bin/openssl - {{ parameter_dict['poppler'] }}/bin/pdfinfo - {{ parameter_dict['poppler'] }}/bin/pdfseparate - {{ parameter_dict['poppler'] }}/bin/pdftotext - {{ parameter_dict['poppler'] }}/bin/pdftohtml - {{ parameter_dict['poppler'] }}/bin/pdfunite - {{ parameter_dict['dmtx-utils'] }}/bin/dmtxwrite - -############################# -# CA -############################# -[cadirectory] -recipe = slapos.cookbook:mkdirectory -requests = ${directory:ca-dir}/requests -private = ${directory:ca-dir}/private -certs = ${directory:ca-dir}/certs -newcerts = ${directory:ca-dir}/newcerts -crl = ${directory:ca-dir}/crl - -[certificate-authority] -recipe = slapos.cookbook:certificate_authority -openssl-binary = {{ parameter_dict['openssl'] }}/bin/openssl -ca-dir = ${directory:ca-dir} -requests-directory = ${cadirectory:requests} -wrapper = ${directory:services}/ca -ca-private = ${cadirectory:private} -ca-certs = ${cadirectory:certs} -ca-newcerts = ${cadirectory:newcerts} -ca-crl = ${cadirectory:crl} - -############################# -# ERP5 CA -############################# -[erp5-cadirectory] -recipe = slapos.cookbook:mkdirectory -requests = ${directory:erp5-ca-dir}/requests -private = ${directory:erp5-ca-dir}/private -certs = ${directory:erp5-ca-dir}/certs -newcerts = ${directory:erp5-ca-dir}/newcerts -crl = ${directory:erp5-ca-dir}/crl - -[erp5-certificate-authority] -recipe = slapos.cookbook:certificate_authority -openssl-binary = {{ parameter_dict['openssl'] }}/bin/openssl -ca-dir = ${directory:erp5-ca-dir} -requests-directory = ${erp5-cadirectory:requests} -wrapper = ${directory:services}/erp5-ca -ca-private = ${erp5-cadirectory:private} -ca-certs = ${erp5-cadirectory:certs} -ca-newcerts = ${erp5-cadirectory:newcerts} -ca-crl = ${erp5-cadirectory:crl} -country-code = {{ erp5_ca['country-code'] }} -email = {{ erp5_ca['email'] }} -state = {{ erp5_ca['state'] }} -city = {{ erp5_ca['city'] }} -company = {{ erp5_ca['company'] }} - -############################# -# CRON -############################# -[cron-base] -cron-entries = ${directory:cron-entries} - -[cron] -< = cron-base -recipe = slapos.cookbook:cron -dcrond-binary = {{ parameter_dict['dcron'] }}/sbin/crond -crontabs = ${directory:crontabs} -cronstamps = ${directory:cronstamps} -catcher = ${cron-simplelogger:wrapper} -binary = ${directory:services}/crond - -[cron-simplelogger] -recipe = slapos.cookbook:simplelogger -wrapper = ${directory:bin}/cron_simplelogger -log = ${directory:log}/cron.log - -############################# -# Logrotate -############################# -[logrotate-base] -logrotate-entries = ${directory:logrotate-entries} -backup = ${directory:logrotate-backup} - -[logrotate] -< = logrotate-base -recipe = slapos.cookbook:logrotate -# Binaries -logrotate-binary = {{ parameter_dict['logrotate'] }}/usr/sbin/logrotate -gzip-binary = {{ parameter_dict['gzip'] }}/bin/gzip -gunzip-binary = {{ parameter_dict['gzip'] }}/bin/gunzip -# Directories -wrapper = ${directory:bin}/logrotate -conf = ${directory:etc}/logrotate.conf -state-file = ${directory:srv}/logrotate.status - -[cron-entry-logrotate] -< = cron-base -recipe = slapos.cookbook:cron.d -name = logrotate -frequency = 0 0 * * * -command = ${logrotate:wrapper} - -############################# -# ERP5 bootstrap -############################# -[erp5-bootstrap] -recipe = slapos.cookbook:erp5.bootstrap -runner-path = ${directory:services}/erp5-bootstrap -mysql-url = {{ slapparameter_dict['mysql-url'] }} -zope-url = http://${zope-admin:user}:${zope-admin:password}@${zope-admin:ip}:${zope-admin:port}/{{ site_id }} - -############################# -# ERP5 promise -############################# -[erp5-promise] -recipe = slapos.cookbook:erp5.promise -promise-path = ${directory:etc}/erp5promise.cfg -kumofs-url = {{ slapparameter_dict['kumofs-url'] }} -memcached-url = {{ slapparameter_dict['memcached-url'] }} -cloudooo-url = {{ slapparameter_dict['cloudooo-url'] }} -smtp-url = ${slap-parameter:smtp-url} -bt5 = ${slap-parameter:bt5} -bt5-repository-url = ${slap-parameter:bt5-repository-url} - -############################# -# ZEO -############################# -[zeo-instance-entry-common] -recipe = slapos.cookbook:zeo -ip = {{ ipv4 }} -binary-path = {{ bin_directory }}/runzeo - -{% for zeo_id, zeo_configuration_list in json['zeo'].iteritems() -%} -{% set current_zeo_port = current_zeo_port + 1 -%} -{% set storage_list = [] -%} -{% for zeo_slave in zeo_configuration_list -%} -{% do zodb_connection_list.append(( - zeo_slave['storage-name'], - zeo_slave['mount-point'] % {'site-id': site_id}, - zeo_slave['zope-cache-size'], - 'zeoclient', - { - 'cache-size': zeo_slave['zeo-cache-size'], - 'server': ipv4 ~ ':' ~ current_zeo_port, - 'storage': zeo_slave['storage-name'], - 'name': zeo_slave['storage-name'], - }, - )) -%} -{% set zodb_path = '${directory:zodb}/' ~ zeo_slave['storage-name'] ~ '.fs' -%} -{% do storage_list.append('storage-name=%(storage-name)s zodb-path=%(zodb-path)s' % {'zodb-path': zodb_path, 'storage-name': zeo_slave['storage-name']}) -%} -{% do known_tid_storage_identifier_dict.__setitem__("((('${zeo-instance-%(zeo_id)s:ip}', ${zeo-instance-%(zeo_id)s:port}),), '%(storage_name)s')" % { - 'zeo_id': zeo_id, - 'storage_name': zeo_slave['storage-name'] - }, (zodb_path, '${directory:zodb-backup}/%s/' % zeo_slave['storage-name'], zeo_slave['serialize-path'] % {'site-id': site_id})) -%} -{% endfor -%} -[{{ section('zeo-instance-%s' % zeo_id) }}] -< = zeo-instance-entry-common -log-path = ${directory:log}/zeo-{{ zeo_id }}.log -pid-path = ${directory:run}/zeo-{{ zeo_id }}.pid -conf-path = ${directory:etc}/zeo-{{ zeo_id }}.conf -port = {{ current_zeo_port }} -storage = - {{ storage_list | join('\n ') }} -wrapper-path = ${directory:services}/zeo-{{ zeo_id }} - -[{{ section('logrotate-entry-zeo-%s' % zeo_id) }}] -< = logrotate-base -recipe = slapos.cookbook:logrotate.d -name = zeo-{{ zeo_id }} -log = ${zeo-instance-{{ zeo_id }}:log-path} -post = {{ bin_directory }}/killpidfromfile ${zeo-instance-{{ zeo_id }}:pid-path} SIGUSR2 - -{% endfor -%} -############################# -# Zope -############################# -[zope-base] -recipe = slapos.cookbook:generic.zope.zeo.client -user = zope -ip = {{ ipv4 }} -timezone = {{ json['timezone'] }} -tidstorage-ip = ${tidstorage:ip} -tidstorage-port = ${tidstorage:port} -instance-etc = ${directory:instance-etc} -bt5-repository = ${directory:var}/bt5_repository -tmp-path = ${directory:tmp} -bin-path = ${directory:bin} -site-zcml = ${:instance-etc}/site.zcml -inituser = ${directory:instance}/inituser -runzope-binary = {{ bin_directory }}/runzope -bt5-repository-list = - -[deadlock-debugger-password] -recipe = slapos.cookbook:pwgen.stable - -[zope-conf-parameter-base] -ip = {{ ipv4 }} -site-id = {{ site_id }} -zodb-list = {{ dumps(zodb_connection_list) }} - -[zope-conf-base] -recipe = slapos.recipe.template:jinja2 -template = {{ parameter_dict['zope-conf-template'] }} -extra-context = -context = - key instance directory:instance - key instance_products directory:instance-products - raw deadlock_path /manage_debug_threads - key deadlock_debugger_password deadlock-debugger-password:password - key tidstorage_ip tidstorage:ip - key tidstorage_port tidstorage:port - key promise_path erp5-promise:promise-path - ${:extra-context} - -# Distribution node -{{ zope('zope-distribution', timerserver_interval=1) }} -# Admin node -{{ zope('zope-admin') }} -# Activity nodes -{% for q in range(1, json['activity']['zopecount'] + 1) -%} -{{ zope('zope-activity-%s' % q, timerserver_interval=1) }} -{%- endfor %} -# Other zopes, apaches and haproxies -{% set publish_url_list = [] -%} -{% for backend_name, backend_configuration in json['backend'].iteritems() -%} -{% set haproxy_backend_list = [] -%} -{% set longrequest_logger = backend_configuration.get('longrequest-logger') -%} -{% for q in range(1, backend_configuration['zopecount'] + 1) -%} -{% set part_name = 'zope-%s-%s' % (backend_name, q) -%} -{% if longrequest_logger != None -%} -{% set longrequest_logger_file = '${directory:log}/%s-longrequest.log' % (part_name, ) -%} -{% set longrequest_logger_timeout = longrequest_logger.get('timeout', '4') -%} -{% set longrequest_logger_interval = longrequest_logger.get('interval', '2') -%} -{% else -%} -{% set longrequest_logger_file = '' -%} -{% set longrequest_logger_timeout = '' -%} -{% set longrequest_logger_interval = '' -%} -{% endif -%} -{{ zope( - part_name, - thread_amount=backend_configuration['thread-amount'], - longrequest_logger_file=longrequest_logger_file, - longrequest_logger_timeout=longrequest_logger_timeout, - longrequest_logger_interval=longrequest_logger_interval, - ) }} -{% do haproxy_backend_list.append('${%(part_name)s:ip}:${%(part_name)s:port}' % {'part_name': part_name}) -%} -{% endfor -%} -{% set scheme = backend_configuration.get('scheme', ['https']) -%} -{% set current_apache_port = current_apache_port + 2 -%} -{% set current_haproxy_port = current_haproxy_port + 1 -%} -{% if 'http' in scheme -%} -{% set section_name = 'apache-public-' ~ backend_name -%} -{% do publish_url_list.append( - 'url-public-%(backend_name)s = http://[${%(section_name)s:ip}]:${%(section_name)s:port}' % { - 'backend_name': backend_name, - 'section_name': section_name, - }) -%} -[{{ section(section_name) }}] -recipe = slapos.cookbook:apache.zope.backend -backend = http://${haproxy-{{ backend_name }}:ip}:${haproxy-{{ backend_name }}:port} -ip = {{ ipv6 }} -port = {{ current_apache_port }} -scheme = http -wrapper = ${directory:services}/apache-public-{{ backend_name }} -configuration-file = ${directory:apache-conf}/apache-public-{{ backend_name }}.conf -access-control-string = {{ backend_configuration['access-control-string'] }} -pid-file = ${directory:run}/apache-public-{{ backend_name }}.pid -lock-file = ${directory:run}/apache-public-{{ backend_name }}.lock -error-log = ${directory:log}/apache-public-{{ backend_name }}-error.log -access-log = ${directory:log}/apache-public-{{ backend_name }}-access.log -apache-binary = {{ parameter_dict['apache'] }}/bin/httpd - -[{{ section('logrotate-entry-apache-public-' ~ backend_name) }}] -< = logrotate-base -recipe = slapos.cookbook:logrotate.d -name = apache-public-{{ backend_name }} -log = ${apache-public-{{ backend_name }}:error-log} ${apache-public-{{ backend_name }}:access-log} -post = {{ bin_directory }}/killpidfromfile ${apache-public-{{ backend_name }}:pid-file} SIGUSR1 -{% endif -%} -{% if 'https' in scheme -%} -{% set section_name = 'apache-' ~ backend_name -%} -{% do publish_url_list.append( - 'url-%(backend_name)s = https://[${%(section_name)s:ip}]:${%(section_name)s:port}' % { - 'backend_name': backend_name, - 'section_name': section_name, - }) -%} -[{{ section(section_name) }}] -recipe = slapos.cookbook:apache.zope.backend -backend = http://${haproxy-{{ backend_name }}:ip}:${haproxy-{{ backend_name }}:port} -ip = {{ ipv6 }} -port = {{ current_apache_port - 1 }} -wrapper = ${directory:bin}/apache-{{ backend_name }} -scheme = https -key-file = ${directory:apache-conf}/apache-{{ backend_name }}.key -cert-file = ${directory:apache-conf}/apache-{{ backend_name }}.crt -configuration-file = ${directory:apache-conf}/apache-{{ backend_name }}.conf -access-control-string = {{ backend_configuration['access-control-string'] }} -pid-file = ${directory:run}/apache-{{ backend_name }}.pid -lock-file = ${directory:run}/apache-{{ backend_name }}.lock -ssl-session-cache = $${directory:log}/apache-ssl-session-cache -error-log = ${directory:log}/apache-{{ backend_name }}-error.log -access-log = ${directory:log}/apache-{{ backend_name }}-access.log -apache-binary = {{ parameter_dict['apache'] }}/bin/httpd -ssl-authentication = {{ backend_configuration.get('ssl-authentication', False) }} -backend-path = {{ backend_configuration.get('backend-path', '/') % {'site-id': site_id} }} -# Note: Without erp5-certificate-authority main certificate have to be hardcoded -ssl-authentication-certificate = ${erp5-certificate-authority:ca-dir}/cacert.pem -ssl-authentication-crl = ${erp5-certificate-authority:ca-crl} - -[{{ section('ca-apache-' ~ backend_name) }}] -< = certificate-authority -recipe = slapos.cookbook:certificate_authority.request -key-file = ${apache-{{ backend_name }}:key-file} -cert-file = ${apache-{{ backend_name }}:cert-file} -executable = ${apache-{{ backend_name }}:wrapper} -wrapper = ${directory:services}/apache-{{ backend_name }} - -[{{ section('logrotate-entry-apache-' ~ backend_name) }}] -< = logrotate-base -recipe = slapos.cookbook:logrotate.d -name = apache-{{ backend_name }} -log = ${apache-{{ backend_name }}:error-log} ${apache-{{ backend_name }}:access-log} -post = {{ bin_directory }}/killpidfromfile ${apache-{{ backend_name }}:pid-file} SIGUSR1 - -[{{ section('haproxy-' ~ backend_name) }}] -recipe = slapos.cookbook:haproxy -name = {{ backend_name }} -conf-path = ${directory:etc}/haproxy-{{ backend_name }}.cfg -socket-path = ${directory:run}/haproxy-{{ backend_name }}.sock -ip = {{ ipv4 }} -port = {{ current_haproxy_port }} -maxconn = {{ backend_configuration['maxconn'] }} -server-check-path = /{{ site_id }}/getId -wrapper-path = ${directory:services}/haproxy-{{ backend_name }} -binary-path = {{ parameter_dict['haproxy'] }}/sbin/haproxy -ctl-path = ${directory:bin}/haproxy-{{ backend_name }}-ctl -backend-list = {{ haproxy_backend_list | join(' ')}} -{%- endif %} -{% endfor -%} -[{{ section('publish-apache-backend-list') }}] -recipe = slapos.cookbook:publish -{{ publish_url_list | join('\n') }} - -############################# -# tidstorage -############################# -[tidstorage] -recipe = slapos.cookbook:tidstorage -known-tid-storage-identifier-dict = {{ known_tid_storage_identifier_dict }} -configuration-path = ${directory:etc}/tidstorage.py -ip = {{ ipv4 }} -port = 6001 -timestamp-file-path = ${directory:tidstorage}/repozo_tidstorage_timestamp.log -logfile-name = ${directory:log}/tidstorage.log -pidfile-name = ${directory:run}/tidstorage.pid -status-file = ${directory:tidstorage}/tidstorage.tid -tidstorage-repozo-binary = {{ bin_directory }}/tidstorage_repozo -tidstoraged-binary = {{ bin_directory }}/tidstoraged -repozo-binary = {{ bin_directory }}/repozo -tidstorage-wrapper = ${directory:services}/tidstoraged -repozo-wrapper = ${buildout:bin-directory}/tidstorage-repozo - -[cron-entry-tidstorage-backup] -< = cron-base -recipe = slapos.cookbook:cron.d -name = tidstorage -frequency = 0 0 * * * -command = ${tidstorage:repozo-wrapper} - -[logrotate-entry-tidstorage] -< = logrotate-base -recipe = slapos.cookbook:logrotate.d -name = tidstorage -log = ${tidstorage:logfile-name} -post = {{ bin_directory }}/killpidfromfile ${tidstorage:pidfile-name} SIGHUP - -############################# -# buildout main section -############################# -[buildout] -parts = - logrotate - cron - cron-entry-logrotate - certificate-authority - erp5-certificate-authority - tidstorage - cron-entry-tidstorage-backup - logrotate-entry-tidstorage - binary-link - erp5-promise - erp5-bootstrap - {{ part_list | join('\n ') }} - -eggs-directory = {{ eggs_directory }} -develop-eggs-directory = {{ develop_eggs_directory }} -offline = true -{%- endif %} diff --git a/stack/erp5/instance-varnish.cfg.in b/stack/erp5/instance-varnish.cfg.in index 7867ba1c1..ab03c36d6 100644 --- a/stack/erp5/instance-varnish.cfg.in +++ b/stack/erp5/instance-varnish.cfg.in @@ -18,7 +18,7 @@ develop-eggs-directory = {{ develop_eggs_directory }} offline = true [publish-varnish-connection-information] -recipe = slapos.cookbook:publishurl +recipe = slapos.cookbook:publish.serialised url = http://${varnish-instance:ip}:${varnish-instance:server-port}/ [varnish-instance] diff --git a/stack/erp5/instance-zeo.cfg.in b/stack/erp5/instance-zeo.cfg.in new file mode 100644 index 000000000..0b5d91ea3 --- /dev/null +++ b/stack/erp5/instance-zeo.cfg.in @@ -0,0 +1,220 @@ +{% if software_type == slap_software_type -%} +{% set use_ipv6 = slapparameter_dict.get('use-ipv6', False) -%} +{% set next_port = slapparameter_dict['tcpv4-port'] -%} +{% set part_list = [] -%} +{% set storage_dict = {} -%} +{% set zodb_dict = {} -%} +{% set tidstorage_dict = slapparameter_dict.get('tidstorage-dict') -%} +{% set known_tid_storage_identifier_dict = {} -%} +{% set default_zodb_path = buildout_directory ~ '/srv/zodb' -%} +{% set zodb_backup_path = buildout_directory ~ '/srv/backup/zodb' -%} +{% set default_tidstorage_timestamp_path = buildout_directory ~ '/srv/backup/tidstorage' -%} +{% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%} +{# +XXX: This template only supports exactly one IPv6 per +partition. No more (undefined result), no less (IndexError). +-#} +{% set ipv4 = (ipv4_set | list)[0] -%} +{% set tidstorage_ip = ipv4 -%} +{% set tidstorage_port = 6001 -%} +{% set bin_directory = parameter_dict['buildout-bin-directory'] -%} + +[logrotate-base] +recipe = slapos.cookbook:logrotate.d +logrotate-entries = ${logrotate:logrotate-entries} +backup = ${logrotate:backup} + +[zeo-base] +recipe = slapos.cookbook:zeo +log-path = ${directory:log}/${:base-name}.log +pid-path = ${directory:run}/${:base-name}.pid +conf-path = ${directory:etc}/${:base-name}.conf +wrapper-path = ${directory:services}/${:base-name} +binary-path = {{ bin_directory }}/runzeo +ip = {{ ipv4 }} + +{% if use_ipv6 -%} +{% set ipv6 = (ipv6_set | list)[0] -%} +[ipv6toipv4-base] +recipe = slapos.cookbook:ipv6toipv4 +runner-path = ${directory:services}/${:base-name} +6tunnel-path = {{ parameter_dict['6tunnel'] }}/bin/6tunnel +shell-path = {{ parameter_dict['dash'] }}/bin/dash +ipv4 = {{ ipv4 }} +ipv6 = {{ ipv6 }} +{% endif -%} + +{% for export_id, mountpoint_dict in slapparameter_dict['zodb-dict'].items() -%} +{% do storage_dict.setdefault( + mountpoint_dict.get('storage-family', 'default'), [] + ).append(( + export_id, + mountpoint_dict.get('mount-point', '/'), + mountpoint_dict.get('cache-size', -1), + mountpoint_dict.get('storage-dict', {}), + )) -%} +{% endfor -%} +{% for storage_family, export_list in storage_dict.items() -%} +{% set known_tid_storage_identifier_host = ((ipv4, next_port), ) -%} +{% set client_dict = {} -%} +{% for export_id, mount_point, cache_size, storage_dict in export_list -%} +{% do storage_dict.__setitem__('path', storage_dict.get('path', '%(zodb)s/' ~ export_id ~ '.fs') % {'zodb': default_zodb_path}) -%} +{% do client_dict.update(storage_dict.get('client', {})) -%} +{% do client_dict.__setitem__('storage', export_id) -%} +{# XXX: I would like to raise if export_id is present in zodb_dict -#} +{% do zodb_dict.__setitem__(export_id, [ + mount_point, + cache_size, + client_dict, + ]) -%} +{% if tidstorage_dict != None -%} +{% do known_tid_storage_identifier_dict.__setitem__( + json_module.dumps((known_tid_storage_identifier_host, export_id)), ( + storage_dict['path'], + tidstorage_dict.get('zodb-dict', {}).get(export_id, '%(backup)s/' ~ export_id) % {'backup': zodb_backup_path}, + mount_point, + ), + ) -%} +{% endif -%} +{% endfor -%} +{% set zeo_section_name = 'zeo-' ~ storage_family %} +[{{ zeo_section_name }}] +< = zeo-base +base-name = zeo-{{ storage_family }} +port = {{ next_port }} +{% set storage_list = [] -%} +{% for storage_name, _, _, storage_dict in export_list -%} +{% do storage_list.append((storage_name, storage_dict['path'])) -%} +{% endfor -%} +storage = {{ dumps(storage_list) }} + +[{{ section("logrotate-" ~ zeo_section_name) }}] +< = logrotate-base +name = {{ "${" ~ zeo_section_name ~ ":base-name}" }} +log = {{ "${" ~ zeo_section_name ~ ":log-path}" }} +post = {{ bin_directory }}/killpidfromfile {{ "${" ~ zeo_section_name ~ ":pid-path}" }} SIGUSR2 + +[{{ section(zeo_section_name ~ "-promise") }}] +recipe = slapos.cookbook:check_port_listening +hostname = {{ "${" ~ zeo_section_name ~ ":ip}" }} +port = {{ next_port }} +path = ${directory:promises}/zeo-{{ storage_family }} + +{% if use_ipv6 -%} +{% do client_dict.__setitem__('server', '[' ~ ipv6 ~ ']:' ~ next_port) -%} +{% set tunnel_section_name = zeo_section_name ~ "-tunnel" -%} +[{{ tunnel_section_name }}] +< = ipv6toipv4-base +base-name = zeo-tunnel-{{ storage_family }} +ipv6-port = {{ next_port }} +ipv4-port = {{ next_port }} + +[{{ section(tunnel_section_name ~ "-promise") }}] +recipe = slapos.cookbook:check_port_listening +hostname = {{ "${" ~ tunnel_section_name ~ ":ipv6}" }} +port = {{ next_port }} +path = ${directory:promises}/zeo-tunnel-{{ storage_family }} +{% else -%} +{% do client_dict.__setitem__('server', ipv4 ~ ':' ~ next_port) -%} +{% endif -%} +{% set next_port = next_port + 1 -%} +{% endfor -%} + +{% if tidstorage_dict != None -%} +[tidstorage] +recipe = slapos.cookbook:tidstorage +known-tid-storage-identifier-dict = {{ dumps(known_tid_storage_identifier_dict) }} +configuration-path = ${directory:etc}/tidstorage.py +ip = {{ tidstorage_ip }} +port = {{ tidstorage_port }} +timestamp-file-path = {{ tidstorage_dict.get('timestamp-path', '%(backup)s/repozo_tidstorage_timestamp.log' % {'backup': default_tidstorage_timestamp_path}) }} +logfile-name = ${directory:log}/tidstorage.log +pidfile-name = ${directory:run}/tidstorage.pid +status-file = ${directory:tidstorage}/tidstorage.tid +tidstorage-repozo-binary = {{ bin_directory }}/tidstorage_repozo +tidstoraged-binary = {{ bin_directory }}/tidstoraged +repozo-binary = {{ bin_directory }}/repozo +tidstorage-wrapper = ${directory:services}/tidstoraged +repozo-wrapper = ${buildout:bin-directory}/tidstorage-repozo + +[{{ section("promise-tidstorage") }}] +recipe = slapos.cookbook:check_port_listening +hostname = ${tidstorage:ip} +port = ${tidstorage:port} +path = ${directory:promises}/tidstorage + +{% if use_ipv6 -%} +[tidstorage-tunnel] +< = ipv6toipv4-base +base-name = tidstorage-tunnel +ipv4-port = ${tidstorage:port} +ipv6-port = ${tidstorage:port} + +[{{ section("promise-tidstorage-tunnel") }}] +recipe = slapos.cookbook:check_port_listening +hostname = ${tidstorage-tunnel:ipv6} +port = ${tidstorage-tunnel:ipv6-port} +path = ${directory:promises}/tidstorage +{% endif -%} + +[{{ section("cron-entry-tidstorage-backup") }}] +# TODO: +# - configurable periodicity +# - configurable full/incremental +# - configurable retention +recipe = slapos.cookbook:cron.d +cron-entries = ${cron:cron-entries} +name = tidstorage +frequency = 0 0 * * * +command = ${tidstorage:repozo-wrapper} + +[{{ section("logrotate-tidstorage") }}] +< = logrotate-base +name = tidstorage +log = ${tidstorage:logfile-name} +post = {{ bin_directory }}/killpidfromfile ${tidstorage:pidfile-name} SIGHUP +{% else -%} +[tidstorage] +ip = +port = + +[tidstorage-tunnel] +ipv6 = +ipv6-port = +{% endif -%} + +[publish] +recipe = slapos.cookbook:publish.serialised +zodb-storage-type = zeoclient +zodb-dict = {{ dumps(zodb_dict) }} +{% if use_ipv6 -%} +tidstorage-ip = ${tidstorage-tunnel:ipv6} +tidstorage-port = ${tidstorage-tunnel:ipv6-port} +{% else -%} +tidstorage-ip = ${tidstorage:ip} +tidstorage-port = ${tidstorage:port} +{% endif -%} + +[directory] +recipe = slapos.cookbook:mkdirectory +etc = ${buildout:directory}/etc +services = ${:etc}/run +promises = ${:etc}/promise +var = ${buildout:directory}/var +log = ${:var}/log +run = ${:var}/run +backup-zodb = {{ zodb_backup_path }} +zodb = {{ default_zodb_path }} +tidstorage = {{ default_tidstorage_timestamp_path }} + +[buildout] +extends = + {{ parameter_dict['instance-logrotate-cfg'] }} +parts += + {{ part_list | join('\n ') }} + publish +eggs-directory = {{ eggs_directory }} +develop-eggs-directory = {{ develop_eggs_directory }} +offline = true + +{% endif %} diff --git a/stack/erp5/instance-zope.cfg.in b/stack/erp5/instance-zope.cfg.in index 1d6ee1b62..d8453e728 100644 --- a/stack/erp5/instance-zope.cfg.in +++ b/stack/erp5/instance-zope.cfg.in @@ -21,7 +21,7 @@ offline = true {% if 'frontend-software-url' in slapparameter_dict -%} [request-slave-frontend] -recipe = slapos.cookbook:request +recipe = slapos.cookbook:request.serialised software-url = {{ slapparameter_dict['frontend-software-url'] }} software-type = {{ slapparameter_dict.get('frontend-software-type', 'RootSoftwareInstance') }} sla = instance_guid @@ -47,14 +47,14 @@ config-{{ name }} = {{ value }} return = site_url [publish-apache-zope-backend-connection-string] -recipe = slapos.cookbook:publish +recipe = slapos.cookbook:publish.serialised url = ${request-slave-frontend:connection-site_url} login = ${zope-instance:user} password = ${zope-instance:password} url-deadlock = ${:url}/${zope-instance:deadlock-path}?${zope-instance:deadlock-password} {% else %} [publish-apache-zope-backend-connection-string] -recipe = slapos.cookbook:publish +recipe = slapos.cookbook:publish.serialised url = https://${zope-instance:user}:${zope-instance:password}@[${apache-zope-backend-instance:ip}]:${apache-zope-backend-instance:port} url-deadlock = ${:url}/${zope-instance:deadlock-path}?${zope-instance:deadlock-password} {% endif %} diff --git a/stack/erp5/instance.cfg.in b/stack/erp5/instance.cfg.in index 945345284..1f058a7e5 100644 --- a/stack/erp5/instance.cfg.in +++ b/stack/erp5/instance.cfg.in @@ -6,8 +6,8 @@ eggs-directory = {{ eggs_directory }} develop-eggs-directory = {{ develop_eggs_directory }} offline = true -[slap-parameters] -recipe = slapos.cookbook:slapconfiguration +[slap-configuration] +recipe = slapos.cookbook:slapconfiguration.serialised computer = ${slap-connection:computer-id} partition = ${slap-connection:partition-id} url = ${slap-connection:server-url} @@ -19,53 +19,15 @@ recipe = slapos.recipe.template:jinja2 rendered = ${buildout:parts-directory}/${:_buildout_section_name_}/${:filename} extra-context = context = + key ipv4_set slap-configuration:ipv4 + key ipv6_set slap-configuration:ipv6 key eggs_directory buildout:eggs-directory key develop_eggs_directory buildout:develop-eggs-directory - key slap_software_type slap-parameters:slap-software-type - key slapparameter_dict slap-parameters:configuration + key slap_software_type slap-configuration:slap-software-type + key slapparameter_dict slap-configuration:configuration + key computer_id slap-configuration:computer ${:extra-context} -[dynamic-template-tidstorage-parameters] -apache = {{ apache_location }} -haproxy = {{ haproxy_location }} -dcron = {{ dcron_location }} -logrotate = {{ logrotate_location }} -gzip = {{ gzip_location }} -openssl = {{ openssl_location }} -coreutils = {{ coreutils_location }} -git = {{ git_location }} -graphviz = {{ graphviz_location }} -grep = {{ grep_location }} -imagemagick = {{ imagemagick_location }} -librsvg = {{ librsvg_location }} -mariadb = {{ mariadb_location }} -sed = {{ sed_location }} -tesseract = {{ tesseract_location }} -w3m = {{ w3m_location }} -aspell = {{ aspell_location }} -poppler = {{ poppler_location }} -dmtx-utils = {{ dmtx_utils_location }} -buildout-bin-directory = {{ buildout_bin_directory }} -zope-conf-template = {{ template_zope_conf }} -jsl = {{ jsl_location }} - -[dynamic-template-tidstorage] -< = jinja2-template-base -template = {{ template_tidstorage }} -filename = instance-tidstorage.cfg -extensions = jinja2.ext.do -extra-context = - section parameter_dict dynamic-template-tidstorage-parameters - import json_module json - raw zope_port_base 12000 - raw zeo_port_base 15000 - raw haproxy_port_base 11000 - raw apache_port_base 10000 - key ipv4_set slap-parameters:ipv4 - key ipv6_set slap-parameters:ipv6 -# Must match the key id in [switch-softwaretype] which uses this section. - raw software_type tidstorage - [dynamic-template-cloudooo-parameters] file = {{ file_location }} fontconfig = {{ fontconfig_location }} @@ -83,6 +45,8 @@ zlib = {{ zlib_location }} libreoffice-bin = {{ libreoffice_bin_location }} fonts = {{ fonts_location }} buildout-bin-directory = {{ buildout_bin_directory }} +6tunnel = {{ sixtunnel_location }} +dash = {{ dash_location }} [dynamic-template-cloudooo] < = jinja2-template-base @@ -90,7 +54,6 @@ template = {{ template_cloudooo }} filename = instance-cloudoo.cfg extra-context = section parameter_dict dynamic-template-cloudooo-parameters - import json_module json # Must match the key id in [switch-softwaretype] which uses this section. raw software_type cloudooo @@ -188,37 +151,139 @@ extra-context = key template_zope dynamic-template-zope:rendered key local_bt5_repository template-erp5-single-parameters:local_bt5_repository -[dynamic-template-cluster-parameters] +[dynamic-template-cluster-erp5-parameters] local-bt5-repository = {{ local_bt5_repository }} -[dynamic-template-cluster] +[dynamic-template-cluster-erp5] < = jinja2-template-base template = {{ template_erp5_cluster }} -filename = instance-erp5-cluster.cfg +filename = instance-cluster-erp5.cfg +extensions = jinja2.ext.do extra-context = - key local_bt5_repository dynamic-template-cluster-parameters:local-bt5-repository + key local_bt5_repository dynamic-template-cluster-erp5-parameters:local-bt5-repository # Must match the key id in [switch-softwaretype] which uses this section. -# XXX: it is dangerous to use buildout-style expansion in jinja context declaration - raw software_type ${:software-type} -software-type = cluster + raw software_type cluster -# BBB: use "cluster" software type instead -[dynamic-template-production] -< = dynamic-template-cluster -software-type = production +[dynamic-template-balancer-parameters] +apache = {{ apache_location }} +openssl = {{ openssl_location }} +haproxy = {{ haproxy_location }} +instance-logrotate-cfg = {{ template_logrotate_base }} +bin-directory = {{ bin_directory }} +6tunnel = {{ sixtunnel_location }} +dash = {{ dash_location }} + +[dynamic-template-balancer] +< = jinja2-template-base +template = {{ template_balancer }} +extensions = jinja2.ext.do +filename = instance-balancer.cfg +extra-context = + section parameter_dict dynamic-template-balancer-parameters +# Must match the key id in [switch-softwaretype] which uses this section. + raw software_type balancer + +[dynamic-template-zeo-parameters] +buildout-bin-directory = {{ buildout_bin_directory }} +instance-logrotate-cfg = {{ template_logrotate_base }} +6tunnel = {{ sixtunnel_location }} +dash = {{ dash_location }} + +[dynamic-template-zeo] +< = jinja2-template-base +template = {{ template_zeo }} +filename = instance-zeo.cfg +extensions = jinja2.ext.do +extra-context = + key buildout_directory buildout:directory + section parameter_dict dynamic-template-zeo-parameters + import json_module json +# Must match the key id in [switch-softwaretype] which uses this section. + raw software_type zodb-zeo + +[dynamic-template-cluster-zope-parameters] +zope-conf-template = {{ template_zope_conf }} +instance-logrotate-cfg = {{ template_logrotate_base }} +git = {{ git_location }} +graphviz = {{ graphviz_location }} +buildout-bin-directory = {{ buildout_bin_directory }} +coreutils = {{ coreutils_location }} +grep = {{ grep_location }} +imagemagick = {{ imagemagick_location }} +sed = {{ sed_location }} +tesseract = {{ tesseract_location }} +w3m = {{ w3m_location }} +poppler = {{ poppler_location }} +dmtx-utils = {{ dmtx_utils_location }} +openssl = {{ openssl_location }} +6tunnel = {{ sixtunnel_location }} +dash = {{ dash_location }} +jsl = {{ jsl_location }} + +[dynamic-template-cluster-zope] +< = jinja2-template-base +template = {{ template_cluster_zope }} +filename = instance-cluster-zope.cfg +extensions = jinja2.ext.do +extra-context = + key buildout_directory buildout:directory + section parameter_dict dynamic-template-cluster-zope-parameters +# Must match the key id in [switch-softwaretype] which uses this section. + raw software_type cluster-zope + +[dynamic-template-kumofs-parameters] +dash-location = {{ dash_location }} +dcron-location = {{ dcron_location }} +gzip-location = {{ gzip_location }} +kumo-location = {{ kumo_location }} +logrotate-location = {{ logrotate_location }} + +[dynamic-template-kumofs] +< = jinja2-template-base +template = {{ template_kumofs }} +filename = instance-kumofs.cfg +extra-context = + section parameter_dict dynamic-template-kumofs-parameters +# Must match the key id in [switch-softwaretype] which uses this section. + raw software_type kumofs + +[dynamic-template-mariadb-parameters] +coreutils-location = {{ coreutils_location }} +dcron-location = {{ dcron_location }} +findutils-location = {{ findutils_location }} +gettext-location = {{ gettext_location }} +grep-location = {{ grep_location }} +gzip-location = {{ gzip_location }} +logrotate-location = {{ logrotate_location }} +mariadb-location = {{ mariadb_location }} +perl-location = {{ perl_location }} +perl-siteprefix = {{ perl_siteprefix }} +sed-location = {{ sed_location }} +template-my-cnf = {{ template_my_cnf }} +template-mariadb-initial-setup = {{ template_mariadb_initial_setup }} + +[dynamic-template-mariadb] +< = jinja2-template-base +template = {{ template_mariadb }} +filename = instance-mariadb.cfg +extensions = jinja2.ext.do +extra-context = + section parameter_dict dynamic-template-mariadb-parameters +# Must match the key id in [switch-softwaretype] which uses this section. + raw software_type mariadb [switch-softwaretype] recipe = slapos.cookbook:softwaretype default = ${template-erp5-single:rendered} -production = ${dynamic-template-production:rendered} -cluster = ${dynamic-template-cluster:rendered} -kumofs = {{ template_kumofs }} -memcached = {{ template_memcached }} +cluster = ${dynamic-template-cluster-erp5:rendered} +kumofs = ${dynamic-template-kumofs:rendered} cloudooo = ${dynamic-template-cloudooo:rendered} zope = ${dynamic-template-zope:rendered} -mariadb = {{ template_mariadb }} -tidstorage = ${dynamic-template-tidstorage:rendered} +mariadb = ${dynamic-template-mariadb:rendered} varnish = ${dynamic-template-varnish:rendered} +balancer = ${dynamic-template-balancer:rendered} +zodb-zeo = ${dynamic-template-zeo:rendered} +cluster-zope = ${dynamic-template-cluster-zope:rendered} [slap-connection] # part to migrate to new - separated words diff --git a/stack/erp5/mariadb_initial_setup.sql.in b/stack/erp5/mariadb_initial_setup.sql.in new file mode 100644 index 000000000..96f666072 --- /dev/null +++ b/stack/erp5/mariadb_initial_setup.sql.in @@ -0,0 +1,16 @@ +USE mysql; +DROP FUNCTION IF EXISTS last_insert_grn_id; +{% set mroonga = parameter_dict.get('mroonga', 'ha_mroonga.so') -%} +{% if mroonga %}CREATE FUNCTION last_insert_grn_id RETURNS INTEGER SONAME '{{ mroonga }}';{% endif %} +DROP FUNCTION IF EXISTS sphinx_snippets; +#CREATE FUNCTION sphinx_snippets RETURNS STRING SONAME 'ha_sphinx.so'; + +{% macro database(name, user, password) -%} +CREATE DATABASE IF NOT EXISTS {{ name }}; +GRANT ALL PRIVILEGES ON {{ name }}.* TO {{ user }}@'%' IDENTIFIED BY '{{ password }}'; +GRANT ALL PRIVILEGES ON {{ name }}.* TO {{ user }}@'localhost' IDENTIFIED BY '{{ password }}'; +{% endmacro -%} + +{% for entry in parameter_dict['database-list'] -%} +{{ database(entry['name'], entry['user'], entry['password']) }} +{% endfor -%} diff --git a/stack/erp5/my.cnf.in b/stack/erp5/my.cnf.in new file mode 100644 index 000000000..049094082 --- /dev/null +++ b/stack/erp5/my.cnf.in @@ -0,0 +1,79 @@ +{% set socket = parameter_dict['socket'] -%} +# ERP5 buildout my.cnf template based on my-huge.cnf shipped with mysql +# The MySQL server +[mysqld] +# ERP5 by default requires InnoDB storage. MySQL by default fallbacks to using +# different engine, like MyISAM. Such behaviour generates problems only, when +# tables requested as InnoDB are silently created with MyISAM engine. +# +# Loud fail is really required in such case. +sql_mode="NO_ENGINE_SUBSTITUTION" + +# Workaround for https://bugs.launchpad.net/maria/+bug/985828 +# that causes wrong result in Resource_zGetInventoryList etc. +optimizer_switch = derived_merge=off + +skip_show_database +{% set ip = parameter_dict.get('ip') -%} +{% if ip -%} +bind_address = {{ ip }} +port = {{ parameter_dict['port'] }} +{% else -%} +skip_networking +{% endif -%} +socket = {{ socket }} +datadir = {{ parameter_dict['data-directory'] }} +pid_file = {{ parameter_dict['pid-file'] }} +log_error = {{ parameter_dict['error-log'] }} +slow_query_log +slow_query_log_file = {{ parameter_dict['slow-query-log'] }} +long_query_time = 1 +max_allowed_packet = 128M +query_cache_size = 32M + +plugin_load = ha_mroonga.so;handlersocket.so + +# By default only 100 connections are allowed, when using zeo +# we may have much more connections +max_connections = 1000 + +{% set innodb_buffer_pool_size = parameter_dict.get('innodb-buffer-pool-size') -%} +{% if innodb_buffer_pool_size %}innodb_buffer_pool_size = {{ innodb_buffer_pool_size }}{% endif %} +{% set innodb_log_file_size = parameter_dict.get('innodb-log-file-size') -%} +{% if innodb_log_file_size %} innodb_log_file_size = {{ innodb_log_file_size }}{% endif %} +{% set innodb_log_buffer_size = parameter_dict.get('innodb-log-buffer-size') -%} +{% if innodb_log_buffer_size %} innodb_log_buffer_size = {{ innodb_log_buffer_size }}{% endif %} + +# very important to allow parallel indexing +# Note: this is compatible with binlog-based incremental backups, because ERP5 +# doesn't use "insert ... select" (in any number of queries) pattern. +innodb_locks_unsafe_for_binlog = 1 + +{% set log_bin = parameter_dict.get('binlog-path') -%} +{% if log_bin -%} +log_bin = {{ log_bin }} +{% set binlog_expire_days = parameter_dict.get('binlog-expire-days') -%} +{% if binlog_expire_days > 0 %}expire_logs_days = {{ binlog_expire_days }}{% endif %} +{% endif -%} + +{# Note: strictly check equality to one, as a last line of defense against users not reading the doc. -#} +{% if parameter_dict.get('relaxed_writes') == 1 -%} +innodb_flush_log_at_trx_commit = 0 +innodb_flush_method = nosync +innodb_doublewrite = 0 +sync_frm = 0 +{% endif -%} + +# Force utf8 usage +collation_server = utf8_unicode_ci +character_set_server = utf8 +skip_character_set_client_handshake + +[client] +socket = {{ socket }} + +[mysql] +no_auto_rehash + +[mysqlhotcopy] +interactive_timeout diff --git a/stack/erp5/zope.conf.in b/stack/erp5/zope.conf.in index 8094bb435..153fffe1a 100644 --- a/stack/erp5/zope.conf.in +++ b/stack/erp5/zope.conf.in @@ -43,7 +43,7 @@ products {{ instance_products }} </product-config> {% endif -%} -{% set timerserver_interval = parameter_dict.get('timerserver-interval', '0') | int -%} +{% set timerserver_interval = parameter_dict.get('timerserver-interval', 0) -%} {% if timerserver_interval -%} %import timerserver <timer-server> @@ -83,7 +83,7 @@ products {{ instance_products }} {% for db_name, mount_point, cache_size, storage_type, storage_dict in parameter_dict['zodb-list'] -%} <zodb_db {{ db_name }}> -{%- if cache_size %} +{%- if cache_size >= 0 %} cache-size {{ cache_size }} {%- endif %} mount-point {{ mount_point }} -- 2.30.9