repman: fixup, update configs to use more tags by default

software/repman/buildout.hash.cfg

@@ -14,11 +14,11 @@
 # not need these here).
 [instance.cfg]
 filename = instance.cfg.in
-md5sum = da50540b1c0fc69ffbf8f6e345a3baad
+md5sum = b41f521b5f7980c64260ed0e5c494450
 
 [instance-repman.cfg]
 _update_hash_filename_ = instance-repman.cfg.jinja2.in
-md5sum = 2515f1a13030b05d0d018b3ccb2b2eb5
+md5sum = 4ddb88852c1bb89d64ecca2f2a7ada0a
 
 [config-toml.in]
 _update_hash_filename_ = templates/config.toml.in
@@ -34,15 +34,15 @@ md5sum = 0eeb24c6aa0760f0d33c4cc2828ddf30
 
 [template-mariadb.cfg]
 _update_hash_filename_ = instance-mariadb.cfg.jinja2.in
-md5sum = d7206889f165dbedc5b4ace98e4105d0
+md5sum = 44c4b22c37adc6b7add3581aee7bbadf
 
 [template-my-cnf]
 _update_hash_filename_ = templates/my.cnf.in
-md5sum = d2b53cb8ee245c79939c6eab3bde4a0b
+md5sum = f3661b788099bb31d71ba6e7d36836d9
 
 [template-mariadb-initial-setup]
 _update_hash_filename_ = templates/mariadb_initial_setup.sql.in
-md5sum = c31618ef115ecd72b30ae59d0440a3ec
+md5sum = 9be53e2e92333b93e92556b8a01d9c42
 
 [mariadb-init-root-sql]
 _update_hash_filename_ = templates/mariadb_init_root.sql.in
@@ -54,11 +54,11 @@ md5sum = 3ba02b99c3fcbc24b3659917980b310f
 
 [repman-manager-sh.in]
 _update_hash_filename_ = templates/repman-manager.sh.in
-md5sum = 99324b56192003254081ef336dcee94c
+md5sum = 96fcd571ebf5546957a9a8677a87c129
 
 [dbjobs-in]
 _update_hash_filename_ = templates/dbjobs.in
-md5sum = ccdf01a2af32936ddfb58f7bab6154c5
+md5sum = f273ed3cb2173f36ff8230b04424d4e3
 
 [mysqld-need-start.sh.in]
 _update_hash_filename_ = templates/mysqld-need-start.sh.in

software/repman/instance-mariadb.cfg.jinja2.in

@@ -26,17 +26,8 @@ receiver-port = ${dbjob-parameter:socat-port}
 [publish-early]
 recipe = slapos.cookbook:publish-early
 -init =
-  server-id gen-server-id:value
-{%- set server_id = slapparameter_dict.get('server-id') %}
-{%- if server_id %}
-server-id = {{ dumps(server_id) }}
-{%- endif %}
-database-list = {{ dumps(database_list) }}
 
-[gen-server-id]
-recipe = slapos.cookbook:random.integer
-minimum = {{ dumps(1) }}
-maximum = {{ dumps(2**32 - 1) }}
+database-list = {{ dumps(database_list) }}
 
 [jinja2-template-base]
 recipe = slapos.recipe.template:jinja2
@@ -67,14 +58,14 @@ socket = ${directory:run}/mariadb.sock
 ip = {{ ip }}
 data-directory = ${directory:srv}/mariadb
 pid-file = ${directory:run}/mariadb.pid
-max-connection-count = {{ dumps(slapparameter_dict.get('max-connection-count', 1000)) }}
+plugin-directory = {{ dumps(parameter_dict['mroonga-mariadb-plugin-dir']) }}
+groonga-plugins-path = {{ parameter_dict['groonga-plugins-path'] }}
 innodb-buffer-pool-size = {{ dumps(slapparameter_dict.get('innodb-buffer-pool-size', 0)) }}
 innodb-buffer-pool-instances = {{ dumps(slapparameter_dict.get('innodb-buffer-pool-instances', 0)) }}
 innodb-log-file-size = {{ dumps(slapparameter_dict.get('innodb-log-file-size', 0)) }}
 innodb-file-per-table = {{ dumps(slapparameter_dict.get('innodb-file-per-table', 0)) }}
 innodb-log-buffer-size = {{ dumps(slapparameter_dict.get('innodb-log-buffer-size', 0)) }}
 relaxed-writes = {{ dumps(slapparameter_dict.get('relaxed-writes', False)) }}
-server-id = ${publish-early:server-id}
 ssl-crt = ${directory:mariadb-ssl}/crt.pem
 ssl-key = ${directory:mariadb-ssl}/key.pem
 ssl-ca-crt = ${certificate-authority:ca-dir}/cacert.pem
@@ -95,6 +86,7 @@ mode = 600
 [init-script-parameters]
 password = {{ slapparameter_dict['root-password'] }}
 database-list = {{ dumps(database_list) }}
+mroonga-mariadb-install-sql = {{ dumps(parameter_dict['mroonga-mariadb-install-sql']) }}
 root-user = repman
 
 [init-script]
@@ -136,6 +128,7 @@ context =
   key datadir my-cnf-parameters:data-directory
   key environ :environ
 environ =
+  GRN_PLUGINS_PATH='${my-cnf-parameters:groonga-plugins-path}'
   ODBCSYSINI='${directory:etc}'
   LD_LIBRARY_PATH=$${LD_LIBRARY_PATH:+$LD_LIBRARY_PATH:}'{{ parameter_dict['unixodbc-location'] }}/lib'
   {%- for variable in slapparameter_dict.get('environment-variables', ()) %}

software/repman/instance-repman-input-schema.json

@@ -87,7 +87,7 @@
 			"title": "Provisioning db tags",
 			"description": "Provisioning db tags",
 			"type": "string",
-			"default": "gtidstrict,bind,pkg,innodb,noquerycache,slow,pfs,linux,readonly,diskmonitor,sqlerror,compressbinlog,bm4ci,mroonga,utctime"
+			"default": "gtidstrict,bind,pkg,innodb,noquerycache,slow,pfs,linux,readonly,diskmonitor,sqlerror,compressbinlog,bm4ci,mroonga,utctime,readcommitted,nohandshake"
 		},
 		"http-session-lifetime": {
 			"title": "Web Session life time in s",

software/repman/instance-repman.cfg.jinja2.in

@@ -8,7 +8,7 @@
 {% set ip = (ipv6_set | list)[0] -%}
 {% set ipv4 = (ipv4_set | list)[0] -%}
 {% set cluster_list = [] -%}
-{% set tags = "gtidstrict,bind,pkg,innodb,noquerycache,slow,pfs,linux,readonly,diskmonitor,sqlerror,compressbinlog,bm4ci,mroonga,utctime" -%}
+{% set tags = "gtidstrict,bind,pkg,innodb,noquerycache,slow,pfs,linux,readonly,diskmonitor,sqlerror,compressbinlog,bm4ci,mroonga,utctime,readcommitted,nohandshake" -%}
 {% set frontend_parameter_dict = slapparameter_dict.get('slave-frontend', {}) -%}
 
 [directory]

software/repman/instance.cfg.in

@@ -91,6 +91,9 @@ unixodbc-location = {{ unixodbc_location }}
 curl-location = {{ curl_location }}
 dbjobs-template = {{ dbjobs_in }}
 socat-location = {{ socat_location }}
+mroonga-mariadb-install-sql = {{ mroonga_mariadb_install_sql }}
+mroonga-mariadb-plugin-dir = {{ mroonga_mariadb_plugin_dir }}
+groonga-plugins-path = {{ groonga_plugin_dir }}:{{ groonga_mysql_normalizer_plugin_dir }}
 
 [template-instance-mariadb.cfg]
 <= jinja2-template-base

software/repman/software.cfg

@@ -2,8 +2,6 @@
 
 extends =
   buildout.hash.cfg
-  ../neoppod/software-common.cfg
-  ../../stack/monitor/buildout.cfg
   ../../component/restic/buildout.cfg
   ../../component/replication-manager/buildout.cfg
   ../../component/mariadb/buildout.cfg
@@ -21,7 +19,7 @@ extends =
   ../../component/rsync/buildout.cfg
   ../../stack/supervisord/buildout.cfg
   ../../stack/monitor/buildout.cfg
-  ../../stack/slapos.cfg
+  ../neoppod/software-common.cfg
 
 parts =
   slapos-cookbook
@@ -58,6 +56,10 @@ context =
     key mariadb_link_binary template-mariadb.cfg:link-binary
     key mariadb_location mariadb:location
     key mysqld_start_template mysqld-need-start.sh.in:target
+    key mroonga_mariadb_install_sql mroonga-mariadb:install-sql
+    key mroonga_mariadb_plugin_dir mroonga-mariadb:plugin-dir
+    key groonga_plugin_dir groonga:groonga-plugin-dir
+    key groonga_mysql_normalizer_plugin_dir groonga-normalizer-mysql:groonga-plugin-dir
     key nginx_conf_in nginx.conf.in:target
     key nginx_location nginx:location
     key percona_toolkit_location percona-toolkit:location

software/repman/templates/dbjobs.in

@@ -77,7 +77,7 @@ do
       reseedmysqldump)
        echo "Waiting backup." >  {{ parameter_dict['log-dir'] }}/dbjob.out
        pauseJob
-       socat -u {{ listen }}:{{ parameter_dict['socat-port'] }},bind={{ parameter_dict['host'] }},reuseaddr STDOUT | gunzip | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$USER --init-command="set sql_log_bin=0" > {{ parameter_dict['log-dir'] }}/dbjob.out 2>&1
+       socat -u {{ listen }}:{{ parameter_dict['socat-port'] }},bind={{ parameter_dict['host'] }},reuseaddr STDOUT | gunzip | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$USER --init-command="reset master;set sql_log_bin=0" > {{ parameter_dict['log-dir'] }}/dbjob.out 2>&1
         mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }}  -e 'start slave;'
       ;;
       flashbackmysqldump)

software/repman/templates/mariadb_initial_setup.sql.in

+SET @@SESSION.SQL_LOG_BIN=0;
+
 USE mysql;
-DROP FUNCTION IF EXISTS last_insert_grn_id;
-DROP FUNCTION IF EXISTS mroonga_snippet;
-DROP FUNCTION IF EXISTS mroonga_command;
 {% set mroonga = parameter_dict.get('mroonga', 'ha_mroonga.so') -%}
-{% if mroonga %}CREATE FUNCTION last_insert_grn_id RETURNS INTEGER SONAME '{{ mroonga }}';
-CREATE FUNCTION mroonga_snippet RETURNS STRING SONAME '{{ mroonga }}';
-CREATE FUNCTION mroonga_command RETURNS STRING SONAME '{{ mroonga }}';{% endif %}
+{% if mroonga %}
+SOURCE {{ parameter_dict['mroonga-mariadb-install-sql'] }};
+{% endif %}
 DROP FUNCTION IF EXISTS sphinx_snippets;
 #CREATE FUNCTION sphinx_snippets RETURNS STRING SONAME 'ha_sphinx.so';
 

software/repman/templates/my.cnf.in

@@ -26,8 +26,10 @@ pid_file = {{ parameter_dict['pid-file'] }}
 # very important to allow parallel indexing
 # Note: this is compatible with binlog-based incremental backups, because ERP5
 # doesn't use "insert ... select" (in any number of queries) pattern.
-innodb_locks_unsafe_for_binlog = 1
+# innodb_locks_unsafe_for_binlog = 1
 
+#plugin_load = ha_mroonga
+plugin-dir = {{ parameter_dict['plugin-directory'] }}
 
 {% if 'ssl-key' in parameter_dict -%}
 ssl
@@ -44,8 +46,6 @@ ssl-cipher = {{ parameter_dict['ssl-cipher'] }}
 {%-  endif %}
 {%- endif %}
 
-#server_id = {{ parameter_dict['server-id'] }}
-
 
 # Some dangerous settings you may want to uncomment temporarily
 # if you only want performance or less disk access.
@@ -55,7 +55,7 @@ ssl-cipher = {{ parameter_dict['ssl-cipher'] }}
 {{x}}innodb_doublewrite = 0
 {{x}}sync_frm = 0
 
-skip_character_set_client_handshake
+# skip_character_set_client_handshake
 
 
 [client]

software/repman/templates/repman-manager.sh.in

@@ -8,7 +8,7 @@ curl () {
 
 wait_database () {
   NAME=$1
-  for retry in {1..5}; do
+  for retry in {1..10}; do
     echo ">> Wait until $NAME databases are ready...";
     CODE=$(curl -H "Authorization: Bearer ${TOKEN}" -o /dev/null -w "%{http_code}" {{ secure_url }}/api/clusters/$NAME/actions/waitdatabases);
     if [ $CODE -eq 504 ]; then