Mariadb in ERP5 SR test

Test Mariadb in ERP5 SR test and a few cleanups, mainly around mariadb:
 - We need a more "strict" promise than just checking if something is listening on the port
 - We don't need to wait so long between setup retries
 - We can use mroonga provided setup script instead of writing the setup ourselves.

See merge request !722

component/mariadb/buildout.cfg

@@ -104,6 +104,7 @@ configure-options =
   --with-mysql-config=${mariadb:location}/bin/mysql_config
   --disable-static
   --disable-document
+install-sql = @@LOCATION@@/share/mroonga/install.sql
 patch-options = -p1
 patches =
   ${:_profile_base_location_}/mroonga_boolean.patch#c818568fe35ca6a4298f18e575d962a0

slapos/recipe/generic_mysql/mysql.py

@@ -5,7 +5,7 @@ import sys
 import pytz
 
 def updateMysql(mysql_upgrade_binary, mysql_binary, mysql_script_file):
-  sleep = 30
+  sleep = 0
   with open(mysql_script_file) as script_file:
     mysql_script = script_file.read()
   mysql_list = mysql_binary, '-B'
@@ -43,6 +43,7 @@ def updateMysql(mysql_upgrade_binary, mysql_binary, mysql_script_file):
         break
       print 'SlapOS initialisation script succesfully applied on database.'
       return
+    sleep = max(sleep+1, 30)
     print 'Sleeping for %ss and retrying' % sleep
     sys.stdout.flush()
     sys.stderr.flush()

software/erp5/software-shared.cfg

-[buildout]
-extends = software.cfg
-shared-parts = /opt/slapgrid/shared-parts
-eggs-directory = /opt/slapgrid/shared-eggs
-abi-tag-eggs = true

software/erp5/test/setup.py

@@ -47,6 +47,7 @@ setup(name=name,
         'erp5.util',
         'psutil',
         'requests',
+        'mysqlclient',
         ],
       zip_safe=True,
       test_suite='test',

software/erp5/test/test/test_mariadb.py

+##############################################################################
+# coding: utf-8
+#
+# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import os
+import json
+import glob
+import urlparse
+import socket
+import time
+import contextlib
+import datetime
+
+import MySQLdb
+
+from . import ERP5InstanceTestCase
+from . import setUpModule
+setUpModule  # pyflakes
+
+
+class MariaDBTestCase(ERP5InstanceTestCase):
+  """Base test case for mariadb tests.
+  """
+  __partition_reference__ = 'm'
+  # XXX mariadb_update service sleeps for 30 seconds before retrying,
+  # so this takes a lot of time to instance
+  instance_max_retry = 60
+
+  @classmethod
+  def getInstanceSoftwareType(cls):
+    return "mariadb"
+
+  @classmethod
+  def _getInstanceParameterDict(cls):
+    return {
+        'tcpv4-port': 3306,
+        'max-connection-count': 5,
+        'max-slowqueries-threshold': 5,
+        'slowest-query-threshold': 10,
+        # XXX what is this ? should probably not be needed here
+        'name': cls.__name__,
+        'monitor-passwd': 'secret',
+        # XXX should probably not be needed here
+        'computer-memory-percent-threshold': 100,
+    }
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {'_': json.dumps(cls._getInstanceParameterDict())}
+
+  def getDatabaseConnection(self):
+    connection_parameter_dict = json.loads(
+        self.computer_partition.getConnectionParameterDict()['_'])
+    db_url = urlparse.urlparse(connection_parameter_dict['database-list'][0])
+    self.assertEqual('mysql', db_url.scheme)
+
+    self.assertTrue(db_url.path.startswith('/'))
+    database_name = db_url.path[1:]
+    return MySQLdb.connect(
+        user=db_url.username,
+        passwd=db_url.password,
+        host=db_url.hostname,
+        port=db_url.port,
+        db=database_name,
+    )
+
+
+class TestMariaDB(MariaDBTestCase):
+  def test_utf8_collation(self):
+    cnx = self.getDatabaseConnection()
+    with contextlib.closing(cnx):
+      cnx.query(
+          """
+          CREATE TABLE test_utf8_collation (
+            col1 CHAR(10)
+          )
+          """)
+
+      cnx.store_result()
+      cnx.query(
+          """
+          insert into test_utf8_collation values ("à"), ("あ")
+          """)
+      cnx.store_result()
+
+      cnx.query(
+          """
+          select * from test_utf8_collation where col1 = "a"
+          """)
+      self.assertEqual((('à',),), cnx.store_result().fetch_row(maxrows=2))
+
+
+class TestMroonga(MariaDBTestCase):
+  def test_mroonga_plugin_loaded(self):
+    cnx = self.getDatabaseConnection()
+    with contextlib.closing(cnx):
+      cnx.query("show plugins")
+      plugins = cnx.store_result().fetch_row(maxrows=1000)
+      self.assertIn(
+          ('Mroonga', 'ACTIVE', 'STORAGE ENGINE', 'ha_mroonga.so', 'GPL'),
+          plugins)
+
+  def test_mroonga_normalize_udf(self):
+    # example from https://mroonga.org/docs/reference/udf/mroonga_normalize.html#usage
+    cnx = self.getDatabaseConnection()
+    with contextlib.closing(cnx):
+      cnx.query(
+          """
+          SELECT mroonga_normalize("ABCDあぃうぇ㍑")
+          """)
+      self.assertEqual((('abcdあぃうぇリットル',),),
+                       cnx.store_result().fetch_row(maxrows=2))
+
+      if 0:
+        # this example fail with:
+        # OperationalError: (1123, "Can't initialize function 'mroonga_normalize'; mroonga_normalize(): nonexistent normalizer NormalizerMySQLUnicodeCIExceptKanaCI")
+        # same error on mroonga "official" docker images using mysql
+        # https://hub.docker.com/layers/groonga/mroonga/latest/images/sha256-e5a979801c95544ca3a1228d2c4d819820850e0162649553f2e94850e5e1c988?context=explore
+        # so it's probably OK to ignore
+        cnx.query(
+            """
+            SELECT mroonga_normalize("aBｃＤあぃウェ㍑", "NormalizerMySQLUnicodeCIExceptKanaCIKanaWithVoicedSoundMark")
+            """)
+        self.assertEqual((('ABCDあぃうぇ㍑',),),
+                         cnx.store_result().fetch_row(maxrows=2))
+
+  def test_mroonga_full_text_normalizer(self):
+    # example from https://mroonga.org//docs/tutorial/storage.html#how-to-specify-the-normalizer
+    cnx = self.getDatabaseConnection()
+    with contextlib.closing(cnx):
+
+      cnx.query("SET NAMES utf8")
+      cnx.store_result()
+      cnx.query(
+          """
+          CREATE TABLE diaries (
+            day DATE PRIMARY KEY,
+            content VARCHAR(64) NOT NULL,
+            FULLTEXT INDEX (content) COMMENT 'normalizer "NormalizerAuto"'
+          ) Engine=Mroonga DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci
+          """)
+      cnx.store_result()
+      cnx.query(
+          """INSERT INTO diaries VALUES ("2013-04-23", "ブラックコーヒーを飲んだ。")""")
+      cnx.store_result()
+      cnx.query(
+          """
+          SELECT *
+            FROM diaries
+           WHERE MATCH (content) AGAINST ("+ふらつく" IN BOOLEAN MODE)
+          """)
+      self.assertEqual((), cnx.store_result().fetch_row(maxrows=2))
+      cnx.query(
+          """
+          SELECT *
+            FROM diaries
+           WHERE MATCH (content) AGAINST ("+ﾌﾞﾗｯｸ" IN BOOLEAN MODE)
+          """)
+      self.assertEqual(
+          ((datetime.date(2013, 4, 23), 'ブラックコーヒーを飲んだ。'),),
+          cnx.store_result().fetch_row(maxrows=2),
+      )
+
+  def test_mroonga_full_text_stem(self):
+    # example from https://mroonga.org//docs/tutorial/storage.html#how-to-specify-the-token-filters
+    cnx = self.getDatabaseConnection()
+    with contextlib.closing(cnx):
+      cnx.query("SELECT mroonga_command('register token_filters/stem')")
+      self.assertEqual((('true',),), cnx.store_result().fetch_row(maxrows=2))
+      cnx.query(
+          """
+          CREATE TABLE memos (
+            id INT NOT NULL PRIMARY KEY,
+            content TEXT NOT NULL,
+            FULLTEXT INDEX (content) COMMENT 'normalizer "NormalizerAuto", token_filters "TokenFilterStem"'
+          ) Engine=Mroonga DEFAULT CHARSET=utf8
+          """)
+      cnx.store_result()
+      cnx.query(
+          """INSERT INTO memos VALUES (1, "I develop Groonga"), (2, "I'm developing Groonga"), (3, "I developed Groonga")"""
+      )
+      cnx.store_result()
+      cnx.query(
+          """
+          SELECT *
+            FROM memos
+           WHERE MATCH (content) AGAINST ("+develops" IN BOOLEAN MODE)
+          """)
+      self.assertEqual([
+          (1, "I develop Groonga"),
+          (2, "I'm developing Groonga"),
+          (3, "I developed Groonga"),
+      ], list(sorted(cnx.store_result().fetch_row(maxrows=4))))

software/slapos-sr-testing/software.cfg

@@ -6,6 +6,7 @@ extends =
   ../../component/git/buildout.cfg
   ../../component/pillow/buildout.cfg
   ../../component/python-cryptography/buildout.cfg
+  ../../component/python-mysqlclient/buildout.cfg
   ../../component/python-pynacl/buildout.cfg
   ../../component/quic_client-bin/buildout.cfg
   ../../component/python-backports-lzma/buildout.cfg
@@ -156,6 +157,7 @@ eggs =
   erp5.util
   ${python-pynacl:egg}
   ${python-cryptography:egg}
+  ${python-mysqlclient:egg}
   ${bcrypt:egg}
   slapos.libnetworkcache
   supervisor
@@ -282,3 +284,4 @@ mock = 2.0.0
 testfixtures = 6.11
 funcsigs = 1.0.2
 PyPDF2 = 1.26.0
+mysqlclient = 1.3.12
\ No newline at end of file

stack/erp5/buildout.cfg

@@ -203,6 +203,7 @@ context =
     key mariadb_resiliency_after_import_script mariadb-resiliency-after-import-script:target
     key mariadb_slow_query_report_script mariadb-slow-query-report-script:target
     key mariadb_start_clone_from_backup mariadb-start-clone-from-backup:target
+    key mroonga_mariadb_install_sql mroonga-mariadb:install-sql
     key matplotlibrc_location matplotlibrc:location
     key parts_directory buildout:parts-directory
     key openssl_location openssl:location

stack/erp5/buildout.hash.cfg

@@ -26,7 +26,7 @@ md5sum = d10b8e35b02b5391cf46bf0c7dbb1196
 
 [template-mariadb]
 filename = instance-mariadb.cfg.in
-md5sum = 1c332e1a3388258572fcc6528aca5725
+md5sum = 63bac9fc58537e55a6c8c42d0be54fbe
 
 [template-kumofs]
 filename = instance-kumofs.cfg.in
@@ -46,7 +46,7 @@ md5sum = 771bfd921aa5e59e1b4712ba113fa84b
 
 [template-mariadb-initial-setup]
 filename = mariadb_initial_setup.sql.in
-md5sum = dec33a617fa1b307c8ddb883efcfe3ce
+md5sum = 1102c3e37a5a2e8aa2d8a2607ab633c8
 
 [template-postfix]
 filename = instance-postfix.cfg.in
@@ -70,7 +70,7 @@ md5sum = cc19560b9400cecbd23064d55c501eec
 
 [template]
 filename = instance.cfg.in
-md5sum = 520b6bf3461dddc9c8b862e50b14465d
+md5sum = 1a218a5676208dda092e6045f25bf77f
 
 [monitor-template-dummy]
 filename = dummy.cfg

stack/erp5/instance-mariadb.cfg.in

@@ -160,6 +160,7 @@ context = section parameter_dict my-cnf-parameters
 
 [init-script-parameters]
 database-list = {{ dumps(database_list + test_database_list) }}
+mroonga-mariadb-install-sql = {{ dumps(parameter_dict['mroonga-mariadb-install-sql']) }}
 
 [init-script]
 < = jinja2-template-executable
@@ -344,7 +345,7 @@ config-command = "{{ parameter_dict["check-computer-memory-binary"] }}" -db ${mo
 <= monitor-promise-base
 module = check_command_execute
 name = mariadb.py
-config-command = "{{ parameter_dict['bin-directory'] }}/is-local-tcp-port-opened" "${my-cnf-parameters:ip}" "${my-cnf-parameters:port}"
+config-command = "${binary-wrap-mysql:wrapper-path}"  --execute ';' {% if database_list and database_list[0].get('user') %} --host="${my-cnf-parameters:ip}" --port="${my-cnf-parameters:port}" --user="{{ database_list[0]['user'] }}" --password="{{ database_list[0]['password'] }}" {% endif %}
 
 [monitor-instance-parameter]
 monitor-httpd-ipv6 = {{ (ipv6_set | list)[0] }}

stack/erp5/instance.cfg.in

@@ -151,6 +151,7 @@ mariadb-start-clone-from-backup = {{ mariadb_start_clone_from_backup }}
 promise-check-slow-queries-digest-result = {{ bin_directory }}/check-slow-queries-digest-result
 percona-tools-location = {{ percona_toolkit_location }}
 unixodbc-location = {{ unixodbc_location }}
+mroonga-mariadb-install-sql = {{ mroonga_mariadb_install_sql }}
 
 [dynamic-template-mariadb]
 <= jinja2-template-base

stack/erp5/mariadb_initial_setup.sql.in

 USE mysql;
-DROP FUNCTION IF EXISTS last_insert_grn_id;
-DROP FUNCTION IF EXISTS mroonga_snippet;
-DROP FUNCTION IF EXISTS mroonga_command;
 {% set mroonga = parameter_dict.get('mroonga', 'ha_mroonga.so') -%}
-{% if mroonga %}CREATE FUNCTION last_insert_grn_id RETURNS INTEGER SONAME '{{ mroonga }}';
-CREATE FUNCTION mroonga_snippet RETURNS STRING SONAME '{{ mroonga }}';
-CREATE FUNCTION mroonga_command RETURNS STRING SONAME '{{ mroonga }}';{% endif %}
+{% if mroonga %}
+SOURCE {{ parameter_dict['mroonga-mariadb-install-sql'] }};
+{% endif %}
 DROP FUNCTION IF EXISTS sphinx_snippets;
 #CREATE FUNCTION sphinx_snippets RETURNS STRING SONAME 'ha_sphinx.so';
 

stack/lamp/buildout.cfg

@@ -103,6 +103,7 @@ context =
     key mariadb_resiliency_after_import_script mariadb-resiliency-after-import-script:target
     key mariadb_slow_query_report_script mariadb-slow-query-report-script:target
     key mariadb_start_clone_from_backup mariadb-start-clone-from-backup:target
+    key mroonga_mariadb_install_sql mroonga-mariadb:install-sql
     key percona_toolkit_location percona-toolkit:location
     key template_php_ini template-php.ini:output
     key template_apache_conf template-apache.conf:output

stack/lamp/buildout.hash.cfg

@@ -14,7 +14,7 @@
 # not need these here).
 [instance]
 filename = instance.cfg.in
-md5sum = b4fbd7dc8d7fda7dbd1b80f3de273ecf
+md5sum = 92a93bb3b9f6663d967b0fb57244a1fa
 
 [instance-apache-php]
 filename = instance-apache-php.cfg.in

stack/lamp/instance.cfg.in

@@ -89,6 +89,7 @@ mariadb-start-clone-from-backup = {{ mariadb_start_clone_from_backup }}
 promise-check-slow-queries-digest-result = {{ bin_directory }}/check-slow-queries-digest-result
 percona-tools-location = {{ percona_toolkit_location }}
 unixodbc-location = {{ unixodbc_location }}
+mroonga-mariadb-install-sql = {{ mroonga_mariadb_install_sql }}
 check-computer-memory-binary = {{ bin_directory }}/check-computer-memory
 bin-directory = {{ bin_directory }}