Continue mariadb profilisation.

Turn on xtrabackup. Use maaakit.

Note: This is blind adaptation of current code, without any tests. This might
be incomplete code. Profiles *IS* incomplete, it will be fixed during runtime
checks.

slapos/recipe/erp5/__init__.py

@@ -56,7 +56,7 @@ class Recipe(BaseSlapRecipe):
 #     kumo_conf = self.installKumo(self.getLocalIPv4Address())
 #    conversion_server_conf = self.installConversionServer(
 #        self.getLocalIPv4Address(), 23000, 23060)
-    mysql_conf = self.installMysqlServer(self.getLocalIPv4Address(), 45678)
+#    mysql_conf = self.installMysqlServer(self.getLocalIPv4Address(), 45678)
     user, password = self.installERP5()
 
     if self.parameter_dict.get("slap_software_type", "").lower() == "cluster":
@@ -848,142 +848,3 @@ SSLCARevocationPath %(ca_crl)s"""
           ]))
     # Note: IPv6 is assumed always
     return 'https://[%(ip)s]:%(port)s' % apache_conf
-
-  def installMysqlServer(self, ip, port, database='erp5', user='user',
-      test_database='test_erp5', test_user='test_user', template_filename=None,
-      parallel_test_database_amount=100, mysql_conf=None, with_backup=True,
-      with_maatkit=True):
-    if mysql_conf is None:
-      mysql_conf = {}
-    backup_directory = self.createBackupDirectory('mysql')
-    if template_filename is None:
-      template_filename = self.getTemplateFilename('my.cnf.in')
-    error_log = os.path.join(self.log_directory, 'mysqld.log')
-    slow_query_log = os.path.join(self.log_directory, 'mysql-slow.log')
-    mysql_conf.update(
-        ip=ip,
-        data_directory=os.path.join(self.data_root_directory,
-          'mysql'),
-        tcp_port=port,
-        pid_file=os.path.join(self.run_directory, 'mysqld.pid'),
-        socket=os.path.join(self.run_directory, 'mysqld.sock'),
-        error_log=error_log,
-        slow_query_log=slow_query_log,
-        mysql_database=database,
-        mysql_user=user,
-        mysql_password=self.generatePassword(),
-        mysql_test_password=self.generatePassword(),
-        mysql_test_database=test_database,
-        mysql_test_user=test_user,
-        mysql_parallel_test_dict=[
-            ('test_%i' % x,)*2 + (self.generatePassword(),) \
-                 for x in xrange(0,parallel_test_database_amount)],
-    )
-    self.registerLogRotation('mysql', [error_log, slow_query_log],
-        '%(mysql_binary)s --no-defaults -B --user=root '
-        '--socket=%(mysql_socket)s -e "FLUSH LOGS"' % dict(
-          mysql_binary=self.options['mysql_binary'],
-          mysql_socket=mysql_conf['socket']))
-    self._createDirectory(mysql_conf['data_directory'])
-
-    mysql_conf_path = self.createConfigurationFile("my.cnf",
-        self.substituteTemplate(template_filename,
-          mysql_conf))
-
-    mysql_script_list = []
-    for x_database, x_user, x_password in \
-          [(mysql_conf['mysql_database'],
-            mysql_conf['mysql_user'],
-            mysql_conf['mysql_password']),
-           (mysql_conf['mysql_test_database'],
-            mysql_conf['mysql_test_user'],
-            mysql_conf['mysql_test_password']),
-          ] + mysql_conf['mysql_parallel_test_dict']:
-      mysql_script_list.append(pkg_resources.resource_string(__name__,
-                     'template/initmysql.sql.in') % {
-                        'mysql_database': x_database,
-                        'mysql_user': x_user,
-                        'mysql_password': x_password})
-    mysql_script_list.append('EXIT')
-    mysql_script = '\n'.join(mysql_script_list)
-    self.path_list.extend(zc.buildout.easy_install.scripts([('mysql_update',
-      __name__ + '.mysql', 'updateMysql')], self.ws,
-      sys.executable, self.wrapper_directory, arguments=[dict(
-        mysql_script=mysql_script,
-        mysql_binary=self.options['mysql_binary'].strip(),
-        mysql_upgrade_binary=self.options['mysql_upgrade_binary'].strip(),
-        socket=mysql_conf['socket'],
-        )]))
-    self.path_list.extend(zc.buildout.easy_install.scripts([('mysqld',
-      __name__ + '.mysql', 'runMysql')], self.ws,
-        sys.executable, self.wrapper_directory, arguments=[dict(
-        mysql_install_binary=self.options['mysql_install_binary'].strip(),
-        mysqld_binary=self.options['mysqld_binary'].strip(),
-        data_directory=mysql_conf['data_directory'].strip(),
-        mysql_binary=self.options['mysql_binary'].strip(),
-        socket=mysql_conf['socket'].strip(),
-        configuration_file=mysql_conf_path,
-       )]))
-    self.path_list.extend([mysql_conf_path])
-
-    if with_backup:
-      # backup configuration
-      backup_directory = self.createBackupDirectory('mysql')
-      full_backup = os.path.join(backup_directory, 'full')
-      incremental_backup = os.path.join(backup_directory, 'incremental')
-      self._createDirectory(full_backup)
-      self._createDirectory(incremental_backup)
-      innobackupex_argument_list = [self.options['perl_binary'],
-          self.options['innobackupex_binary'],
-          '--defaults-file=%s' % mysql_conf_path,
-          '--socket=%s' %mysql_conf['socket'].strip(), '--user=root',
-          '--ibbackup=%s'% self.options['xtrabackup_binary']]
-      environment = dict(PATH='%s' % self.bin_directory)
-      innobackupex_incremental = zc.buildout.easy_install.scripts([(
-        'innobackupex_incremental','slapos.recipe.librecipe.execute', 'executee')],
-        self.ws, sys.executable, self.bin_directory, arguments=[
-          innobackupex_argument_list + ['--incremental'],
-          environment])[0]
-      self.path_list.append(innobackupex_incremental)
-      innobackupex_full = zc.buildout.easy_install.scripts([('innobackupex_full',
-       'slapos.recipe.librecipe.execute', 'executee')], self.ws,
-        sys.executable, self.bin_directory, arguments=[
-          innobackupex_argument_list,
-          environment])[0]
-      self.path_list.append(innobackupex_full)
-      backup_controller = zc.buildout.easy_install.scripts([
-        ('innobackupex_controller', __name__ + '.innobackupex', 'controller')],
-        self.ws, sys.executable, self.bin_directory,
-        arguments=[innobackupex_incremental, innobackupex_full, full_backup,
-          incremental_backup])[0]
-      self.path_list.append(backup_controller)
-      mysql_backup_cron = os.path.join(self.cron_d, 'mysql_backup')
-      open(mysql_backup_cron, 'w').write('0 0 * * * ' + backup_controller)
-      self.path_list.append(mysql_backup_cron)
-
-    if with_maatkit:
-      # maatkit installation
-      for mk_script_name in (
-          'mk-variable-advisor',
-          'mk-table-usage',
-          'mk-visual-explain',
-          'mk-config-diff',
-          'mk-deadlock-logger',
-          'mk-error-log',
-          'mk-index-usage',
-          'mk-query-advisor',
-          ):
-        mk_argument_list = [self.options['perl_binary'],
-            self.options['%s_binary' % mk_script_name],
-            '--defaults-file=%s' % mysql_conf_path,
-            '--socket=%s' %mysql_conf['socket'].strip(), '--user=root',
-            ]
-        environment = dict(PATH='%s' % self.bin_directory)
-        mk_exe = zc.buildout.easy_install.scripts([(
-          mk_script_name,'slapos.recipe.librecipe.execute', 'executee')],
-          self.ws, sys.executable, self.bin_directory, arguments=[
-            mk_argument_list, environment])[0]
-        self.path_list.append(mk_exe)
-
-    # The return could be more explicit database, user ...
-    return mysql_conf

slapos/recipe/generic_mysql/__init__.py

@@ -31,6 +31,9 @@ class Recipe(GenericBaseRecipe):
 
   def _options(self, options):
     options['password'] = self.generatePassword()
+    options['test_password'] = self.generatePassword()
+    for x in xrange(0, int(options['parallel-test-database-amount'])):
+      options['test_password_%s' % x] = self.generatePassword()
 
   def install(self):
     path_list = []
@@ -48,6 +51,9 @@ class Recipe(GenericBaseRecipe):
         mysql_database=self.options['database'],
         mysql_user=self.options['user'],
         mysql_password=self.options['password'],
+        mysql_test_database=self.options['test-database'],
+        mysql_test_user=self.options['test-user'],
+        mysql_test_password=self.options['test-password'],
     )
 
     mysql_binary = self.options['mysql-binary']
@@ -68,15 +74,34 @@ class Recipe(GenericBaseRecipe):
 
     mysql_script_list = []
 
-    init_script = self.substituteTemplate(
+    # real database
+    mysql_script_list.append(self.substituteTemplate(
       self.getTemplateFilename('initmysql.sql.in'),
       {
         'mysql_database': mysql_conf['mysql_database'],
         'mysql_user': mysql_conf['mysql_user'],
         'mysql_password': mysql_conf['mysql_password']
       }
-    )
-    mysql_script_list.append(init_script)
+    ))
+    # default test database
+    mysql_script_list.append(self.substituteTemplate(
+      self.getTemplateFilename('initmysql.sql.in'),
+      {
+        'mysql_database': mysql_conf['mysql_test_database'],
+        'mysql_user': mysql_conf['mysql_test_user'],
+        'mysql_password': mysql_conf['mysql_test_password']
+      }
+    ))
+    # parallel test databases
+    for x in xrange(0, int(self.options['parallel-test-database-amount'])):
+      mysql_script_list.append(self.substituteTemplate(
+        self.getTemplateFilename('initmysql.sql.in'),
+        {
+          'mysql_database': self.options['mysql-test-database-base'] + '_%s' % x,
+          'mysql_user': self.options['mysql-test-user-base'] + '_%s' % x,
+          'mysql_password': self.options['test-password-%s' % x]
+        }
+      ))
     mysql_script_list.append('EXIT')
     mysql_script = '\n'.join(mysql_script_list)
 
@@ -107,5 +132,39 @@ class Recipe(GenericBaseRecipe):
        )
     )
     path_list.append(mysqld)
+    # backup configuration
+    full_backup = self.options['full-backup-path']
+    incremental_backup = self.options['incremental-backup-path']
+    innobackupex_argument_list = [self.options['perl-binary'],
+        self.options['innobackupex-binary'],
+        '--defaults-file=%s' % mysql_conf_file,
+        '--socket=%s' %mysql_conf['socket'].strip(), '--user=root',
+        '--ibbackup=%s'% self.options['xtrabackup-binary']]
+    environment = dict(PATH='%s' % self.bin_directory)
+    innobackupex_incremental = self.createPythonScript(self.options['innobackupex-incremental'], 'slapos.recipe.librecipe.execute.executee', [innobackupex_argument_list + ['--incremental'], environment])
+    path_list.append(innobackupex_incremental)
+    innobackupex_full = self.createPythonScript(self.options['innobackupex-full'], 'slapos.recipe.librecipe.execute.executee', [innobackupex_argument_list, environment])
+    path_list.append(innobackupex_full)
+    backup_controller = self.createPythonScript(self.options['innobackupex-controller'], __name__ + '.innobackupex.controller', [innobackupex_incremental, innobackupex_full, full_backup, incremental_backup])
+    path_list.append(backup_controller)
+    # maatkit installation
+    for mk_script_name in (
+        'mk-variable-advisor',
+        'mk-table-usage',
+        'mk-visual-explain',
+        'mk-config-diff',
+        'mk-deadlock-logger',
+        'mk-error-log',
+        'mk-index-usage',
+        'mk-query-advisor',
+        ):
+      mk_argument_list = [self.options['perl_binary'],
+          self.options['%s_binary' % mk_script_name],
+          '--defaults-file=%s' % mysql_conf_file,
+          '--socket=%s' %mysql_conf['socket'].strip(), '--user=root',
+          ]
+      environment = dict(PATH='%s' % self.bin_directory)
+      mk_exe = self.createPythonScript(os.path.join(self.bin_directory, 'mk_script_name,'), 'slapos.recipe.librecipe.execute.executee', [mk_argument_list, environment])
+      path_list.append(mk_exe)
 
     return path_list

software/erp5/instance-mariadb.cfg

@@ -6,20 +6,27 @@ parts =
   logrotate-entry-mariadb
   cron
   cron-entry-logrotate
+  cron-entry-mariadb-backup
 
 eggs-directory = ${buildout:eggs-directory}
 develop-eggs-directory = ${buildout:develop-eggs-directory}
 offline = true
 
-[url]
+[publish-mariadb-url]
 recipe = slapos.cookbook:publishurl
 url = mysqls://$${mariadb:user}:$${mariadb:password}@$${mariadb:ip}:$${mariadb:port}/$${mariadb:database}
 
+[cron-entry-mariadb-backup]
+<= cron
+recipe = slapos.cookbook:cron.d
+name = mariadb-backup
+frequency = 0 0 * * *
+command = $${mariadb:innobackupex-controller}
+
 [mariadb]
 recipe = slapos.cookbook:mysql
 
 # Options
-recovering = false
 user = user
 port = 45678
 ip = $${slap-network-information:local-ipv4}
@@ -39,12 +46,20 @@ backup-pending-directory = $${directory:mariadb-backup-pending}
 dumpname = dump.sql.gz
 
 # Binary information
+innobackupex-binary = ${xtrabackup:location}/bin/innobackupex
 mysql-binary = ${mariadb:location}/bin/mysql
 mysql-install-binary = ${mariadb:location}/bin/mysql_install_db
 mysql-upgrade-binary = ${mariadb:location}/bin/mysql_upgrade
 mysqld-binary = ${mariadb:location}/libexec/mysqld
-mysqldump-binary = ${mariadb:location}/bin/mysqldump
-
+mk-variable-advisor_binary = ${perl:siteprefix}/bin/mk-variable-advisor
+mk-table-usage_binary = ${perl:siteprefix}/bin/mk-table-usage
+mk-visual-explain_binary = ${perl:siteprefix}/bin/mk-visual-explain
+mk-config-diff_binary = ${perl:siteprefix}/bin/mk-config-diff
+mk-deadlock-logger_binary = ${perl:siteprefix}/bin/mk-deadlock-logger
+mk-error-log_binary = ${perl:siteprefix}/bin/mk-error-log
+mk-index-usage_binary = ${perl:siteprefix}/bin/mk-index-usage
+mk-query-advisor_binary = ${perl:siteprefix}/bin/mk-query-advisor
+xtrabackup-binary = ${xtrabackup:location}/bin/xtrabackup_51
 
 [logrotate]
 recipe = slapos.cookbook:logrotate

software/erp5/instance.cfg

@@ -10,12 +10,7 @@ recipe = ${instance-recipe:egg}:${instance-recipe:module}
 dcrond_binary = ${dcron:location}/sbin/crond
 gzip_binary = ${gzip:location}/bin/gzip
 httpd_binary = ${apache:location}/bin/httpd
-innobackupex_binary = ${xtrabackup:location}/bin/innobackupex
 logrotate_binary = ${logrotate:location}/usr/sbin/logrotate
-mysql_binary = ${mariadb:location}/bin/mysql
-mysql_install_binary = ${mariadb:location}/bin/mysql_install_db
-mysql_upgrade_binary = ${mariadb:location}/bin/mysql_upgrade
-mysqld_binary = ${mariadb:location}/libexec/mysqld
 openssl_binary = ${openssl:location}/bin/openssl
 perl_binary = ${perl:location}/bin/perl
 rdiff_backup_binary = ${buildout:bin-directory}/rdiff-backup
@@ -26,16 +21,7 @@ runzope_binary = ${buildout:bin-directory}/runzope
 sphinx_searchd_binary = ${sphinx:location}/bin/searchd
 tidstorage_repozo_binary = ${buildout:bin-directory}/tidstorage_repozo
 tidstoraged_binary = ${buildout:bin-directory}/tidstoraged
-xtrabackup_binary = ${xtrabackup:location}/bin/xtrabackup_51
 zabbix_agentd_binary = ${zabbix-agent:location}/sbin/zabbix_agentd
-mk-variable-advisor_binary = ${perl:siteprefix}/bin/mk-variable-advisor
-mk-table-usage_binary = ${perl:siteprefix}/bin/mk-table-usage
-mk-visual-explain_binary = ${perl:siteprefix}/bin/mk-visual-explain
-mk-config-diff_binary = ${perl:siteprefix}/bin/mk-config-diff
-mk-deadlock-logger_binary = ${perl:siteprefix}/bin/mk-deadlock-logger
-mk-error-log_binary = ${perl:siteprefix}/bin/mk-error-log
-mk-index-usage_binary = ${perl:siteprefix}/bin/mk-index-usage
-mk-query-advisor_binary = ${perl:siteprefix}/bin/mk-query-advisor
 
 link_binary_list =
   ${coreutils:location}/bin/basename

software/erp5/software.cfg

@@ -25,7 +25,7 @@ unzip = true
 [template-mariadb]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-mariadb.cfg
-md5sum = 06da9043777ba8e1fda5798cc8000535
+md5sum = bf1b49d90b268ba4e69d8423c3435b65
 output = ${buildout:directory}/template-mariadb.cfg
 mode = 0644