Mariadb 10.4
Update mariadb and related components:
- mariadb: 10.3.35 / 10.4.25 => 10.4.28
- mroonga: 12.09 => 13.01
- groonga: 12.0.7 => 13.0.1
- groonga-normalizer-mysql: 1.1.8 => 1.2.1
Drop mariadb 10.3
Updating to 10.4 took time, because running the test from software/erp5/test/test/benchmarks.py
sometimes caused mariadb to crash after some time - usually running the test 10 times in a loop was causing the crash (after 1 or 2 days).
Most of the time, mariadb process is still running, but no longer handle clients, gdb always showed a thread like this:
Thread 160 (Thread 0x7f15a505c700 (LWP 122797)):
#0 0x00007f1e0454771b in ?? () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007f1e044c5558 in malloc () from /lib/x86_64-linux-gnu/libc.so.6
#2 0x00007f1e04548432 in backtrace_symbols () from /lib/x86_64-linux-gnu/libc.so.6
#3 0x00007f1df4c8a7c4 in grn_ctx_log_back_trace_backtrace (level=GRN_LOG_ERROR, ctx=0x7f132014bbc0) at ctx.c:2141
#4 grn_ctx_log_back_trace (ctx=ctx@entry=0x7f132014bbc0, level=level@entry=GRN_LOG_ERROR) at ctx.c:2178
#5 0x00007f1df4dd2ce6 in grn_io_lock (ctx=ctx@entry=0x7f132014bbc0, io=0x7f1465bac4b0, timeout=<optimized out>) at io.c:1475
#6 0x00007f1df4db2fd3 in grn_ii_column_update (ctx=ctx@entry=0x7f132014bbc0, ii=ii@entry=0x7f1465a5d860, rid=62265, section=1, oldvalue=oldvalue@entry=0x7f15a5057fc0, newvalue=newvalue@entry=0x7f15a5058190, posting=0x0) at ii.c:8463
#7 0x00007f1df4c99248 in grn_obj_default_set_value_hook (ctx=ctx@entry=0x7f132014bbc0, nargs=nargs@entry=1, args=args@entry=0x7f15a5057f98, user_data=user_data@entry=0x7f15a5058020) at db.c:3564
#8 0x00007f1df4ca04ec in call_hook (flags=1, value=<optimized out>, id=62265, obj=<optimized out>, ctx=0x7f132014bbc0) at db.c:8013
#9 grn_obj_set_value_column_var_size (flags=1, value=0x0, id=62265, column=0x7f1465a5cb80, ctx=0x7f132014bbc0) at db.c:8133
#10 grn_obj_set_value (ctx=0x7f132014bbc0, obj=obj@entry=0x7f1465a5cb80, id=62265, value=value@entry=0x7f15a5058190, flags=flags@entry=1) at db.c:8267
#11 0x00007f1e0041cb2c in ha_mroonga::storage_write_row (this=0x7f132014a0d0, buf=0x7f1320174c00 "\376k\032\030") at ha_mroonga.cpp:7350
#12 0x000055f10715d2df in handler::ha_write_row (this=0x7f132014a0d0, buf=0x7f1320174c00 "\376k\032\030") at /srv/slapgrid/slappart15/srv/runner/instance/slappart7/tmp/shared/mariadb/978a7604a7585b1e57406d8c07e0bc55/.build/mariadb-10.4.25/sql/handler.cc:6791
#13 0x000055f106f2cd29 in write_record (thd=thd@entry=0x7f1320000c08, table=table@entry=0x7f13201492b8, info=info@entry=0x7f15a5059450) at /srv/slapgrid/slappart15/srv/runner/instance/slappart7/tmp/shared/mariadb/978a7604a7585b1e57406d8c07e0bc55/.build/mariadb-10.4.25/sql/sql_insert.cc:1752
#14 0x000055f106f337ea in mysql_insert (thd=thd@entry=0x7f1320000c08, table_list=<optimized out>, fields=..., values_list=..., update_fields=..., update_values=..., duplic=<optimized out>, ignore=<optimized out>) at /srv/slapgrid/slappart15/srv/runner/instance/slappart7/tmp/shared/mariadb/978a7604a7585b1e57406d8c07e0bc55/.build/mariadb-10.4.25/sql/sql_insert.cc:1083
#15 0x000055f106f631e9 in mysql_execute_command (thd=0x7f1320000c08) at /srv/slapgrid/slappart15/srv/runner/instance/slappart7/tmp/shared/mariadb/978a7604a7585b1e57406d8c07e0bc55/.build/mariadb-10.4.25/sql/sql_parse.cc:4598
#16 0x000055f106f68089 in mysql_parse (thd=0x7f1320000c08, rawbuf=<optimized out>, length=120, parser_state=0x7f15a505b580, is_com_multi=<optimized out>, is_next_command=<optimized out>) at /srv/slapgrid/slappart15/srv/runner/instance/slappart7/tmp/shared/mariadb/978a7604a7585b1e57406d8c07e0bc55/.build/mariadb-10.4.25/sql/sql_parse.cc:7995
#17 0x000055f106f6a525 in dispatch_command (command=COM_QUERY, thd=0x7f1320000c08, packet=<optimized out>, packet_length=<optimized out>, is_com_multi=<optimized out>, is_next_command=<optimized out>) at /srv/slapgrid/slappart15/srv/runner/instance/slappart7/tmp/shared/mariadb/978a7604a7585b1e57406d8c07e0bc55/.build/mariadb-10.4.25/sql/sql_class.h:1201
#18 0x000055f106f6b965 in do_command (thd=0x7f1320000c08) at /srv/slapgrid/slappart15/srv/runner/instance/slappart7/tmp/shared/mariadb/978a7604a7585b1e57406d8c07e0bc55/.build/mariadb-10.4.25/sql/sql_parse.cc:1378
#19 0x000055f107051c1e in do_handle_one_connection (connect=connect@entry=0x55f10a4ff608) at /srv/slapgrid/slappart15/srv/runner/instance/slappart7/tmp/shared/mariadb/978a7604a7585b1e57406d8c07e0bc55/.build/mariadb-10.4.25/sql/sql_connect.cc:1420
#20 0x000055f107051cdd in handle_one_connection (arg=0x55f10a4ff608) at /srv/slapgrid/slappart15/srv/runner/instance/slappart7/tmp/shared/mariadb/978a7604a7585b1e57406d8c07e0bc55/.build/mariadb-10.4.25/sql/sql_connect.cc:1316
#21 0x00007f1e0491cfa3 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#22 0x00007f1e0453a4cf in clone () from /lib/x86_64-linux-gnu/libc.so.6
or similar, another example:
Thread 162 (Thread 0x7f082db7f700 (LWP 51130)):
#0 0x00007f11d10e171b in ?? () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007f11d105f1bb in ?? () from /lib/x86_64-linux-gnu/libc.so.6
#2 0x00007f11d10a18ca in fork () from /lib/x86_64-linux-gnu/libc.so.6
#3 0x000055d4831dfd20 in start_addr2line_fork (binary_path=0x7ffcfa86ad29 "/srv/slapgrid/slappart15/srv/runner/instance/slappart7/tmp/shared/mariadb/bc5b4cdfa9be701b27f7ba3d46532f17/bin/mysqld") at /srv/slapgrid/slappart15/srv/runner/instance/slappart7/tmp/shared/mariadb/bc5b4cdfa9be701b27f7ba3d46532f17/.build/mariadb-10.4.28/mysys/my_addr_resolve.c:192
#4 start_addr2line_fork (binary_path=0x7ffcfa86ad29 "/srv/slapgrid/slappart15/srv/runner/instance/slappart7/tmp/shared/mariadb/bc5b4cdfa9be701b27f7ba3d46532f17/bin/mysqld") at /srv/slapgrid/slappart15/srv/runner/instance/slappart7/tmp/shared/mariadb/bc5b4cdfa9be701b27f7ba3d46532f17/.build/mariadb-10.4.28/mysys/my_addr_resolve.c:175
#5 0x000055d4831dfe54 in my_addr_resolve (ptr=0x55d4831c9fbe <my_print_stacktrace+46>, loc=loc@entry=0x7f082db7b600) at /srv/slapgrid/slappart15/srv/runner/instance/slappart7/tmp/shared/mariadb/bc5b4cdfa9be701b27f7ba3d46532f17/.build/mariadb-10.4.28/mysys/my_addr_resolve.c:241
#6 0x000055d4831ca013 in print_with_addr_resolve (n=<optimized out>, addrs=0x7f082db7b620) at /srv/slapgrid/slappart15/srv/runner/instance/slappart7/tmp/shared/mariadb/bc5b4cdfa9be701b27f7ba3d46532f17/.build/mariadb-10.4.28/mysys/stacktrace.c:159
#7 my_print_stacktrace (stack_bottom=<optimized out>, thread_stack=<optimized out>, silent=silent@entry=0 '\000') at /srv/slapgrid/slappart15/srv/runner/instance/slappart7/tmp/shared/mariadb/bc5b4cdfa9be701b27f7ba3d46532f17/.build/mariadb-10.4.28/mysys/stacktrace.c:178
#8 0x000055d482c9759d in handle_fatal_signal (sig=6) at /srv/slapgrid/slappart15/srv/runner/instance/slappart7/tmp/shared/mariadb/bc5b4cdfa9be701b27f7ba3d46532f17/.build/mariadb-10.4.28/sql/signal_handler.cc:231
#9 <signal handler called>
#10 0x00007f11d10127bb in raise () from /lib/x86_64-linux-gnu/libc.so.6
#11 0x00007f11d0ffd535 in abort () from /lib/x86_64-linux-gnu/libc.so.6
#12 0x00007f11d1054508 in ?? () from /lib/x86_64-linux-gnu/libc.so.6
#13 0x00007f11d105ac1a in ?? () from /lib/x86_64-linux-gnu/libc.so.6
#14 0x00007f11d105c730 in ?? () from /lib/x86_64-linux-gnu/libc.so.6
#15 0x00007f11ccf95a0b in ha_mroonga::~ha_mroonga (this=0x7f06d818c960, __in_chrg=<optimized out>) at ha_mroonga.cpp:3185
#16 0x000055d482b63416 in closefrm (table=table@entry=0x7f06d8032ff8) at /srv/slapgrid/slappart15/srv/runner/instance/slappart7/tmp/shared/mariadb/bc5b4cdfa9be701b27f7ba3d46532f17/.build/mariadb-10.4.28/sql/table.cc:4292
#17 0x000055d482c1e797 in intern_close_table (table=0x7f06d8032ff8) at /srv/slapgrid/slappart15/srv/runner/instance/slappart7/tmp/shared/mariadb/bc5b4cdfa9be701b27f7ba3d46532f17/.build/mariadb-10.4.28/sql/table_cache.cc:221
#18 tc_remove_table (table=table@entry=0x7f06d8032ff8) at /srv/slapgrid/slappart15/srv/runner/instance/slappart7/tmp/shared/mariadb/bc5b4cdfa9be701b27f7ba3d46532f17/.build/mariadb-10.4.28/sql/table_cache.cc:259
#19 0x000055d482c1ec13 in tc_release_table (table=0x7f06d8032ff8) at /srv/slapgrid/slappart15/srv/runner/instance/slappart7/tmp/shared/mariadb/bc5b4cdfa9be701b27f7ba3d46532f17/.build/mariadb-10.4.28/sql/table_cache.cc:471
#20 0x000055d482a42842 in close_thread_table (thd=thd@entry=0x7f06d8000c08, table_ptr=table_ptr@entry=0x7f06d8000cf0) at /srv/slapgrid/slappart15/srv/runner/instance/slappart7/tmp/shared/mariadb/bc5b4cdfa9be701b27f7ba3d46532f17/.build/mariadb-10.4.28/sql/sql_base.cc:1090
#21 0x000055d482a42a33 in close_thread_tables (thd=thd@entry=0x7f06d8000c08) at /srv/slapgrid/slappart15/srv/runner/instance/slappart7/tmp/shared/mariadb/bc5b4cdfa9be701b27f7ba3d46532f17/.build/mariadb-10.4.28/sql/sql_base.cc:1030
#22 0x000055d482aa584b in mysql_execute_command (thd=<optimized out>) at /srv/slapgrid/slappart15/srv/runner/instance/slappart7/tmp/shared/mariadb/bc5b4cdfa9be701b27f7ba3d46532f17/.build/mariadb-10.4.28/sql/sql_parse.cc:6262
#23 0x000055d482aad1f9 in mysql_parse (thd=0x7f06d8000c08, rawbuf=<optimized out>, length=70, parser_state=0x7f082db7e550, is_com_multi=<optimized out>, is_next_command=<optimized out>) at /srv/slapgrid/slappart15/srv/runner/instance/slappart7/tmp/shared/mariadb/bc5b4cdfa9be701b27f7ba3d46532f17/.build/mariadb-10.4.28/sql/sql_parse.cc:7986
#24 0x000055d482aaf905 in dispatch_command (command=COM_QUERY, thd=0x7f06d8000c08, packet=<optimized out>, packet_length=<optimized out>, is_com_multi=<optimized out>, is_next_command=<optimized out>) at /srv/slapgrid/slappart15/srv/runner/instance/slappart7/tmp/shared/mariadb/bc5b4cdfa9be701b27f7ba3d46532f17/.build/mariadb-10.4.28/sql/sql_class.h:1231
#25 0x000055d482ab0ad5 in do_command (thd=0x7f06d8000c08) at /srv/slapgrid/slappart15/srv/runner/instance/slappart7/tmp/shared/mariadb/bc5b4cdfa9be701b27f7ba3d46532f17/.build/mariadb-10.4.28/sql/sql_parse.cc:1378
#26 0x000055d482b982be in do_handle_one_connection (connect=connect@entry=0x55d4b07df2f8) at /srv/slapgrid/slappart15/srv/runner/instance/slappart7/tmp/shared/mariadb/bc5b4cdfa9be701b27f7ba3d46532f17/.build/mariadb-10.4.28/sql/sql_connect.cc:1420
#27 0x000055d482b9837d in handle_one_connection (arg=0x55d4b07df2f8) at /srv/slapgrid/slappart15/srv/runner/instance/slappart7/tmp/shared/mariadb/bc5b4cdfa9be701b27f7ba3d46532f17/.build/mariadb-10.4.28/sql/sql_connect.cc:1324
#28 0x00007f11d14b6fa3 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#29 0x00007f11d10d44cf in clone () from /lib/x86_64-linux-gnu/libc.so.6
On the machine used to reproduce this (COMP-3490), this reproduced for all combinations of mariadb/mroonga/groonga I tried, even the versions that we are using in production without seeing this problem, so my conclusions are that it should not be worse to use mariadb 10.4 regarding this problem.