trying to stabilize floating-point tests

mysql-test/main/vector.result

@@ -108,14 +108,14 @@ id	hex(v)
 flush tables;
 select id,vec_distance_euclidean(v, x'B047263c9f87233fcfd27e3eae493e3f0329f43e') d from t1 order by d limit 3;
 id	d
-9	0.47199761867523193
-10	0.5069010853767395
-3	0.5865673422813416
+9	0.47199
+10	0.50690
+3	0.58656
 select id,vec_distance_euclidean(x'b047263C9f87233fcfd27e3eae493e3f0329f43e', v) d from t1 order by d limit 3;
 id	d
-9	0.47199761867523193
-10	0.5069010853767395
-3	0.5865673422813416
+9	0.47199
+10	0.50690
+3	0.58656
 select id>0,vec_distance_euclidean(v, NULL) d from t1 order by d limit 3;
 id>0	d
 1	NULL
@@ -138,149 +138,149 @@ id1	id2	vec_distance_euclidean(t1.v, t2.v)
 8	8	0
 9	9	0
 10	10	0
-7	10	0.35209009051322937
-10	7	0.35209009051322937
-1	7	0.5572673082351685
-7	1	0.5572673082351685
-2	3	0.6065129041671753
-3	2	0.6065129041671753
-1	3	0.6128237843513489
-3	1	0.6128237843513489
-5	8	0.6219995617866516
-8	5	0.6219995617866516
-3	10	0.65231853723526
-10	3	0.65231853723526
-9	10	0.6732681393623352
-10	9	0.6732681393623352
-3	7	0.679989218711853
-7	3	0.679989218711853
-3	9	0.6820752024650574
-9	3	0.6820752024650574
-2	10	0.6916305422782898
-10	2	0.6916305422782898
-2	9	0.6966650485992432
-9	2	0.6966650485992432
-3	6	0.7102823853492737
-6	3	0.7102823853492737
-2	7	0.7120217680931091
-7	2	0.7120217680931091
-2	6	0.7351617813110352
-6	2	0.7351617813110352
-1	10	0.7386864423751831
-10	1	0.7386864423751831
-4	6	0.7784357666969299
-6	4	0.7784357666969299
-4	8	0.779583752155304
-8	4	0.779583752155304
-4	5	0.8132007122039795
-5	4	0.8132007122039795
-2	4	0.826092541217804
-4	2	0.826092541217804
-5	10	0.8286488652229309
-10	5	0.8286488652229309
-5	9	0.8769351243972778
-9	5	0.8769351243972778
-1	6	0.8861410617828369
-6	1	0.8861410617828369
-3	5	0.9224202036857605
-5	3	0.9224202036857605
-4	7	0.934791624546051
-7	4	0.934791624546051
-7	9	0.9364253282546997
-9	7	0.9364253282546997
-3	4	0.9757105708122253
-4	3	0.9757105708122253
-1	2	0.9810272455215454
-2	1	0.9810272455215454
-1	4	0.9965475797653198
-4	1	0.9965475797653198
-5	7	0.9976863861083984
-7	5	0.9976863861083984
-4	10	1.010934591293335
-10	4	1.010934591293335
-1	5	1.0208359956741333
-5	1	1.0208359956741333
-6	7	1.022133231163025
-7	6	1.022133231163025
-2	5	1.050769329071045
-5	2	1.050769329071045
-6	8	1.103420376777649
-8	6	1.103420376777649
-3	8	1.1170300245285034
-8	3	1.1170300245285034
-6	10	1.1523451805114746
-10	6	1.1523451805114746
-1	9	1.163775086402893
-9	1	1.163775086402893
-2	8	1.1736570596694946
-8	2	1.1736570596694946
-4	9	1.1746894121170044
-9	4	1.1746894121170044
-1	8	1.1909960508346558
-8	1	1.1909960508346558
-8	10	1.2093596458435059
-10	8	1.2093596458435059
-6	9	1.2145298719406128
-9	6	1.2145298719406128
-5	6	1.2272785902023315
-6	5	1.2272785902023315
-8	9	1.2575258016586304
-9	8	1.2575258016586304
-7	8	1.288239598274231
-8	7	1.288239598274231
+7	10	0.35209
+10	7	0.35209
+1	7	0.55726
+7	1	0.55726
+2	3	0.60651
+3	2	0.60651
+1	3	0.61282
+3	1	0.61282
+5	8	0.62199
+8	5	0.62199
+3	10	0.65231
+10	3	0.65231
+9	10	0.67326
+10	9	0.67326
+3	7	0.67998
+7	3	0.67998
+3	9	0.68207
+9	3	0.68207
+2	10	0.69163
+10	2	0.69163
+2	9	0.69666
+9	2	0.69666
+3	6	0.71028
+6	3	0.71028
+2	7	0.71202
+7	2	0.71202
+2	6	0.73516
+6	2	0.73516
+1	10	0.73868
+10	1	0.73868
+4	6	0.77843
+6	4	0.77843
+4	8	0.77958
+8	4	0.77958
+4	5	0.81320
+5	4	0.81320
+2	4	0.82609
+4	2	0.82609
+5	10	0.82864
+10	5	0.82864
+5	9	0.87693
+9	5	0.87693
+1	6	0.88614
+6	1	0.88614
+3	5	0.92242
+5	3	0.92242
+4	7	0.93479
+7	4	0.93479
+7	9	0.93642
+9	7	0.93642
+3	4	0.97571
+4	3	0.97571
+1	2	0.98102
+2	1	0.98102
+1	4	0.99654
+4	1	0.99654
+5	7	0.99768
+7	5	0.99768
+4	10	1.01093
+10	4	1.01093
+1	5	1.02083
+5	1	1.02083
+6	7	1.02213
+7	6	1.02213
+2	5	1.05076
+5	2	1.05076
+6	8	1.10342
+8	6	1.10342
+3	8	1.11703
+8	3	1.11703
+6	10	1.15234
+10	6	1.15234
+1	9	1.16377
+9	1	1.16377
+2	8	1.17365
+8	2	1.17365
+4	9	1.17468
+9	4	1.17468
+1	8	1.19099
+8	1	1.19099
+8	10	1.20935
+10	8	1.20935
+6	9	1.21452
+9	6	1.21452
+5	6	1.22727
+6	5	1.22727
+8	9	1.25752
+9	8	1.25752
+7	8	1.28823
+8	7	1.28823
 flush session status;
 select id,vec_distance_euclidean(v, x'B047263c9f87233fcfd27e3eae493e3f0329f43e') d from t1 order by d limit 3;
 id	d
-9	0.47199761867523193
-10	0.5069010853767395
-3	0.5865673422813416
+9	0.47199
+10	0.50690
+3	0.58656
 show status like 'handler_read_rnd_next';
 Variable_name	Value
 Handler_read_rnd_next	0
 select id,vec_distance_euclidean(v, x'B047263c9f87233fcfd27e3eae493e3f0329f43e') d from t1 use index () order by d limit 3;
 id	d
-9	0.47199761867523193
-10	0.5069010853767395
-3	0.5865673422813416
+9	0.47199
+10	0.50690
+3	0.58656
 show status like 'handler_read_rnd_next';
 Variable_name	Value
 Handler_read_rnd_next	11
 flush session status;
 select id,vec_distance_cosine(v, x'B047263c9f87233fcfd27e3eae493e3f0329f43e') d from t1 order by d limit 3;
 id	d
-10	0.059055447578430176
-9	0.06546902656555176
-3	0.10750287771224976
+10	0.05905
+9	0.06546
+3	0.10750
 show status like 'handler_read_rnd_next';
 Variable_name	Value
 Handler_read_rnd_next	11
 delete from t1 where v = x'7b713f3e5258323f80d1113d673b2b3f66e3583f';
 select id,vec_distance_euclidean(v, x'B047263C9f87233fcfd27e3eae493e3f0329f43e') d from t1 order by d limit 3;
 id	d
-10	0.5069010853767395
-3	0.5865673422813416
-7	0.7344464659690857
+10	0.50690
+3	0.58656
+7	0.73444
 insert t1 (v) values (x'7b713f3e5258323f80d1113d673b2b3f66e3583f');
 select id,vec_distance_euclidean(v, x'b047263c9F87233fcfd27e3eae493e3f0329f43e') d from t1 order by d limit 3;
 id	d
-11	0.47199761867523193
-10	0.5069010853767395
-3	0.5865673422813416
+11	0.47199
+10	0.50690
+3	0.58656
 select id,vec_distance_euclidean(v, x'B047263c9F87233fcfd27e3eae493e3f0329f43e') d from t1 order by d limit 5;
 id	d
-11	0.47199761867523193
-10	0.5069010853767395
-3	0.5865673422813416
-7	0.7344464659690857
-5	0.7671033143997192
+11	0.47199
+10	0.50690
+3	0.58656
+7	0.73444
+5	0.76710
 update t1 set v=x'76EDFC3E4B57243F10F8423FB158713F020BAA3E' where v=x'6CA1D43E9DF91B3FE580DA3E1C247D3F147CF33E';
 select id,vec_distance_euclidean(v, x'B047263C9F87233fcfd27e3eae493e3f0329f43e') d from t1 order by d limit 5;
 id	d
-11	0.47199761867523193
-3	0.5865673422813416
-7	0.7344464659690857
-10	0.7468367218971252
-5	0.7671033143997192
+11	0.47199
+3	0.58656
+7	0.73444
+10	0.74683
+5	0.76710
 delete from t1;
 insert t1 (v) values (x'e360d63ebe554f3fcdbc523f4522193f5236083d'),
 (x'f511303f72224a3fdd05fe3eb22a133ffae86a3f'),
@@ -294,11 +294,11 @@ insert t1 (v) values (x'e360d63ebe554f3fcdbc523f4522193f5236083d'),
 (x'6ca1d43e9df91b3fe580da3e1c247d3f147cf33e');
 select id,vec_distance_euclidean(v, x'b047263c9f87233Fcfd27e3eae493e3f0329f43e') d from t1 order by d limit 5;
 id	d
-20	0.47199761867523193
-21	0.5069010853767395
-14	0.5865673422813416
-18	0.7344464659690857
-16	0.7671033143997192
+20	0.47199
+21	0.50690
+14	0.58656
+18	0.73444
+16	0.76710
 insert t1 (v) values ('');
 ERROR 22007: Incorrect vector value: '...' for column `test`.`t1`.`v` at row 1
 insert t1 (v) values (x'1234');
@@ -426,32 +426,32 @@ insert t1 (v) values (x'e360d63ebe554f3fcdbc523f4522193f5236083d'),
 (x'6ca1d43e9df91b3fe580da3e1c247d3f147cf33e');
 select id,vec_distance_cosine(v, x'B047263c9f87233fcfd27e3eae493e3f0329f43e') d from t1 order by d limit 3;
 id	d
-10	0.059055447578430176
-9	0.06546902656555176
-3	0.10750287771224976
+10	0.05905
+9	0.06546
+3	0.10750
 flush session status;
 select id,vec_distance_cosine(v, x'B047263c9f87233fcfd27e3eae493e3f0329f43e') d from t1 order by d limit 3;
 id	d
-10	0.059055447578430176
-9	0.06546902656555176
-3	0.10750287771224976
+10	0.05905
+9	0.06546
+3	0.10750
 show status like 'handler_read_rnd_next';
 Variable_name	Value
 Handler_read_rnd_next	0
 select id,vec_distance_cosine(v, x'B047263c9f87233fcfd27e3eae493e3f0329f43e') d from t1 use index () order by d limit 3;
 id	d
-10	0.059055447578430176
-9	0.06546902656555176
-3	0.10750287771224976
+10	0.05905
+9	0.06546
+3	0.10750
 show status like 'handler_read_rnd_next';
 Variable_name	Value
 Handler_read_rnd_next	11
 flush session status;
 select id,vec_distance_euclidean(v, x'B047263c9f87233fcfd27e3eae493e3f0329f43e') d from t1 order by d limit 3;
 id	d
-9	0.47199761867523193
-10	0.5069010853767395
-3	0.5865673422813416
+9	0.47199
+10	0.50690
+3	0.58656
 show status like 'handler_read_rnd_next';
 Variable_name	Value
 Handler_read_rnd_next	11

mysql-test/main/vector.test

@@ -41,24 +41,32 @@ insert t1 (v) values (x'e360d63ebe554f3fcdbc523f4522193f5236083d'),
 select id, hex(v) from t1;
 flush tables;
 # test with a valid query vector
+--replace_regex /(\.\d{5})\d+/\1/
 select id,vec_distance_euclidean(v, x'B047263c9f87233fcfd27e3eae493e3f0329f43e') d from t1 order by d limit 3;
 # swapped arguments
+--replace_regex /(\.\d{5})\d+/\1/
 select id,vec_distance_euclidean(x'b047263C9f87233fcfd27e3eae493e3f0329f43e', v) d from t1 order by d limit 3;
 # test with NULL (id is unpredictable)
+--replace_regex /(\.\d{5})\d+/\1/
 select id>0,vec_distance_euclidean(v, NULL) d from t1 order by d limit 3;
 # test with invalid query vector (id is unpredictable)
+--replace_regex /(\.\d{5})\d+/\1/
 select id>0,vec_distance_euclidean(v, x'123456') d from t1 order by d limit 3;
+--replace_regex /(\.\d{5})\d+/\1/
 select t1.id as id1, t2.id as id2, vec_distance_euclidean(t1.v, t2.v) from t1, t1 as t2 order by 3,1,2;
 
 # see if order by uses index:
 --disable_view_protocol
 --disable_ps2_protocol
 flush session status;
+--replace_regex /(\.\d{5})\d+/\1/
 select id,vec_distance_euclidean(v, x'B047263c9f87233fcfd27e3eae493e3f0329f43e') d from t1 order by d limit 3;
 show status like 'handler_read_rnd_next'; # used
+--replace_regex /(\.\d{5})\d+/\1/
 select id,vec_distance_euclidean(v, x'B047263c9f87233fcfd27e3eae493e3f0329f43e') d from t1 use index () order by d limit 3;
 show status like 'handler_read_rnd_next'; # not used
 flush session status;
+--replace_regex /(\.\d{5})\d+/\1/
 select id,vec_distance_cosine(v, x'B047263c9f87233fcfd27e3eae493e3f0329f43e') d from t1 order by d limit 3;
 show status like 'handler_read_rnd_next'; # not used, wrong distance metric
 --enable_ps2_protocol
@@ -66,15 +74,19 @@ show status like 'handler_read_rnd_next'; # not used, wrong distance metric
 
 # test delete
 delete from t1 where v = x'7b713f3e5258323f80d1113d673b2b3f66e3583f';
+--replace_regex /(\.\d{5})\d+/\1/
 select id,vec_distance_euclidean(v, x'B047263C9f87233fcfd27e3eae493e3f0329f43e') d from t1 order by d limit 3;
 
 # test insert deleted vec
 insert t1 (v) values (x'7b713f3e5258323f80d1113d673b2b3f66e3583f');
+--replace_regex /(\.\d{5})\d+/\1/
 select id,vec_distance_euclidean(v, x'b047263c9F87233fcfd27e3eae493e3f0329f43e') d from t1 order by d limit 3;
 
 # test update
+--replace_regex /(\.\d{5})\d+/\1/
 select id,vec_distance_euclidean(v, x'B047263c9F87233fcfd27e3eae493e3f0329f43e') d from t1 order by d limit 5;
 update t1 set v=x'76EDFC3E4B57243F10F8423FB158713F020BAA3E' where v=x'6CA1D43E9DF91B3FE580DA3E1C247D3F147CF33E';
+--replace_regex /(\.\d{5})\d+/\1/
 select id,vec_distance_euclidean(v, x'B047263C9F87233fcfd27e3eae493e3f0329f43e') d from t1 order by d limit 5;
 
 # test delete all and reinsert
@@ -89,6 +101,7 @@ insert t1 (v) values (x'e360d63ebe554f3fcdbc523f4522193f5236083d'),
                      (x'56926c3fdf098d3e2c8c5e3d1ad4953daa9d0b3e'),
                      (x'7b713f3e5258323f80d1113d673b2b3f66e3583f'),
                      (x'6ca1d43e9df91b3fe580da3e1c247d3f147cf33e');
+--replace_regex /(\.\d{5})\d+/\1/
 select id,vec_distance_euclidean(v, x'b047263c9f87233Fcfd27e3eae493e3f0329f43e') d from t1 order by d limit 5;
 
 
@@ -190,16 +203,20 @@ insert t1 (v) values (x'e360d63ebe554f3fcdbc523f4522193f5236083d'),
                      (x'6ca1d43e9df91b3fe580da3e1c247d3f147cf33e');
 
 # make sure the graph is loaded
+--replace_regex /(\.\d{5})\d+/\1/
 select id,vec_distance_cosine(v, x'B047263c9f87233fcfd27e3eae493e3f0329f43e') d from t1 order by d limit 3;
 
 --disable_view_protocol
 --disable_ps2_protocol
 flush session status;
+--replace_regex /(\.\d{5})\d+/\1/
 select id,vec_distance_cosine(v, x'B047263c9f87233fcfd27e3eae493e3f0329f43e') d from t1 order by d limit 3;
 show status like 'handler_read_rnd_next';
+--replace_regex /(\.\d{5})\d+/\1/
 select id,vec_distance_cosine(v, x'B047263c9f87233fcfd27e3eae493e3f0329f43e') d from t1 use index () order by d limit 3;
 show status like 'handler_read_rnd_next';
 flush session status;
+--replace_regex /(\.\d{5})\d+/\1/
 select id,vec_distance_euclidean(v, x'B047263c9f87233fcfd27e3eae493e3f0329f43e') d from t1 order by d limit 3;
 show status like 'handler_read_rnd_next';
 --enable_ps2_protocol

mysql-test/main/vector_innodb.result

@@ -40,18 +40,18 @@ delete from t1 where id=7;
 rollback;
 select id,vec_distance_euclidean(v, x'c923e33dc0da313fe7c7983e526b3d3fde63963e6eaf3a3f27fa133fe27a583f') d from t1 order by d limit 5;
 id	d
-2	0.8781474828720093
-10	0.8856208324432373
-30	1.0162643194198608
-7	1.0263972282409668
-5	1.0308160781860352
+2	0.87814
+10	0.88562
+30	1.01626
+7	1.02639
+5	1.03081
 select id,vec_distance_euclidean(v, x'754b5f3ea2312b3fc169f43e4604883e1d20173e8dd7443f421b703fb11e0d3e') d from t1 order by d limit 5;
 id	d
-2	0.9426904320716858
-33	0.9477554559707642
-30	1.1114054918289185
-10	1.118630290031433
-8	1.140573263168335
+2	0.94269
+33	0.94775
+30	1.11140
+10	1.11863
+8	1.14057
 create table t2 (id int auto_increment primary key, v blob not null, vector index (v)) engine=innodb;
 insert t2 (v) values
 (x'45cf153f830a313f7a0a113fb1ff533f47a1533fcf9e6e3f'),
@@ -73,16 +73,16 @@ insert t2 values
 commit;
 select id,vec_distance_euclidean(v, x'1f4d053f7056493f937da03dd8c97a3f220cbb3c926c1c3facca213ec0618a3e') d from t1 order by d limit 5;
 id	d
-6	0.9309383034706116
-5	0.9706304669380188
-30	0.9814448952674866
-50	1.0798625946044922
-2	1.0907139778137207
+6	0.93093
+5	0.97063
+30	0.98144
+50	1.07986
+2	1.09071
 select id,vec_distance_euclidean(v, x'f618663f256be73e62cd453f8bcdbf3e16ae503c3858313f') d from t2 order by d limit 5;
 id	d
-21	0.43559178709983826
-20	0.643505334854126
-6	0.6942000389099121
-2	0.7971622347831726
-9	0.8298588991165161
+21	0.43559
+20	0.64350
+6	0.69420
+2	0.79716
+9	0.82985
 drop table t1, t2;

mysql-test/main/vector_innodb.test

@@ -37,7 +37,9 @@ insert t1 values
 delete from t1 where id=7;
 rollback;
 
+--replace_regex /(\.\d{5})\d+/\1/
 select id,vec_distance_euclidean(v, x'c923e33dc0da313fe7c7983e526b3d3fde63963e6eaf3a3f27fa133fe27a583f') d from t1 order by d limit 5;
+--replace_regex /(\.\d{5})\d+/\1/
 select id,vec_distance_euclidean(v, x'754b5f3ea2312b3fc169f43e4604883e1d20173e8dd7443f421b703fb11e0d3e') d from t1 order by d limit 5;
 
 ### two indexes in one transaction:
@@ -63,7 +65,9 @@ insert t2 values
      (21, x'35e05d3f18e8513fb81a3d3f8acf7d3e794a1d3c72f9613f');
 commit;
 
+--replace_regex /(\.\d{5})\d+/\1/
 select id,vec_distance_euclidean(v, x'1f4d053f7056493f937da03dd8c97a3f220cbb3c926c1c3facca213ec0618a3e') d from t1 order by d limit 5;
+--replace_regex /(\.\d{5})\d+/\1/
 select id,vec_distance_euclidean(v, x'f618663f256be73e62cd453f8bcdbf3e16ae503c3858313f') d from t2 order by d limit 5;
 
 drop table t1, t2;