Bug#30625 (Performance, reduce depth for expressions)

This is a performance bug, affecting in particular the bison generated code
for the parser.

Prior to this fix, the grammar used a long chain of reduces to parse an
expression, like:
  bit_expr -> bit_term
  bit_term -> bit_factor
  bit_factor -> value_expr
  value_expr -> term
  term -> factor
etc

This chain of reduces cause the internal state automaton in the generated
parser to execute more state transitions and more reduces, so that the
generated MySQLParse() function would spend a lot of time looping to execute
all the grammar reductions.

With this patch, the grammar has been reorganized so that rules are more
"flat", limiting the depth of reduces needed to parse <expr>.

Tests have been written to enforce that relative priorities and properties
of operators have not changed while changing the grammar.

See the bug report for performance data.


mysql-test/r/parser_precedence.result:
  Improved test coverage for operator precedence
mysql-test/t/parser_precedence.test:
  Improved test coverage for operator precedence
sql/sql_yacc.yy:
  Simplified the grammar to improve performances

mysql-test/t/parser_precedence.test

@@ -91,3 +91,243 @@ select count(*) from t1_30237_bool
 
 drop table t1_30237_bool;
 
+--echo Testing that NOT has precedence over OR
+select (NOT FALSE) OR TRUE, NOT (FALSE OR TRUE), NOT FALSE OR TRUE;
+
+--echo Testing that NOT has precedence over XOR
+select (NOT FALSE) XOR FALSE, NOT (FALSE XOR FALSE), NOT FALSE XOR FALSE;
+
+--echo Testing that NOT has precedence over AND
+select (NOT FALSE) AND FALSE, NOT (FALSE AND FALSE), NOT FALSE AND FALSE;
+
+--echo Testing that NOT is associative
+select NOT NOT TRUE, NOT NOT NOT FALSE;
+
+--echo Testing that IS has precedence over NOT
+select (NOT NULL) IS TRUE, NOT (NULL IS TRUE), NOT NULL IS TRUE;
+select (NOT NULL) IS NOT TRUE, NOT (NULL IS NOT TRUE), NOT NULL IS NOT TRUE;
+select (NOT NULL) IS FALSE, NOT (NULL IS FALSE), NOT NULL IS FALSE;
+select (NOT NULL) IS NOT FALSE, NOT (NULL IS NOT FALSE), NOT NULL IS NOT FALSE;
+select (NOT TRUE) IS UNKNOWN, NOT (TRUE IS UNKNOWN), NOT TRUE IS UNKNOWN;
+select (NOT TRUE) IS NOT UNKNOWN, NOT (TRUE IS NOT UNKNOWN), NOT TRUE IS NOT UNKNOWN;
+select (NOT TRUE) IS NULL, NOT (TRUE IS NULL), NOT TRUE IS NULL;
+select (NOT TRUE) IS NOT NULL, NOT (TRUE IS NOT NULL), NOT TRUE IS NOT NULL;
+
+--echo Testing that IS [NOT] TRUE/FALSE/UNKNOWN predicates are not associative
+# Documenting existing behavior in 5.0.48
+-- error ER_PARSE_ERROR
+select TRUE IS TRUE IS TRUE IS TRUE;
+-- error ER_PARSE_ERROR
+select FALSE IS NOT TRUE IS NOT TRUE IS NOT TRUE;
+-- error ER_PARSE_ERROR
+select NULL IS FALSE IS FALSE IS FALSE;
+-- error ER_PARSE_ERROR
+select TRUE IS NOT FALSE IS NOT FALSE IS NOT FALSE;
+-- error ER_PARSE_ERROR
+select FALSE IS UNKNOWN IS UNKNOWN IS UNKNOWN;
+-- error ER_PARSE_ERROR
+select TRUE IS NOT UNKNOWN IS NOT UNKNOWN IS NOT UNKNOWN;
+
+--echo Testing that IS [NOT] NULL predicates are associative
+# Documenting existing behavior in 5.0.48
+select FALSE IS NULL IS NULL IS NULL;
+select TRUE IS NOT NULL IS NOT NULL IS NOT NULL;
+
+--echo Testing that comparison operators are left associative
+select 1 <=> 2 <=> 2, (1 <=> 2) <=> 2, 1 <=> (2 <=> 2);
+select 1 = 2 = 2, (1 = 2) = 2, 1 = (2 = 2);
+select 1 != 2 != 3, (1 != 2) != 3, 1 != (2 != 3);
+select 1 <> 2 <> 3, (1 <> 2) <> 3, 1 <> (2 <> 3);
+select 1 < 2 < 3, (1 < 2) < 3, 1 < (2 < 3);
+select 3 <= 2 <= 1, (3 <= 2) <= 1, 3 <= (2 <= 1);
+select 1 > 2 > 3, (1 > 2) > 3, 1 > (2 > 3);
+select 1 >= 2 >= 3, (1 >= 2) >= 3, 1 >= (2 >= 3);
+
+-- echo Testing that | is associative
+select 0xF0 | 0x0F | 0x55, (0xF0 | 0x0F) | 0x55, 0xF0 | (0x0F | 0x55);
+
+-- echo Testing that & is associative
+select 0xF5 & 0x5F & 0x55, (0xF5 & 0x5F) & 0x55, 0xF5 & (0x5F & 0x55);
+
+-- echo Testing that << is left associative
+select 4 << 3 << 2, (4 << 3) << 2, 4 << (3 << 2);
+
+-- echo Testing that >> is left associative
+select 256 >> 3 >> 2, (256 >> 3) >> 2, 256 >> (3 >> 2);
+
+--echo Testing that & has precedence over |
+select 0xF0 & 0x0F | 0x55, (0xF0 & 0x0F) | 0x55, 0xF0 & (0x0F | 0x55);
+select 0x55 | 0xF0 & 0x0F, (0x55 | 0xF0) & 0x0F, 0x55 | (0xF0 & 0x0F);
+
+--echo Testing that << has precedence over |
+select 0x0F << 4 | 0x0F, (0x0F << 4) | 0x0F, 0x0F << (4 | 0x0F);
+select 0x0F | 0x0F << 4, (0x0F | 0x0F) << 4, 0x0F | (0x0F << 4);
+
+--echo Testing that >> has precedence over |
+select 0xF0 >> 4 | 0xFF, (0xF0 >> 4) | 0xFF, 0xF0 >> (4 | 0xFF);
+select 0xFF | 0xF0 >> 4, (0xFF | 0xF0) >> 4, 0xFF | (0xF0 >> 4);
+
+--echo Testing that << has precedence over &
+select 0x0F << 4 & 0xF0, (0x0F << 4) & 0xF0, 0x0F << (4 & 0xF0);
+select 0xF0 & 0x0F << 4, (0xF0 & 0x0F) << 4, 0xF0 & (0x0F << 4);
+
+--echo Testing that >> has precedence over &
+select 0xF0 >> 4 & 0x55, (0xF0 >> 4) & 0x55, 0xF0 >> (4 & 0x55);
+select 0x0F & 0xF0 >> 4, (0x0F & 0xF0) >> 4, 0x0F & (0xF0 >> 4);
+
+--echo Testing that >> and << have the same precedence
+select 0xFF >> 4 << 2, (0xFF >> 4) << 2, 0xFF >> (4 << 2);
+select 0x0F << 4 >> 2, (0x0F << 4) >> 2, 0x0F << (4 >> 2);
+
+--echo Testing that binary + is associative
+select 1 + 2 + 3, (1 + 2) + 3, 1 + (2 + 3);
+
+--echo Testing that binary - is left associative
+select 1 - 2 - 3, (1 - 2) - 3, 1 - (2 - 3);
+
+--echo Testing that binary + and binary - have the same precedence
+# evaluated left to right
+select 1 + 2 - 3, (1 + 2) - 3, 1 + (2 - 3);
+select 1 - 2 + 3, (1 - 2) + 3, 1 - (2 + 3);
+
+--echo Testing that binary + has precedence over |
+select 0xF0 + 0x0F | 0x55, (0xF0 + 0x0F) | 0x55, 0xF0 + (0x0F | 0x55);
+select 0x55 | 0xF0 + 0x0F, (0x55 | 0xF0) + 0x0F, 0x55 | (0xF0 + 0x0F);
+
+--echo Testing that binary + has precedence over &
+select 0xF0 + 0x0F & 0x55, (0xF0 + 0x0F) & 0x55, 0xF0 + (0x0F & 0x55);
+select 0x55 & 0xF0 + 0x0F, (0x55 & 0xF0) + 0x0F, 0x55 & (0xF0 + 0x0F);
+
+--echo Testing that binary + has precedence over <<
+select 2 + 3 << 4, (2 + 3) << 4, 2 + (3 << 4);
+select 3 << 4 + 2, (3 << 4) + 2, 3 << (4 + 2);
+
+--echo Testing that binary + has precedence over >>
+select 4 + 3 >> 2, (4 + 3) >> 2, 4 + (3 >> 2);
+select 3 >> 2 + 1, (3 >> 2) + 1, 3 >> (2 + 1);
+
+--echo Testing that binary - has precedence over |
+select 0xFF - 0x0F | 0x55, (0xFF - 0x0F) | 0x55, 0xFF - (0x0F | 0x55);
+select 0x55 | 0xFF - 0xF0, (0x55 | 0xFF) - 0xF0, 0x55 | (0xFF - 0xF0);
+
+--echo Testing that binary - has precedence over &
+select 0xFF - 0xF0 & 0x55, (0xFF - 0xF0) & 0x55, 0xFF - (0xF0 & 0x55);
+select 0x55 & 0xFF - 0xF0, (0x55 & 0xFF) - 0xF0, 0x55 & (0xFF - 0xF0);
+
+--echo Testing that binary - has precedence over <<
+select 16 - 3 << 2, (16 - 3) << 2, 16 - (3 << 2);
+select 4 << 3 - 2, (4 << 3) - 2, 4 << (3 - 2);
+
+--echo Testing that binary - has precedence over >>
+select 16 - 3 >> 2, (16 - 3) >> 2, 16 - (3 >> 2);
+select 16 >> 3 - 2, (16 >> 3) - 2, 16 >> (3 - 2);
+
+--echo Testing that * is associative
+select 2 * 3 * 4, (2 * 3) * 4, 2 * (3 * 4);
+
+--echo Testing that * has precedence over |
+select 2 * 0x40 | 0x0F, (2 * 0x40) | 0x0F, 2 * (0x40 | 0x0F);
+select 0x0F | 2 * 0x40, (0x0F | 2) * 0x40, 0x0F | (2 * 0x40);
+
+--echo Testing that * has precedence over &
+select 2 * 0x40 & 0x55, (2 * 0x40) & 0x55, 2 * (0x40 & 0x55);
+select 0xF0 & 2 * 0x40, (0xF0 & 2) * 0x40, 0xF0 & (2 * 0x40);
+
+--echo Testing that * has precedence over << 
+# Actually, can't prove it for the first case,
+# since << is a multiplication by a power of 2,
+# and * is associative
+select 5 * 3 << 4, (5 * 3) << 4, 5 * (3 << 4);
+select 2 << 3 * 4, (2 << 3) * 4, 2 << (3 * 4);
+
+--echo Testing that * has precedence over >>
+# >> is a multiplication by a (negative) power of 2,
+# see above.
+select 3 * 4 >> 2, (3 * 4) >> 2, 3 * (4 >> 2);
+select 4 >> 2 * 3, (4 >> 2) * 3, 4 >> (2 * 3);
+
+--echo Testing that * has precedence over binary +
+select 2 * 3 + 4, (2 * 3) + 4, 2 * (3 + 4);
+select 2 + 3 * 4, (2 + 3) * 4, 2 + (3 * 4);
+
+--echo Testing that * has precedence over binary -
+select 4 * 3 - 2, (4 * 3) - 2, 4 * (3 - 2);
+select 4 - 3 * 2, (4 - 3) * 2, 4 - (3 * 2);
+
+--echo Testing that / is left associative
+select 15 / 5 / 3, (15 / 5) / 3, 15 / (5 / 3);
+
+--echo Testing that / has precedence over |
+select 105 / 5 | 2, (105 / 5) | 2, 105 / (5 | 2);
+select 105 | 2 / 5, (105 | 2) / 5, 105 | (2 / 5);
+
+--echo Testing that / has precedence over &
+select 105 / 5 & 0x0F, (105 / 5) & 0x0F, 105 / (5 & 0x0F);
+select 0x0F & 105 / 5, (0x0F & 105) / 5, 0x0F & (105 / 5);
+
+--echo Testing that / has precedence over << 
+select 0x80 / 4 << 2, (0x80 / 4) << 2, 0x80 / (4 << 2);
+select 0x80 << 4 / 2, (0x80 << 4) / 2, 0x80 << (4 / 2);
+
+--echo Testing that / has precedence over >>
+select 0x80 / 4 >> 2, (0x80 / 4) >> 2, 0x80 / (4 >> 2);
+select 0x80 >> 4 / 2, (0x80 >> 4) / 2, 0x80 >> (4 / 2);
+
+--echo Testing that / has precedence over binary +
+select 0x80 / 2 + 2, (0x80 / 2) + 2, 0x80 / (2 + 2);
+select 0x80 + 2 / 2, (0x80 + 2) / 2, 0x80 + (2 / 2);
+
+--echo Testing that / has precedence over binary -
+select 0x80 / 4 - 2, (0x80 / 4) - 2, 0x80 / (4 - 2);
+select 0x80 - 4 / 2, (0x80 - 4) / 2, 0x80 - (4 / 2);
+
+# TODO: %, DIV, MOD
+
+--echo Testing that ^ is associative
+select 0xFF ^ 0xF0 ^ 0x0F, (0xFF ^ 0xF0) ^ 0x0F, 0xFF ^ (0xF0 ^ 0x0F);
+select 0xFF ^ 0xF0 ^ 0x55, (0xFF ^ 0xF0) ^ 0x55, 0xFF ^ (0xF0 ^ 0x55);
+
+--echo Testing that ^ has precedence over |
+select 0xFF ^ 0xF0 | 0x0F, (0xFF ^ 0xF0) | 0x0F, 0xFF ^ (0xF0 | 0x0F);
+select 0xF0 | 0xFF ^ 0xF0, (0xF0 | 0xFF) ^ 0xF0, 0xF0 | (0xFF ^ 0xF0);
+
+--echo Testing that ^ has precedence over &
+select 0xFF ^ 0xF0 & 0x0F, (0xFF ^ 0xF0) & 0x0F, 0xFF ^ (0xF0 & 0x0F);
+select 0x0F & 0xFF ^ 0xF0, (0x0F & 0xFF) ^ 0xF0, 0x0F & (0xFF ^ 0xF0);
+
+--echo Testing that ^ has precedence over <<
+select 0xFF ^ 0xF0 << 2, (0xFF ^ 0xF0) << 2, 0xFF ^ (0xF0 << 2);
+select 0x0F << 2 ^ 0xFF, (0x0F << 2) ^ 0xFF, 0x0F << (2 ^ 0xFF);
+
+--echo Testing that ^ has precedence over >>
+select 0xFF ^ 0xF0 >> 2, (0xFF ^ 0xF0) >> 2, 0xFF ^ (0xF0 >> 2);
+select 0xFF >> 2 ^ 0xF0, (0xFF >> 2) ^ 0xF0, 0xFF >> (2 ^ 0xF0);
+
+--echo Testing that ^ has precedence over binary +
+select 0xFF ^ 0xF0 + 0x0F, (0xFF ^ 0xF0) + 0x0F, 0xFF ^ (0xF0 + 0x0F);
+select 0x0F + 0xFF ^ 0xF0, (0x0F + 0xFF) ^ 0xF0, 0x0F + (0xFF ^ 0xF0);
+
+--echo Testing that ^ has precedence over binary -
+select 0xFF ^ 0xF0 - 1, (0xFF ^ 0xF0) - 1, 0xFF ^ (0xF0 - 1);
+select 0x55 - 0x0F ^ 0x55, (0x55 - 0x0F) ^ 0x55, 0x55 - (0x0F ^ 0x55);
+
+--echo Testing that ^ has precedence over *
+select 0xFF ^ 0xF0 * 2, (0xFF ^ 0xF0) * 2, 0xFF ^ (0xF0 * 2);
+select 2 * 0xFF ^ 0xF0, (2 * 0xFF) ^ 0xF0, 2 * (0xFF ^ 0xF0);
+
+--echo Testing that ^ has precedence over /
+select 0xFF ^ 0xF0 / 2, (0xFF ^ 0xF0) / 2, 0xFF ^ (0xF0 / 2);
+select 0xF2 / 2 ^ 0xF0, (0xF2 / 2) ^ 0xF0, 0xF2 / (2 ^ 0xF0);
+
+--echo Testing that ^ has precedence over %
+select 0xFF ^ 0xF0 % 0x20, (0xFF ^ 0xF0) % 0x20, 0xFF ^ (0xF0 % 0x20);
+select 0xFF % 0x20 ^ 0xF0, (0xFF % 0x20) ^ 0xF0, 0xFF % (0x20 ^ 0xF0);
+
+--echo Testing that ^ has precedence over DIV
+select 0xFF ^ 0xF0 DIV 2, (0xFF ^ 0xF0) DIV 2, 0xFF ^ (0xF0 DIV 2);
+select 0xF2 DIV 2 ^ 0xF0, (0xF2 DIV 2) ^ 0xF0, 0xF2 DIV (2 ^ 0xF0);
+
+--echo Testing that ^ has precedence over MOD
+select 0xFF ^ 0xF0 MOD 0x20, (0xFF ^ 0xF0) MOD 0x20, 0xFF ^ (0xF0 MOD 0x20);
+select 0xFF MOD 0x20 ^ 0xF0, (0xFF MOD 0x20) ^ 0xF0, 0xFF MOD (0x20 ^ 0xF0);
+

sql/sql_yacc.yy

@@ -1068,9 +1068,9 @@ bool my_yyoverflow(short **a, YYSTYPE **b, ulong *yystacksize);
 %type <item>
 	literal text_literal insert_ident order_ident
 	simple_ident select_item2 expr opt_expr opt_else sum_expr in_sum_expr
-	variable variable_aux bool_factor
-        bool_test bool_pri
-	predicate bit_expr bit_term bit_factor value_expr term factor
+	variable variable_aux
+        bool_pri
+	predicate bit_expr
 	table_wild simple_expr udf_expr
 	expr_or_default set_expr_or_default interval_expr
 	param_marker geometry_function
@@ -4468,8 +4468,7 @@ optional_braces:
 
 /* all possible expressions */
 expr:
-          bool_factor
-        | expr or expr %prec OR_SYM
+          expr or expr %prec OR_SYM
           {
             /*
               Design notes:
@@ -4564,30 +4563,30 @@ expr:
               $$ = new (YYTHD->mem_root) Item_cond_and($1, $3);
             }
           }
-        ;
-
-bool_factor:
-	NOT_SYM bool_factor	{ $$= negate_expression(YYTHD, $2); }
-	| bool_test ;
-
-bool_test:
-          bool_pri IS TRUE_SYM
+	| NOT_SYM expr %prec NOT_SYM
+          { $$= negate_expression(YYTHD, $2); }
+        | bool_pri IS TRUE_SYM %prec IS
           { $$= new (YYTHD->mem_root) Item_func_istrue($1); }
-        | bool_pri IS not TRUE_SYM
+        | bool_pri IS not TRUE_SYM %prec IS
           { $$= new (YYTHD->mem_root) Item_func_isnottrue($1); }
-        | bool_pri IS FALSE_SYM
+        | bool_pri IS FALSE_SYM %prec IS
           { $$= new (YYTHD->mem_root) Item_func_isfalse($1); }
-        | bool_pri IS not FALSE_SYM
+        | bool_pri IS not FALSE_SYM %prec IS
           { $$= new (YYTHD->mem_root) Item_func_isnotfalse($1); }
-        | bool_pri IS UNKNOWN_SYM { $$= new Item_func_isnull($1); }
-        | bool_pri IS not UNKNOWN_SYM { $$= new Item_func_isnotnull($1); }
+        | bool_pri IS UNKNOWN_SYM %prec IS
+          { $$= new Item_func_isnull($1); }
+        | bool_pri IS not UNKNOWN_SYM %prec IS
+          { $$= new Item_func_isnotnull($1); }
         | bool_pri
         ;
 
 bool_pri:
-	bool_pri IS NULL_SYM	{ $$= new Item_func_isnull($1); }
-	| bool_pri IS not NULL_SYM { $$= new Item_func_isnotnull($1); }
-	| bool_pri EQUAL_SYM predicate	{ $$= new Item_func_equal($1,$3); }
+	bool_pri IS NULL_SYM %prec IS
+          { $$= new Item_func_isnull($1); }
+	| bool_pri IS not NULL_SYM %prec IS
+          { $$= new Item_func_isnotnull($1); }
+	| bool_pri EQUAL_SYM predicate %prec EQUAL_SYM
+          { $$= new Item_func_equal($1,$3); }
 	| bool_pri comp_op predicate %prec EQ
 	  { $$= (*$2)(0)->create($1,$3); }
 	| bool_pri comp_op all_or_any '(' subselect ')' %prec EQ
@@ -4648,40 +4647,36 @@ predicate:
 	| bit_expr ;
 
 bit_expr:
-	bit_expr '|' bit_term	{ $$= new Item_func_bit_or($1,$3); }
-	| bit_term ;
-
-bit_term:
-	bit_term '&' bit_factor	{ $$= new Item_func_bit_and($1,$3); }
-	| bit_factor ;
-
-bit_factor:
-	bit_factor SHIFT_LEFT value_expr
+          bit_expr '|' bit_expr %prec '|'
+          { $$= new Item_func_bit_or($1,$3); }
+        | bit_expr '&' bit_expr %prec '&'
+          { $$= new Item_func_bit_and($1,$3); }
+        | bit_expr SHIFT_LEFT bit_expr %prec SHIFT_LEFT
           { $$= new Item_func_shift_left($1,$3); }
-	| bit_factor SHIFT_RIGHT value_expr 
+        | bit_expr SHIFT_RIGHT bit_expr %prec SHIFT_RIGHT
           { $$= new Item_func_shift_right($1,$3); }
-	| value_expr ;
-
-value_expr:
-	value_expr '+' term	{ $$= new Item_func_plus($1,$3); }
-	| value_expr '-' term	{ $$= new Item_func_minus($1,$3); }
-	| value_expr '+' interval_expr interval
+        | bit_expr '+' bit_expr %prec '+'
+          { $$= new Item_func_plus($1,$3); }
+        | bit_expr '-' bit_expr %prec '-'
+          { $$= new Item_func_minus($1,$3); }
+        | bit_expr '+' interval_expr interval %prec '+'
           { $$= new Item_date_add_interval($1,$3,$4,0); }
-	| value_expr '-' interval_expr interval
+        | bit_expr '-' interval_expr interval %prec '-'
           { $$= new Item_date_add_interval($1,$3,$4,1); }
-	| term ;
-
-term:
-	term '*' factor		{ $$= new Item_func_mul($1,$3); }
-	| term '/' factor	{ $$= new Item_func_div($1,$3); }
-	| term '%' factor	{ $$= new Item_func_mod($1,$3); }
-	| term DIV_SYM factor	{ $$= new Item_func_int_div($1,$3); }
-	| term MOD_SYM factor	{ $$= new Item_func_mod($1,$3); }
-	| factor ;
-
-factor:
-        factor '^' simple_expr	{ $$= new Item_func_bit_xor($1,$3); }
-	| simple_expr ;
+        | bit_expr '*' bit_expr %prec '*'
+          { $$= new Item_func_mul($1,$3); }
+        | bit_expr '/' bit_expr %prec '/'
+          { $$= new Item_func_div($1,$3); }
+        | bit_expr '%' bit_expr %prec '%'
+          { $$= new Item_func_mod($1,$3); }
+        | bit_expr DIV_SYM bit_expr %prec DIV_SYM
+          { $$= new Item_func_int_div($1,$3); }
+        | bit_expr MOD_SYM bit_expr %prec MOD_SYM
+          { $$= new Item_func_mod($1,$3); }
+        | bit_expr '^' bit_expr
+          { $$= new Item_func_bit_xor($1,$3); }
+        | simple_expr
+        ;
 
 or:	OR_SYM | OR2_SYM;
 and:	AND_SYM | AND_AND_SYM;