MDEV-21065 UNIQUE constraint causes a query with string comparison to omit a row in the result set

mysql-test/r/type_int.result

@@ -20,5 +20,18 @@ COALESCE(@a)
 1
 DROP TABLE t1;
 #
+# MDEV-21065 UNIQUE constraint causes a query with string comparison to omit a row in the result set
+#
+CREATE TABLE t1 (c0 INT UNIQUE);
+INSERT INTO t1 VALUES (NULL), (NULL), (NULL), (NULL), (1), (0);
+SELECT * FROM t1 WHERE c0 < '\n2';
+c0
+0
+1
+DROP TABLE t1;
+SELECT CAST('\n2' AS INT);
+CAST('\n2' AS INT)
+2
+#
 # End of 5.5 tests
 #

mysql-test/t/type_int.test

@@ -14,6 +14,18 @@ SELECT COALESCE(@a:=1) FROM t1 ORDER BY STRCMP(STDDEV_SAMP(a), 'bar');
 SELECT COALESCE(@a) FROM t1 ORDER BY STRCMP(STDDEV_SAMP(a), 'bar');
 DROP TABLE t1;
 
+--echo #
+--echo # MDEV-21065 UNIQUE constraint causes a query with string comparison to omit a row in the result set
+--echo #
+
+CREATE TABLE t1 (c0 INT UNIQUE);
+INSERT INTO t1 VALUES (NULL), (NULL), (NULL), (NULL), (1), (0);
+SELECT * FROM t1 WHERE c0 < '\n2';
+DROP TABLE t1;
+
+SELECT CAST('\n2' AS INT);
+
+
 --echo #
 --echo # End of 5.5 tests
 --echo #

strings/ctype-simple.c

@@ -1399,7 +1399,8 @@ my_strntoull10rnd_8bit(CHARSET_INFO *cs __attribute__((unused)),
   int shift= 0, digits= 0, negative, addon;
 
   /* Skip leading spaces and tabs */
-  for ( ; str < end && (*str == ' ' || *str == '\t') ; str++);
+  for ( ; str < end && my_isspace(&my_charset_latin1, *str) ; )
+    str++;
 
   if (str >= end)
     goto ret_edom;

strings/my_strtoll10.c

@@ -98,18 +98,25 @@ longlong my_strtoll10(const char *nptr, char **endptr, int *error)
   if (endptr)
   {
     end= *endptr;
-    while (s != end && (*s == ' ' || *s == '\t'))
+    /* Skip leading spaces */
+    for ( ; s < end && my_isspace(&my_charset_latin1, *s) ; )
       s++;
+
     if (s == end)
       goto no_conv;
   }
   else
   {
     endptr= &dummy;				/* Easier end test */
-    while (*s == ' ' || *s == '\t')
-      s++;
-    if (!*s)
-      goto no_conv;
+    /* Skip leading spaces */
+    for ( ; ; s++)
+    {
+      if (!*s)
+        goto no_conv;
+      if (!my_isspace(&my_charset_latin1, *s))
+        break;
+    }
+
     /* This number must be big to guard against a lot of pre-zeros */
     end= s+65535;				/* Can't be longer than this */
   }