Optimise float() also for bytes/bytearray values that contain underscores (PEP-515).

Cython/Utility/Optimize.c

@@ -704,19 +704,48 @@ static double __Pyx_SlowPyString_AsDouble(PyObject *obj) {
     return (double)-1;
 }
 
+static const char* __Pyx__PyBytes_AsDouble_Copy(const char* start, char* buffer, Py_ssize_t length) {
+    Py_ssize_t i;
+    char *digit = buffer;
+    for (i=0; i < length; i++) {
+        if (start[i] != '_') *buffer++ = start[i];
+    }
+    *buffer = '\0';
+    return buffer;
+}
+
 static double __Pyx__PyBytes_AsDouble(PyObject *obj, const char* start, Py_ssize_t length) {
-    if (likely(!memchr(start, '_', length))) {
-        const char *end, *last = start + length;
-        double value;
-        while (Py_ISSPACE(*start))
-            start++;
-        while (start < last - 1 && Py_ISSPACE(last[-1]))
-            last--;
-        value = PyOS_string_to_double(start, (char**) &end, NULL);
-        if (likely(end == last) || (value == (double)-1 && PyErr_Occurred())) {
-            return value;
-        }
+    double value;
+    Py_ssize_t i, digits;
+    const char *last = start + length;
+    char *end;
+    // strip spaces at start and end
+    while (Py_ISSPACE(*start))
+        start++;
+    while (start < last - 1 && Py_ISSPACE(last[-1]))
+        last--;
+    length = last - start;
+    // look for underscores
+    digits = 0;
+    for (i=0; i < length; digits += start[i++] != '_');
+
+    if (likely(digits == length)) {
+        value = PyOS_string_to_double(start, &end, NULL);
+    } else if (digits < 40) {
+        char number[40];
+        last = __Pyx__PyBytes_AsDouble_Copy(start, number, length);
+        value = PyOS_string_to_double(number, &end, NULL);
+    } else {
+        char *number = (char*) PyMem_Malloc((digits + 1) * sizeof(char));
+        if (unlikely(!number)) goto fallback;
+        last = __Pyx__PyBytes_AsDouble_Copy(start, number, length);
+        value = PyOS_string_to_double(number, &end, NULL);
+        PyMem_Free(number);
+    }
+    if (likely(end == last) || (value == (double)-1 && PyErr_Occurred())) {
+        return value;
     }
+fallback:
     return __Pyx_SlowPyString_AsDouble(obj);
 }
 

tests/run/builtin_float.py

@@ -2,6 +2,18 @@
 # tag: pure3.0
 
 import cython
+import sys
+
+def fix_underscores(s):
+    if sys.version_info < (3, 6):
+        # Py2 float() does not support PEP-515 underscore literals
+        if isinstance(s, bytes):
+            if not cython.compiled and b'_' in s:
+                return s.replace(b'_', b'')
+        elif '_' in s:
+            return s.replace('_', '')
+    return s
+
 
 def empty_float():
     """
@@ -42,6 +54,10 @@ def from_bytes(s: bytes):
     123.0
     >>> from_bytes(b"123.25")
     123.25
+    >>> from_bytes(fix_underscores(b"98_5_6.2_1"))
+    9856.21
+    >>> from_bytes(fix_underscores(b"12_4_131_123123_1893798127398123_19238712_128937198237.8222113_519879812387"))
+    1.2413112312318938e+47
     >>> from_bytes(b"123E100")
     1.23e+102
     """
@@ -55,9 +71,29 @@ def from_bytes(s: bytes):
 def from_bytes_literals():
     """
     >>> from_bytes_literals()
-    (123.0, 123.23, 1e+100)
+    (123.0, 123.23, 123.76, 1e+100)
+    """
+    return float(b"123"), float(b"123.23"), float(fix_underscores(b"12_3.7_6")), float(b"1e100")
+
+
+@cython.test_assert_path_exists(
+    "//CoerceToPyTypeNode",
+    "//CoerceToPyTypeNode//PythonCapiCallNode",
+)
+def from_bytearray(s: bytearray):
     """
-    return float(b"123"), float(b"123.23"), float(b"1e100")
+    >>> from_bytearray(bytearray(b"123"))
+    123.0
+    >>> from_bytearray(bytearray(b"123.25"))
+    123.25
+    >>> from_bytearray(bytearray(fix_underscores(b"98_5_6.2_1")))
+    9856.21
+    >>> from_bytearray(bytearray(fix_underscores(b"12_4_131_123123_1893798127398123_19238712_128937198237.8222113_519879812387")))
+    1.2413112312318938e+47
+    >>> from_bytearray(bytearray(b"123E100"))
+    1.23e+102
+    """
+    return float(s)
 
 
 @cython.test_assert_path_exists(
@@ -70,6 +106,10 @@ def from_str(s: 'str'):
     123.0
     >>> from_str("123.25")
     123.25
+    >>> from_str(fix_underscores("3_21.2_5"))
+    321.25
+    >>> from_str(fix_underscores("12_4_131_123123_1893798127398123_19238712_128937198237.8222113_519879812387"))
+    1.2413112312318938e+47
     >>> from_str("123E100")
     1.23e+102
     """
@@ -83,9 +123,9 @@ def from_str(s: 'str'):
 def from_str_literals():
     """
     >>> from_str_literals()
-    (123.0, 123.23, 1e+100)
+    (123.0, 123.23, 124.23, 1e+100)
     """
-    return float("123"), float("123.23"), float("1e100")
+    return float("123"), float("123.23"), float(fix_underscores("1_2_4.2_3")), float("1e100")
 
 
 @cython.test_assert_path_exists(
@@ -98,6 +138,10 @@ def from_unicode(s: 'unicode'):
     123.0
     >>> from_unicode(u"123.25")
     123.25
+    >>> from_unicode(fix_underscores(u"12_4.8_5"))
+    124.85
+    >>> from_unicode(fix_underscores(u"12_4_131_123123_1893798127398123_19238712_128937198237.8222113_519879812387"))
+    1.2413112312318938e+47
     >>> from_unicode(u"123E100")
     1.23e+102
     >>> from_unicode(u"123.23\\N{PUNCTUATION SPACE}")
@@ -113,6 +157,6 @@ def from_unicode(s: 'unicode'):
 def from_unicode_literals():
     """
     >>> from_unicode_literals()
-    (123.0, 123.23, 1e+100, 123.23)
+    (123.0, 123.23, 123.45, 1e+100, 123.23)
     """
-    return float(u"123"), float(u"123.23"), float(u"1e100"), float(u"123.23\N{PUNCTUATION SPACE}")
+    return float(u"123"), float(u"123.23"), float(fix_underscores(u"12_3.4_5")), float(u"1e100"), float(u"123.23\N{PUNCTUATION SPACE}")