Add PyMem_(Raw)Calloc to cpython.mem (GH-4086)

CPython added PyMem_RawCalloc and PyMem_Calloc in version 3.5.

The #defines of PyMem_RawMalloc, PyMem_RawRealloc, and PyMem_RawFree
are removed from Cython/Utility/ModuleSetupCode.c since the Raw versions allow
use without the GIL but the non-Raw versions do not.
Co-authored-by: William Schwartz <wkschwartz@gmail.com>

Closes https://github.com/cython/cython/pull/3050
Closes https://github.com/cython/cython/pull/3047

Cython/Includes/cpython/mem.pxd

@@ -35,6 +35,15 @@ cdef extern from "Python.h":
     # PyMem_Malloc(1) had been called instead. The memory will not
     # have been initialized in any way.
 
+    void* PyMem_RawCalloc(size_t nelem, size_t elsize) nogil
+    void* PyMem_Calloc(size_t nelem, size_t elsize)
+    # Allocates nelem elements each whose size in bytes is elsize and
+    # returns a pointer of type void* to the allocated memory, or NULL if
+    # the request fails. The memory is initialized to zeros. Requesting
+    # zero elements or elements of size zero bytes returns a distinct
+    # non-NULL pointer if possible, as if PyMem_Calloc(1, 1) had been
+    # called instead.
+
     void* PyMem_RawRealloc(void *p, size_t n) nogil
     void* PyMem_Realloc(void *p, size_t n)
     # Resizes the memory block pointed to by p to n bytes. The
@@ -43,13 +52,13 @@ cdef extern from "Python.h":
     # else if n is equal to zero, the memory block is resized but is
     # not freed, and the returned pointer is non-NULL. Unless p is
     # NULL, it must have been returned by a previous call to
-    # PyMem_Malloc() or PyMem_Realloc().
+    # PyMem_Malloc(), PyMem_Realloc(), or PyMem_Calloc().
 
     void PyMem_RawFree(void *p) nogil
     void PyMem_Free(void *p)
     # Frees the memory block pointed to by p, which must have been
-    # returned by a previous call to PyMem_Malloc() or
-    # PyMem_Realloc(). Otherwise, or if PyMem_Free(p) has been called
+    # returned by a previous call to PyMem_Malloc(), PyMem_Realloc(), or
+    # PyMem_Calloc(). Otherwise, or if PyMem_Free(p) has been called
     # before, undefined behavior occurs. If p is NULL, no operation is
     # performed.
 

Cython/Utility/ModuleSetupCode.c

@@ -633,12 +633,6 @@ class __Pyx_FakeReference {
   #define PyObject_Realloc(p)  PyMem_Realloc(p)
 #endif
 
-#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030400A1
-  #define PyMem_RawMalloc(n)           PyMem_Malloc(n)
-  #define PyMem_RawRealloc(p, n)       PyMem_Realloc(p, n)
-  #define PyMem_RawFree(p)             PyMem_Free(p)
-#endif
-
 #if CYTHON_COMPILING_IN_LIMITED_API
   #define __Pyx_PyCode_HasFreeVars(co)  (PyCode_GetNumFree(co) > 0)
   #define __Pyx_PyFrame_SetLineNumber(frame, lineno)

runtests.py

@@ -484,6 +484,7 @@ VER_DEP_MODULES = {
                                          'run.pep526_variable_annotations',  # typing module
                                          'run.test_exceptions',  # copied from Py3.7+
                                          'run.time_pxd',  # _PyTime_GetSystemClock doesn't exist in 3.4
+                                         'run.cpython_capi_py35',
                                          'embedding.embedded',  # From the docs, needs Py_DecodeLocale
                                          ]),
     (3,7): (operator.lt, lambda x: x in ['run.pycontextvar',

tests/run/cpython_capi.pyx

@@ -25,31 +25,6 @@ def test_pymalloc():
         mem.PyMem_Free(m)
 
 
-def test_pymalloc_raw():
-    """
-    >>> test_pymalloc_raw()
-    3
-    """
-    cdef char* m
-    cdef char* m2 = NULL
-    with nogil:
-        m = <char*> mem.PyMem_RawMalloc(20)
-        if not m:
-            raise MemoryError()
-        try:
-            m[0] = 1
-            m[1] = 2
-            m[2] = 3
-            m2 = <char*> mem.PyMem_RawRealloc(m, 10)
-            if m2:
-                m = m2
-            retval = m[2]
-        finally:
-            mem.PyMem_RawFree(m)
-    assert m2
-    return retval
-
-
 def test_gilstate():
     """
     >>> test_gilstate()

tests/run/cpython_capi_py35.pyx

+# mode: run
+# tag: c-api
+
+# PyMem_RawMalloc tests that need to be disabled for Python < 3.5
+# (some of these would work of Python 3.4, but it's easier to disable
+# them in one place)
+
+from cpython cimport mem
+
+cdef short _assert_calloc(short* s, int n) except -1 with gil:
+    """Assert array ``s`` of length ``n`` is zero and return 3."""
+    assert not s[0] and not s[n - 1]
+    s[0] += 1
+    s[n - 1] += 3
+    for i in range(1, n - 1):
+        assert not s[i]
+    return s[n - 1]
+
+def test_pycalloc():
+    """
+    >>> test_pycalloc()
+    3
+    """
+    cdef short* s = <short*> mem.PyMem_Calloc(10, sizeof(short))
+    if not s:
+        raise MemoryError()
+    try:
+        return _assert_calloc(s, 10)
+    finally:
+        mem.PyMem_Free(s)
+
+def test_pymalloc_raw():
+    """
+    >>> test_pymalloc_raw()
+    3
+    """
+    cdef short i
+    cdef short* s
+    cdef char* m
+    cdef char* m2 = NULL
+    with nogil:
+        s = <short*> mem.PyMem_RawCalloc(10, sizeof(short))
+        if not s:
+            raise MemoryError()
+        try:
+            i = _assert_calloc(s, 10)
+        finally:
+            mem.PyMem_RawFree(s)
+        m = <char*> mem.PyMem_RawMalloc(20)
+        if not m:
+            raise MemoryError()
+        try:
+            m[0] = 1
+            m[1] = 2
+            m[2] = i
+            m2 = <char*> mem.PyMem_RawRealloc(m, 10)
+            if m2:
+                m = m2
+            retval = m[2]
+        finally:
+            mem.PyMem_RawFree(m)
+    assert m2
+    return retval