Sprinkle lots of branch prediction hints in places where exceptions are being...

Sprinkle lots of branch prediction hints in places where exceptions are being handled or otherwise exceptional cases occur.

Cython/Utility/AsyncGen.c

@@ -929,7 +929,7 @@ __Pyx_async_gen_athrow_send(__pyx_PyAsyncGenAThrow *o, PyObject *arg)
     } else {
         /* aclose() mode */
         if (retval) {
-            if (__pyx__PyAsyncGenWrappedValue_CheckExact(retval)) {
+            if (unlikely(__pyx__PyAsyncGenWrappedValue_CheckExact(retval))) {
                 Py_DECREF(retval);
                 goto yield_close;
             }
@@ -983,7 +983,7 @@ __Pyx_async_gen_athrow_throw(__pyx_PyAsyncGenAThrow *o, PyObject *args)
     } else {
         // aclose() mode
         PyObject *exc_type;
-        if (retval && __pyx__PyAsyncGenWrappedValue_CheckExact(retval)) {
+        if (unlikely(retval && __pyx__PyAsyncGenWrappedValue_CheckExact(retval))) {
             o->agt_gen->ag_running_async = 0;
             o->agt_state = __PYX_AWAITABLE_STATE_CLOSED;
             Py_DECREF(retval);

Cython/Utility/Builtins.c

@@ -68,7 +68,7 @@ static PyObject* __Pyx_PyExec3(PyObject* o, PyObject* globals, PyObject* locals)
 
     if (!globals || globals == Py_None) {
         globals = $moddict_cname;
-    } else if (!PyDict_Check(globals)) {
+    } else if (unlikely(!PyDict_Check(globals))) {
         PyErr_Format(PyExc_TypeError, "exec() arg 2 must be a dict, not %.200s",
                      Py_TYPE(globals)->tp_name);
         goto bad;
@@ -78,12 +78,12 @@ static PyObject* __Pyx_PyExec3(PyObject* o, PyObject* globals, PyObject* locals)
     }
 
     if (__Pyx_PyDict_GetItemStr(globals, PYIDENT("__builtins__")) == NULL) {
-        if (PyDict_SetItem(globals, PYIDENT("__builtins__"), PyEval_GetBuiltins()) < 0)
+        if (unlikely(PyDict_SetItem(globals, PYIDENT("__builtins__"), PyEval_GetBuiltins()) < 0))
             goto bad;
     }
 
     if (PyCode_Check(o)) {
-        if (__Pyx_PyCode_HasFreeVars((PyCodeObject *)o)) {
+        if (unlikely(__Pyx_PyCode_HasFreeVars((PyCodeObject *)o))) {
             PyErr_SetString(PyExc_TypeError,
                 "code object passed to exec() may not contain free variables");
             goto bad;
@@ -99,12 +99,12 @@ static PyObject* __Pyx_PyExec3(PyObject* o, PyObject* globals, PyObject* locals)
         if (PyUnicode_Check(o)) {
             cf.cf_flags = PyCF_SOURCE_IS_UTF8;
             s = PyUnicode_AsUTF8String(o);
-            if (!s) goto bad;
+            if (unlikely(!s)) goto bad;
             o = s;
         #if PY_MAJOR_VERSION >= 3
-        } else if (!PyBytes_Check(o)) {
+        } else if (unlikely(!PyBytes_Check(o))) {
         #else
-        } else if (!PyString_Check(o)) {
+        } else if (unlikely(!PyString_Check(o))) {
         #endif
             PyErr_Format(PyExc_TypeError,
                 "exec: arg 1 must be string, bytes or code object, got %.200s",
@@ -180,7 +180,7 @@ static CYTHON_INLINE int __Pyx_HasAttr(PyObject *o, PyObject *n) {
         return -1;
     }
     r = __Pyx_GetAttr(o, n);
-    if (unlikely(!r)) {
+    if (!r) {
         PyErr_Clear();
         return 0;
     } else {
@@ -196,7 +196,7 @@ static PyObject* __Pyx_Intern(PyObject* s); /* proto */
 //////////////////// Intern ////////////////////
 
 static PyObject* __Pyx_Intern(PyObject* s) {
-    if (!(likely(PyString_CheckExact(s)))) {
+    if (unlikely(!PyString_CheckExact(s))) {
         PyErr_Format(PyExc_TypeError, "Expected %.16s, got %.200s", "str", Py_TYPE(s)->tp_name);
         return 0;
     }

Cython/Utility/Coroutine.c

@@ -522,7 +522,7 @@ static int __Pyx_PyGen__FetchStopIterationValue(CYTHON_UNUSED PyThreadState *$lo
             value = Py_None;
         }
 #if PY_VERSION_HEX >= 0x030300A0
-        else if (__Pyx_IS_TYPE(ev, (PyTypeObject*)PyExc_StopIteration)) {
+        else if (likely(__Pyx_IS_TYPE(ev, (PyTypeObject*)PyExc_StopIteration))) {
             value = ((PyStopIterationObject *)ev)->value;
             Py_INCREF(value);
             Py_DECREF(ev);
@@ -905,7 +905,7 @@ static int __Pyx_Coroutine_CloseIter(__pyx_CoroutineObject *gen, PyObject *yf) {
         } else {
             retval = PyObject_CallFunction(meth, NULL);
             Py_DECREF(meth);
-            if (!retval)
+            if (unlikely(!retval))
                 err = -1;
         }
         gen->is_running = 0;
@@ -1079,7 +1079,7 @@ static PyObject *__Pyx_Coroutine_Throw(PyObject *self, PyObject *args) {
     PyObject *val = NULL;
     PyObject *tb = NULL;
 
-    if (!PyArg_UnpackTuple(args, (char *)"throw", 1, 3, &typ, &val, &tb))
+    if (unlikely(!PyArg_UnpackTuple(args, (char *)"throw", 1, 3, &typ, &val, &tb)))
         return NULL;
 
     return __Pyx__Coroutine_Throw(self, typ, val, tb, args, 1);
@@ -1130,10 +1130,10 @@ static void __Pyx_Coroutine_dealloc(PyObject *self) {
         // Generator is paused or unstarted, so we need to close
         PyObject_GC_Track(self);
 #if PY_VERSION_HEX >= 0x030400a1 && CYTHON_USE_TP_FINALIZE
-        if (PyObject_CallFinalizerFromDealloc(self))
+        if (unlikely(PyObject_CallFinalizerFromDealloc(self)))
 #else
         Py_TYPE(gen)->tp_del(self);
-        if (self->ob_refcnt > 0)
+        if (unlikely(self->ob_refcnt > 0))
 #endif
         {
             // resurrected.  :(
@@ -1249,7 +1249,7 @@ static void __Pyx_Coroutine_del(PyObject *self) {
     // Undo the temporary resurrection; can't use DECREF here, it would
     // cause a recursive call.
     assert(self->ob_refcnt > 0);
-    if (--self->ob_refcnt == 0) {
+    if (likely(--self->ob_refcnt == 0)) {
         // this is the normal path out
         return;
     }
@@ -2061,7 +2061,7 @@ static int __Pyx_patch_abc(void) {
     if (CYTHON_REGISTER_ABCS && !abc_patched) {
         PyObject *module;
         module = PyImport_ImportModule((PY_MAJOR_VERSION >= 3) ? "collections.abc" : "collections");
-        if (!module) {
+        if (unlikely(!module)) {
             PyErr_WriteUnraisable(NULL);
             if (unlikely(PyErr_WarnEx(PyExc_RuntimeWarning,
                     ((PY_MAJOR_VERSION >= 3) ?
@@ -2319,7 +2319,7 @@ static int __pyx_StopAsyncIteration_init(void) {
     __Pyx_PyExc_StopAsyncIteration = (PyObject*) __Pyx_FetchCommonType(&__Pyx__PyExc_StopAsyncIteration_type);
     if (unlikely(!__Pyx_PyExc_StopAsyncIteration))
         return -1;
-    if (builtins && unlikely(PyMapping_SetItemString(builtins, (char*) "StopAsyncIteration", __Pyx_PyExc_StopAsyncIteration) < 0))
+    if (likely(builtins) && unlikely(PyMapping_SetItemString(builtins, (char*) "StopAsyncIteration", __Pyx_PyExc_StopAsyncIteration) < 0))
         return -1;
 #endif
     return 0;

Cython/Utility/CythonFunction.c

@@ -279,7 +279,7 @@ __Pyx_CyFunction_set_defaults(__pyx_CyFunctionObject *op, PyObject* value, CYTHO
     if (!value) {
         // del => explicit None to prevent rebuilding
         value = Py_None;
-    } else if (value != Py_None && !PyTuple_Check(value)) {
+    } else if (unlikely(value != Py_None && !PyTuple_Check(value))) {
         PyErr_SetString(PyExc_TypeError,
                         "__defaults__ must be set to a tuple object");
         return -1;
@@ -294,7 +294,7 @@ __Pyx_CyFunction_get_defaults(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *co
     PyObject* result = op->defaults_tuple;
     if (unlikely(!result)) {
         if (op->defaults_getter) {
-            if (__Pyx_CyFunction_init_defaults(op) < 0) return NULL;
+            if (unlikely(__Pyx_CyFunction_init_defaults(op) < 0)) return NULL;
             result = op->defaults_tuple;
         } else {
             result = Py_None;
@@ -309,7 +309,7 @@ __Pyx_CyFunction_set_kwdefaults(__pyx_CyFunctionObject *op, PyObject* value, CYT
     if (!value) {
         // del => explicit None to prevent rebuilding
         value = Py_None;
-    } else if (value != Py_None && !PyDict_Check(value)) {
+    } else if (unlikely(value != Py_None && !PyDict_Check(value))) {
         PyErr_SetString(PyExc_TypeError,
                         "__kwdefaults__ must be set to a dict object");
         return -1;
@@ -324,7 +324,7 @@ __Pyx_CyFunction_get_kwdefaults(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *
     PyObject* result = op->defaults_kwdict;
     if (unlikely(!result)) {
         if (op->defaults_getter) {
-            if (__Pyx_CyFunction_init_defaults(op) < 0) return NULL;
+            if (unlikely(__Pyx_CyFunction_init_defaults(op) < 0)) return NULL;
             result = op->defaults_kwdict;
         } else {
             result = Py_None;
@@ -338,7 +338,7 @@ static int
 __Pyx_CyFunction_set_annotations(__pyx_CyFunctionObject *op, PyObject* value, CYTHON_UNUSED void *context) {
     if (!value || value == Py_None) {
         value = NULL;
-    } else if (!PyDict_Check(value)) {
+    } else if (unlikely(!PyDict_Check(value))) {
         PyErr_SetString(PyExc_TypeError,
                         "__annotations__ must be set to a dict object");
         return -1;
@@ -696,7 +696,7 @@ static CYTHON_INLINE int __Pyx_CyFunction_Vectorcall_CheckArgs(__pyx_CyFunctionO
         }
         ret = 1;
     }
-    if (unlikely(kwnames) && PyTuple_GET_SIZE(kwnames)) {
+    if (unlikely(kwnames) && unlikely(PyTuple_GET_SIZE(kwnames))) {
         PyErr_Format(PyExc_TypeError,
                      "%.200s() takes no keyword arguments", cyfunc->func.m_ml->ml_name);
         return -1;
@@ -1090,7 +1090,7 @@ __pyx_FusedFunction_descr_get(PyObject *self, PyObject *obj, PyObject *type)
                     ((PyCFunctionObject *) func)->m_module,
                     ((__pyx_CyFunctionObject *) func)->func_globals,
                     ((__pyx_CyFunctionObject *) func)->func_code);
-    if (!meth)
+    if (unlikely(!meth))
         return NULL;
 
     // defaults needs copying fully rather than just copying the pointer
@@ -1100,9 +1100,10 @@ __pyx_FusedFunction_descr_get(PyObject *self, PyObject *obj, PyObject *type)
         PyObject **pydefaults;
         int i;
 
-        if (!__Pyx_CyFunction_InitDefaults((PyObject*)meth,
-                                      func->func.defaults_size,
-                                      func->func.defaults_pyobjects)) {
+        if (unlikely(!__Pyx_CyFunction_InitDefaults(
+                (PyObject*)meth,
+                func->func.defaults_size,
+                func->func.defaults_pyobjects))) {
             Py_XDECREF((PyObject*)meth);
             return NULL;
         }
@@ -1147,7 +1148,7 @@ __pyx_FusedFunction_getitem(__pyx_FusedFunctionObject *self, PyObject *idx)
     PyObject *unbound_result_func;
     PyObject *result_func = NULL;
 
-    if (self->__signatures__ == NULL) {
+    if (unlikely(self->__signatures__ == NULL)) {
         PyErr_SetString(PyExc_TypeError, "Function is not fused");
         return NULL;
     }
@@ -1159,7 +1160,7 @@ __pyx_FusedFunction_getitem(__pyx_FusedFunctionObject *self, PyObject *idx)
         PyObject *sep = NULL;
         int i;
 
-        if (!list)
+        if (unlikely(!list))
             return NULL;
 
         for (i = 0; i < n; i++) {
@@ -1172,14 +1173,14 @@ __pyx_FusedFunction_getitem(__pyx_FusedFunctionObject *self, PyObject *idx)
 #if !(CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS)
             Py_DECREF(item);
 #endif
-            if (!string || PyList_Append(list, string) < 0)
+            if (unlikely(!string) || unlikely(PyList_Append(list, string) < 0))
                 goto __pyx_err;
 
             Py_DECREF(string);
         }
 
         sep = PyUnicode_FromString("|");
-        if (sep)
+        if (likely(sep))
             signature = PyUnicode_Join(sep, list);
 __pyx_err:
 ;
@@ -1189,12 +1190,12 @@ __pyx_err:
         signature = _obj_to_str(idx);
     }
 
-    if (!signature)
+    if (unlikely(!signature))
         return NULL;
 
     unbound_result_func = PyObject_GetItem(self->__signatures__, signature);
 
-    if (unbound_result_func) {
+    if (likely(unbound_result_func)) {
         if (self->self) {
             __pyx_FusedFunctionObject *unbound = (__pyx_FusedFunctionObject *) unbound_result_func;
 
@@ -1251,7 +1252,7 @@ __pyx_FusedFunction_call(PyObject *func, PyObject *args, PyObject *kw)
         PyObject *self;
         Py_ssize_t i;
         new_args = PyTuple_New(argc + 1);
-        if (!new_args)
+        if (unlikely(!new_args))
             return NULL;
 
         self = binding_func->self;
@@ -1426,7 +1427,7 @@ static int __pyx_FusedFunction_init(void) {
     __pyx_FusedFunctionType_type.tp_base = __pyx_CyFunctionType;
     __pyx_FusedFunctionType = __Pyx_FetchCommonType(&__pyx_FusedFunctionType_type);
 #endif
-    if (__pyx_FusedFunctionType == NULL) {
+    if (unlikely(__pyx_FusedFunctionType == NULL)) {
         return -1;
     }
     return 0;
@@ -1452,9 +1453,9 @@ static PyObject* __Pyx_Method_ClassMethod(PyObject *method) {
 #else
     // It appears that PyMethodDescr_Type is not exposed anywhere in the CPython C-API
     static PyTypeObject *methoddescr_type = NULL;
-    if (methoddescr_type == NULL) {
+    if (unlikely(methoddescr_type == NULL)) {
        PyObject *meth = PyObject_GetAttrString((PyObject*)&PyList_Type, "append");
-       if (!meth) return NULL;
+       if (unlikely(!meth)) return NULL;
        methoddescr_type = Py_TYPE(meth);
        Py_DECREF(meth);
     }

Cython/Utility/FunctionArguments.c

@@ -355,7 +355,7 @@ static int __Pyx_MergeKeywords(PyObject *kwdict, PyObject *source_mapping) {
     if (unlikely(!iter)) {
         // slow fallback: try converting to dict, then iterate
         PyObject *args;
-        if (!PyErr_ExceptionMatches(PyExc_AttributeError)) goto bad;
+        if (unlikely(!PyErr_ExceptionMatches(PyExc_AttributeError))) goto bad;
         PyErr_Clear();
         args = PyTuple_Pack(1, source_mapping);
         if (likely(args)) {