pyx api: Provide go

- Add go functionality to libgolang runtime;
- Implement go for thread and gevent runtimes.

  * Thread runtime uses PyThread_start_new_thread which, if used
    carefully, does not depend on Python GIL and on e.g. POSIX reduces to
    tiny wrapper around pthread_create.

  * Gevent runtime uses gevent's Greenlet in Pyx mode. This turns
    gevent to be a build-time dependency.

- Provide low-level _taskgo in C client API;
- Provide type-safe C++-level go wrapper over _taskgo;
- Switch golang.go from py implementation into Pyx wrapper over
  Pyx/nogil API.

This is the first patch that adds Pyx/C++/C-level unit tests and hooks
them into golang_test.py .

This patch brings the following speedup to Python-level interface:

 (on i7@2.6GHz)

thread runtime:

    name             old time/op  new time/op  delta
    go               93.0µs ± 1%  26.6µs ± 1%  -71.41%  (p=0.000 n=10+10)
    chan             13.6µs ± 2%  13.7µs ± 1%     ~     (p=0.280 n=10+10)
    select           29.9µs ± 4%  29.3µs ± 2%   -1.89%  (p=0.017 n=10+9)
    def              61.0ns ± 0%  55.0ns ± 0%   -9.84%  (p=0.000 n=10+10)
    func_def         43.8µs ± 1%  44.0µs ± 1%   +0.66%  (p=0.006 n=10+10)
    call             62.5ns ± 1%  64.0ns ± 0%   +2.40%  (p=0.000 n=10+8)
    func_call        1.06µs ± 1%  1.09µs ± 1%   +2.72%  (p=0.000 n=10+10)
    try_finally       137ns ± 0%   139ns ± 2%   +1.17%  (p=0.033 n=10+10)
    defer            2.34µs ± 1%  2.36µs ± 1%   +0.84%  (p=0.015 n=10+10)
    workgroup_empty  96.1µs ± 1%  58.1µs ± 1%  -39.55%  (p=0.000 n=10+10)
    workgroup_raise   135µs ± 1%    73µs ± 1%  -45.97%  (p=0.000 n=10+10)

gevent runtime:

    name             old time/op  new time/op  delta
    go               68.8µs ± 1%  28.6µs ± 0%  -58.47%  (p=0.000 n=10+8)
    chan             14.8µs ± 1%  15.8µs ± 1%   +6.19%  (p=0.000 n=10+10)
    select           32.0µs ± 0%  33.1µs ± 1%   +3.25%  (p=0.000 n=10+10)
    def              58.0ns ± 0%  55.0ns ± 0%   -5.17%  (p=0.000 n=10+10)
    func_def         43.9µs ± 1%  44.4µs ± 2%   +1.20%  (p=0.007 n=10+10)
    call             63.5ns ± 1%  64.0ns ± 2%     ~     (p=0.307 n=10+10)
    func_call        1.08µs ± 1%  1.06µs ± 0%   -2.55%  (p=0.000 n=10+8)
    try_finally       142ns ± 0%   136ns ± 0%   -4.23%  (p=0.000 n=10+9)
    defer            2.32µs ± 1%  2.29µs ± 1%   -0.96%  (p=0.000 n=10+10)
    workgroup_empty  90.3µs ± 0%  73.8µs ± 1%  -18.29%  (p=0.000 n=10+10)
    workgroup_raise   108µs ± 1%    94µs ± 0%  -13.29%  (p=0.000 n=10+10)

(small changes are probably within noise; "go" and "workgroup_*" should be
representative)

README.rst

@@ -170,13 +170,23 @@ located in `src/` under `$GOPATH`.
 Cython/nogil API
 ----------------
 
-Cython package `golang` provides *nogil* API with
-features that mirror corresponding Python package. Cython API is not only
+Cython package `golang` provides *nogil* API with goroutines and
+other features that mirror corresponding Python package. Cython API is not only
 faster compared to Python version, but also, due to *nogil* property, allows to
 build concurrent systems without limitations imposed by Python's GIL. All that
 while still programming in Python-like language. Brief description of
 Cython/nogil API follows:
 
+`go` spawns new task - a coroutine, or thread, depending on activated runtime.
+For example::
+
+   cdef nogil:
+      void worker():
+         pass
+
+      void myfunc():
+         go(worker)
+
 `panic` stops normal execution of current goroutine by throwing a C-level
 exception. On Python/C boundaries C-level exceptions have to be converted to
 Python-level exceptions with `topyexc`. For example::

golang/.gitignore

 /_golang.cpp
+/_golang_test.cpp
 /_time.cpp

golang/__init__.py

@@ -43,11 +43,6 @@ import six
 from golang._pycompat import im_class
 
 
-from ._golang import    \
-    _PanicError,            \
-    pypanic     as panic
-
-
 # @func is a necessary decorator for functions for selected golang features to work.
 #
 # For example it is required by defer. Usage:
@@ -177,22 +172,13 @@ def defer(f):
 
 
 
+# ---- go + channels ----
 
-# go spawns lightweight thread.
-#
-# go spawns:
-#
-# - lightweight thread (with    gevent integration), or
-# - full OS thread     (without gevent integration).
-#
-# Use gpython to run Python with integrated gevent, or use gevent directly to do so.
-def go(f, *argv, **kw):
-    t = threading.Thread(target=f, args=argv, kwargs=kw)
-    t.daemon = True # leaked goroutines don't prevent program to exit
-    t.start()
-
+from ._golang import    \
+    pygo        as go,      \
+    _PanicError,            \
+    pypanic     as panic
 
-# ---- channels ----
 
 # _RecvWaiting represents a receiver waiting on a chan.
 class _RecvWaiting(object):

golang/_golang.pxd

@@ -22,6 +22,7 @@
 Cython/nogil API
 ----------------
 
+- `go` spawns lightweight thread.
 - `panic` stops normal execution of current goroutine by throwing a C-level exception.
 
 Everything in Cython/nogil API do not depend on Python runtime and in
@@ -55,6 +56,8 @@ cdef extern from "golang/libgolang.h" namespace "golang" nogil:
     void panic(const char *)
     const char *recover()
 
+    void go(...)    # typechecking is done by C
+
 
 # ---- python bits ----
 

golang/_golang.pyx

@@ -31,9 +31,11 @@ from __future__ import print_function, absolute_import
 # init libgolang runtime early
 _init_libgolang()
 
-from cpython cimport PY_MAJOR_VERSION
+from cpython cimport Py_INCREF, Py_DECREF, PY_MAJOR_VERSION
 from cython cimport final
 
+import sys
+
 # ---- panic ----
 
 @final
@@ -64,6 +66,72 @@ cdef extern from "golang/libgolang.h" nogil:
     const char *recover_ "golang::recover" () except +
 
 
+# ---- go ----
+
+# go spawns lightweight thread.
+#
+# go spawns:
+#
+# - lightweight thread (with    gevent integration), or
+# - full OS thread     (without gevent integration).
+#
+# Use gpython to run Python with integrated gevent, or use gevent directly to do so.
+def pygo(f, *argv, **kw):
+    _ = _togo(); _.f = f; _.argv = argv; _.kw    = kw
+    Py_INCREF(_)    # we transfer 1 ref to _goviac
+    with nogil:
+        _taskgo_pyexc(_goviac, <void*>_)
+
+@final
+cdef class _togo:
+    cdef object f
+    cdef tuple  argv
+    cdef dict   kw
+
+cdef extern from "Python.h" nogil:
+    ctypedef struct PyGILState_STATE:
+        pass
+    PyGILState_STATE PyGILState_Ensure()
+    void PyGILState_Release(PyGILState_STATE)
+
+cdef void _goviac(void *arg) nogil:
+    # create new py thread state and keep it alive while __goviac runs.
+    #
+    # Just `with gil` is not enough: for `with gil` if exceptions could be
+    # raised inside, cython generates several GIL release/reacquire calls.
+    # This way the thread state will be deleted on first release and _new_ one
+    # - _another_ thread state - create on acquire. All that implicitly with
+    # the effect of loosing things associated with thread state - e.g. current
+    # exception.
+    #
+    # -> be explicit and manually keep py thread state alive ourselves.
+    gstate = PyGILState_Ensure() # py thread state will stay alive until PyGILState_Release
+    __goviac(arg)
+    PyGILState_Release(gstate)
+
+cdef void __goviac(void *arg) nogil:
+    with gil:
+        try:
+            _ = <_togo>arg
+            Py_DECREF(_)
+            _.f(*_.argv, **_.kw)
+        except:
+            # ignore exceptions during python interpreter shutdown.
+            # python clears sys and other modules at exit which can result in
+            # arbitrary exceptions in still alive "daemon" threads that go
+            # spawns. Similarly to threading.py(*) we just ignore them.
+            #
+            # if we don't - there could lots of output like e.g. "lost sys.stderr"
+            # and/or "sys.excepthook is missing" etc.
+            #
+            # (*) github.com/python/cpython/tree/v2.7.16-121-g53639dd55a0/Lib/threading.py#L760-L778
+            #     see also "Technical details" in stackoverflow.com/a/12807285/9456786.
+            if sys is None:
+                return
+
+            raise   # XXX exception -> exit program with traceback (same as in go) ?
+
+
 # ---- init libgolang runtime ---
 
 cdef extern from "golang/libgolang.h" namespace "golang" nogil:
@@ -93,3 +161,15 @@ cdef void _init_libgolang() except*:
     if runtime_ops == NULL:
         pypanic("init: %s: libgolang_runtime_ops=NULL" % runtimemod)
     _libgolang_init(runtime_ops)
+
+
+
+# ---- misc ----
+
+cdef extern from "golang/libgolang.h" namespace "golang" nogil:
+    void _taskgo(void (*f)(void *), void *arg)
+
+cdef nogil:
+
+    void _taskgo_pyexc(void (*f)(void *) nogil, void *arg)      except +topyexc:
+        _taskgo(f, arg)

golang/_golang_test.pyx

+# -*- coding: utf-8 -*-
+# cython: language_level=2
+# distutils: language=c++
+#
+# Copyright (C) 2018-2019  Nexedi SA and Contributors.
+#                          Kirill Smelkov <kirr@nexedi.com>
+#
+# This program is free software: you can Use, Study, Modify and Redistribute
+# it under the terms of the GNU General Public License version 3, or (at your
+# option) any later version, as published by the Free Software Foundation.
+#
+# You can also Link and Combine this program with other software covered by
+# the terms of any of the Free Software licenses or any of the Open Source
+# Initiative approved licenses and Convey the resulting work. Corresponding
+# source of such a combination shall include the source code for all other
+# software used.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See COPYING file for full licensing terms.
+# See https://www.nexedi.com/licensing for rationale and options.
+
+# small tests that verifies pyx-level channel API.
+# the work of channels themselves is exercised thoroughly mostly in golang_test.py
+
+from __future__ import print_function, absolute_import
+
+from golang cimport go, panic, topyexc
+
+
+# small test to verify pyx(nogil) go.
+cdef void _test_go_nogil() nogil except +topyexc:
+    go(_work, 111)
+    # TODO wait till _work is done
+cdef void _work(int i) nogil:
+    if i != 111:
+        panic("_work: i != 111")
+
+def test_go_nogil():
+    with nogil:
+        _test_go_nogil()
+
+
+# runtime/libgolang_test_c.c
+cdef extern from * nogil:
+    """
+    extern "C" void _test_go_c();
+    """
+    void _test_go_c()   except +topyexc
+def test_go_c():
+    with nogil:
+        _test_go_c()
+
+# runtime/libgolang_test.cpp
+cdef extern from * nogil:
+    """
+    extern void _test_go_cpp();
+    """
+    void _test_go_cpp()                         except +topyexc
+def test_go_cpp():
+    with nogil:
+        _test_go_cpp()

golang/golang_test.py

@@ -33,6 +33,19 @@ import golang
 from golang import _chan_recv, _chan_send
 from golang._pycompat import im_class
 
+# pyx/c/c++ tests -> test_pyx_*
+from golang import _golang_test
+for f in dir(_golang_test):
+    if f.startswith('test_'):
+        gf = 'test_pyx_' + f[len('test_'):] # test_chan_nogil -> test_pyx_chan_nogil
+        # define a python function with gf name (if we use f directly pytest
+        # will say "cannot collect 'test_pyx_chan_nogil' because it is not a function")
+        def _(func=getattr(_golang_test, f)):
+            func()
+        _.__name__ = gf
+        globals()[gf] = _
+
+
 # leaked goroutine behaviour check: done in separate process because we need
 # to test process termination exit there.
 def test_go_leaked():

golang/libgolang.h

@@ -22,8 +22,8 @@
 
 // Library Libgolang provides Go-like features for C and C++.
 //
-// Library Libgolang provides Go-like
-// features. The library consists of high-level type-safe C++ API,
+// Library Libgolang provides goroutines and other
+// accompanying features. The library consists of high-level type-safe C++ API,
 // and low-level unsafe C API.
 //
 // The primary motivation for Libgolang is to serve as runtime for golang.pyx -
@@ -34,17 +34,21 @@
 //
 // C++-level API
 //
+//  - `go` spawns new task.
 //  - `sleep` pauses current task.
 //  - `panic` throws exception that represent C-level panic.
 //
 // For example:
 //
+//      go(worker, 1);                  // spawn worker(int)
+//
 //      if (<bug condition>)
 //          panic("bug");
 //
 //
 // C-level API
 //
+//  - `_taskgo` spawns new task.
 //  - `tasknanosleep` pauses current task.
 //
 //
@@ -96,12 +100,16 @@ extern "C" {
 LIBGOLANG_API void panic(const char *arg);
 LIBGOLANG_API const char *recover(void);
 
+LIBGOLANG_API void _taskgo(void (*f)(void *arg), void *arg);
 LIBGOLANG_API void _tasknanosleep(uint64_t dt);
 LIBGOLANG_API uint64_t _nanotime(void);
 
 
 // libgolang runtime - the runtime must be initialized before any other libgolang use.
 typedef struct _libgolang_runtime_ops {
+    // go should spawn a task (coroutine/thread/...).
+    void    (*go)(void (*f)(void *), void *arg);
+
     // nanosleep should pause current goroutine for at least dt nanoseconds.
     // nanosleep(0) is not noop - such call must be at least yielding to other goroutines.
     void        (*nanosleep)(uint64_t dt);
@@ -123,8 +131,24 @@ LIBGOLANG_API void _libgolang_init(const _libgolang_runtime_ops *runtime_ops);
 
 #ifdef __cplusplus
 
+#include <functional>
+#include <memory>
+
 namespace golang {
 
+// go provides type-safe wrapper over _taskgo.
+template<typename F, typename... Argv>  // F = std::function<void(Argv...)>
+static inline void go(F /*std::function<void(Argv...)>*/ f, Argv... argv) {
+    typedef std::function<void(void)> Frun;
+    Frun *frun = new Frun (std::bind(f, argv...));
+    _taskgo([](void *_frun) {
+        std::unique_ptr<Frun> frun (reinterpret_cast<Frun*>(_frun));
+        (*frun)();
+        // frun deleted here on normal exit or panic.
+    }, frun);
+}
+
+
 namespace time {
 
 // sleep pauses current goroutine for at least dt seconds.

golang/pyx/build.py

@@ -170,10 +170,12 @@ def Extension(name, sources, **kw):
     if exists(venv_inc):
         kw['include_dirs'].append(venv_inc)
 
-    # provide POSIX/... defines to Cython
+    # provide POSIX/PYPY/... defines to Cython
     POSIX = ('posix' in sys.builtin_module_names)
+    PYPY  = (platform.python_implementation() == 'PyPy')
     pyxenv = kw.get('cython_compile_time_env', {})
     pyxenv.setdefault('POSIX',  POSIX)
+    pyxenv.setdefault('PYPY',   PYPY)
     kw['cython_compile_time_env'] = pyxenv
 
     # XXX hack, because setuptools_dso.Extension is not Cython.Extension

golang/pyx/testprog/golang_pyx_user/pyxuser/test.pyx

@@ -23,12 +23,19 @@
 # Small program that uses a bit of golang.pyx nogil features, mainly to verify
 # that external project can build against golang in pyx mode.
 
-from golang cimport topyexc
+from golang cimport go, topyexc
 from libc.stdio cimport printf
 
 cdef nogil:
 
+    void worker(int i, int j):
+        pass
+
     void _main() except +topyexc:
+        cdef int i
+        for i in range(3):
+            go(worker, i, 4)
+
         printf("test.pyx: OK\n")
 
 def main():

golang/runtime/_libgolang.pxd

@@ -23,6 +23,8 @@ from libc.stdint cimport uint64_t
 
 cdef extern from "golang/libgolang.h" nogil:
     struct _libgolang_runtime_ops:
+        void    (*go)(void (*f)(void *) nogil, void *arg);
+
         void        (*nanosleep)(uint64_t)
         uint64_t    (*nanotime)()
 

golang/runtime/_runtime_gevent.pyx

@@ -21,15 +21,53 @@
 
 from __future__ import print_function, absolute_import
 
+# Gevent runtime uses gevent's greenlets.
+
+IF not PYPY:
+    from gevent._greenlet cimport Greenlet
+ELSE:
+    # on pypy gevent does not compile greenlet.py citing that
+    # "there is no greenlet.h on pypy"
+    from gevent.greenlet import Greenlet
+
 from gevent import sleep as pygsleep
+
 from libc.stdint cimport uint64_t
+from cpython cimport Py_INCREF, Py_DECREF
+from cython cimport final
+
 from golang.runtime._libgolang cimport _libgolang_runtime_ops, panic
 from golang.runtime cimport _runtime_thread
 
+
+# _goviapy & _togo serve go
+def _goviapy(_togo _ not None):
+    with nogil:
+        _.f(_.arg)
+
+@final
+cdef class _togo:
+    cdef void (*f)(void *) nogil
+    cdef void *arg
+
+
 cdef nogil:
 
     # XXX better panic with pyexc object and detect that at recover side?
 
+    bint _go(void (*f)(void *), void *arg):
+        with gil:
+            _ = _togo(); _.f = f; _.arg = arg
+            g = Greenlet(_goviapy, _)
+            g.start()
+            return True
+
+    void go(void (*f)(void *), void *arg):
+        ok = _go(f, arg)
+        if not ok:
+            panic("pyxgo: gevent: go: failed")
+
+
     bint _nanosleep(uint64_t dt):
         cdef double dt_s = dt * 1E-9
         with gil:
@@ -43,6 +81,7 @@ cdef nogil:
 
     # XXX const
     _libgolang_runtime_ops gevent_ops = _libgolang_runtime_ops(
+            go              = go,
             nanosleep       = nanosleep,
             nanotime        = _runtime_thread.nanotime, # reuse from _runtime_thread
     )

golang/runtime/_runtime_thread.pyx

@@ -21,6 +21,24 @@
 
 from __future__ import print_function, absolute_import
 
+# Thread runtime reuses C-level Python threadcreate implementation
+# for portability.
+#
+# PyThread_start_new_thread - Python's C function function to create
+# new thread - does not depend on GIL. On POSIX, for example, it is small
+# wrapper around pthread_create.
+
+# make sure python threading is initialized, so that there is no concurrent
+# calls to PyThread_init_thread later.
+#
+# This allows us to treat PyThread_start_new_thread as nogil.
+from cpython.ceval cimport PyEval_InitThreads
+#PyThread_init_thread()     # initializes only threading, but _not_ GIL
+PyEval_InitThreads()        # initializes      threading       and  GIL
+cdef extern from "pythread.h" nogil:
+    # NOTE py3.7 changed to `unsigned long PyThread_start_new_thread ...`
+    long PyThread_start_new_thread(void (*)(void *), void *)
+
 from golang.runtime._libgolang cimport _libgolang_runtime_ops, panic
 
 from libc.stdint cimport uint64_t, UINT64_MAX
@@ -36,6 +54,11 @@ DEF i1E9 = 1000000000
 
 cdef nogil:
 
+    void go(void (*f)(void *), void *arg):
+        pytid = PyThread_start_new_thread(f, arg)
+        if pytid == -1:
+            panic("pygo: failed")
+
     IF POSIX:
         void nanosleep(uint64_t dt):
             cdef timespec ts
@@ -86,6 +109,7 @@ cdef nogil:
 
     # XXX const
     _libgolang_runtime_ops thread_ops = _libgolang_runtime_ops(
+            go              = go,
             nanosleep       = nanosleep,
             nanotime        = nanotime,
     )

golang/runtime/libgolang.cpp

@@ -91,6 +91,10 @@ void _libgolang_init(const _libgolang_runtime_ops *runtime_ops) {
     _runtime = runtime_ops;
 }
 
+void _taskgo(void (*f)(void *), void *arg) {
+    _runtime->go(f, arg);
+}
+
 void _tasknanosleep(uint64_t dt) {
     _runtime->nanosleep(dt);
 }

golang/runtime/libgolang_test.cpp

+// Copyright (C) 2019  Nexedi SA and Contributors.
+//                     Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+// Test that exercises C++-level libgolang.h API and functionality.
+
+#include "golang/libgolang.h"
+using namespace golang;
+
+// small test to verify C++ go.
+static void _work(int i);
+void _test_go_cpp() {
+    go(_work, 111); // not λ to test that go correctly passes arguments
+    // TODO wait till _work is done
+}
+static void _work(int i) {
+    if (i != 111)
+        panic("_work: i != 111");
+}

golang/runtime/libgolang_test_c.c

+// Copyright (C) 2019  Nexedi SA and Contributors.
+//                     Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+// Test that exercises C-level libgolang.h API.
+
+#ifdef __cplusplus
+# error "this file must be compiled with C - not C++ - compiler"
+#endif
+
+#include "golang/libgolang.h"
+#include <stdlib.h>
+
+// small test to verify C _taskgo.
+struct _work_arg{int i;};
+static void _work(void *);
+void _test_go_c(void) {
+    struct _work_arg *_ = malloc(sizeof(*_));
+    if (_ == NULL)
+        panic("malloc _work_arg -> failed");
+    _->i    = 111;
+    _taskgo(_work, _);
+    // TODO wait till _work is done
+}
+static void _work(void *__) {
+    struct _work_arg *_ = (struct _work_arg *)__;
+    if (_->i != 111)
+        panic("_work: i != 111");
+    free(_);
+}

pyproject.toml

 [build-system]
-requires = ["setuptools", "wheel", "setuptools_dso >= 1.2", "cython"]
+requires = ["setuptools", "wheel", "setuptools_dso >= 1.2", "cython", "gevent"]

setup.py

@@ -208,6 +208,11 @@ setup(
                         ['golang/runtime/_runtime_gevent.pyx'],
                         language = 'c'),
 
+                    Ext('golang._golang_test',
+                        ['golang/_golang_test.pyx',
+                         'golang/runtime/libgolang_test_c.c',
+                         'golang/runtime/libgolang_test.cpp']),
+
                     Ext('golang._time',
                         ['golang/_time.pyx']),
                   ],