Merge pull request #29 from samrushing/modernize

Modernize pyrex -> cython

coro/_coro.pxd

+# -*- Mode: Cython -*-
+
+from cpython.ref cimport PyObject
+from libc cimport intptr_t, memcpy, memset, off_t, time_t, timeval, uint64_t, uintptr_t
+
+cdef extern from "Python.h":
+
+    ctypedef struct PyCodeObject:
+        int       co_argcount
+        int       co_nlocals
+        int       co_stacksize
+        int       co_flags
+        PyObject *co_code
+        PyObject *co_consts
+        PyObject *co_names
+        PyObject *co_varnames
+        PyObject *co_freevars
+        PyObject *co_cellvars
+        PyObject *co_filename
+        PyObject *co_name
+        int       co_firstlineno
+        PyObject *co_lnotab
+
+    int PyCode_Addr2Line(PyCodeObject *, int)
+
+cdef extern from "frameobject.h":
+    ctypedef struct PyFrameObject:
+        PyFrameObject *f_back
+        PyCodeObject  *f_code
+        PyObject *f_builtins
+        PyObject *f_globals
+        PyObject *f_locals
+        PyObject *f_trace
+        PyObject *f_exc_type
+        PyObject *f_exc_value
+        PyObject *f_exc_traceback
+        int f_lasti
+        int f_lineno
+        int f_restricted
+        int f_iblock
+        int f_nlocals
+        int f_ncells
+        int f_nfreevars
+        int f_stacksize
+
+cdef enum:
+    EVENT_SCALE = 16384
+
+# used by the profiler
+cdef struct call_stack:
+    # Next is NULL for the end of the linked list.
+    call_stack * next
+    void * func     # Really a <function>
+    void * b        # Really a <bench>
+
+cdef struct machine_state:
+    void * stack_pointer
+    void * frame_pointer
+    void * insn_pointer
+    # other registers that need saving
+    #          #  x86    amd64
+    void * r1  #  ebx     rbx
+    void * r2  #  esi     r12
+    void * r3  #  edi     r13
+    void * r4  #          r14
+    void * r5  #          r15
+
+# from swap.c
+cdef extern int __swap (void * ts, void * fs)
+cdef extern object void_as_object (void * p)
+cdef extern int frame_getlineno (object frame)
+cdef extern int coro_breakpoint()
+cdef extern int SHRAP_STACK_PAD
+
+cdef int default_selfishness
+
+cdef int live_coros
+
+cdef public class coro [ object _coro_object, type _coro_type ]:
+
+    """The coroutine object.
+
+    Do not create this object directly.  Use either :func:`new` or
+    :func:`spawn` to create one.
+    """
+
+    cdef machine_state state
+    cdef object fun
+    cdef object args, kwargs
+    cdef readonly bytes name
+    cdef public int id
+    # XXX think about doing these as a bitfield/property
+    cdef public unsigned char dead, started, scheduled
+    cdef public object value
+    cdef void * stack_copy
+    cdef size_t stack_size
+    cdef PyFrameObject * frame
+    cdef void * saved_exception_data[6]
+    # used only by the profiler, a call_stack object. NULL if the profiler is
+    # not enabled or if this is the first call of the coroutine.
+    cdef call_stack * top
+    cdef int saved_recursion_depth
+    cdef int selfish_acts, max_selfish_acts
+    cdef bint compress, compressed
+    cdef object waiting_joiners
+    # Used for thread-local-storage.
+    cdef dict _tdict
+    cdef __create (self)
+    cdef __destroy (self)
+    cdef __yield (self)
+    cdef __resume (self, value)
+    cdef save_exception_data (self)
+    cdef restore_exception_data (self)
+    cdef _schedule (self, value)
+    cdef _unschedule (self)
+    cdef _die (self)
+    cdef __interrupt (self, the_exception)
+    cdef int try_selfish (self)
+
+# choose a library for stack compression
+IF COMPILE_LZ4:
+    include "zstack_lz4.pxd"
+ELIF COMPILE_LZO:
+    include "zstack_lzo.pxd"
+ELSE:
+    include "zstack_zlib.pxd"
+
+cdef public class sched [ object sched_object, type sched_type ]:
+    cdef machine_state state
+    # this is the stack that all coroutines run on
+    cdef void * stack_base
+    cdef int stack_size
+    cdef public list pending, staging
+    cdef public object _current
+    cdef coro _last
+    cdef int profiling
+    cdef uint64_t latency_threshold
+    cdef zstack squish
+    cdef object events
+    cdef _preserve_last (self)
+    cdef _restore (self, coro co)
+    cdef _schedule (self, coro co, object value)
+    cdef _unschedule (self, coro co)
+    cdef print_latency_warning (self, coro co, uint64_t delta)
+    cdef sleep (self, uint64_t when)
+    cdef schedule_ready_events (self, uint64_t now)
+    cdef get_timeout_to_next_event (self, int default_timeout)
+
+include "socket.pxd"
+# XXX need pxd files for sync.pyx, poller.pyx, etc...

coro/_coro.pyx

@@ -30,39 +30,29 @@ variables are documented in the top level of the coro package ``__init__.py``.
 
 __coro_version__ = "$Id: //prod/main/ap/shrapnel/coro/_coro.pyx#114 $"
 
-DEF CORO_DEBUG = 0
-DEF COMPILE_LIO = 0
-DEF COMPILE_NETDEV = 0
-DEF COMPILE_LZO = 0
-DEF COMPILE_LZ4 = 0
 
 import coro as coro_package
 import warnings
 # Only import things from libc that are very common and have unique names.
-from libc cimport intptr_t,\
-                  memcpy,\
-                  memset,\
-                  off_t,\
-                  time_t,\
-                  timeval,\
-                  uint64_t,\
-                  uintptr_t
+from libc cimport intptr_t, memcpy, memset, off_t, time_t, timeval, uint64_t, uintptr_t
 
 # ================================================================================
 # a re-implementation of the IronPort coro-threading system, this time
-# in Pyrex, and using stack copying and switching with a stripped-down
+# in Cython, and using stack copying and switching with a stripped-down
 # version of the 'set/getcontext' API.
 # ================================================================================
 
 # XXX: blame jj behrens for this.
 # XXX: instead of a two-stack solution, think about an n-stack solution.
-#      [the main drawback is that each coro is then tied to a particular stack]
+#      [the main drawback is that each coro is then tied to a particular stack...
+#       this might be appropriate for a system that needs a small number of high-priority
+#       threads that never get swapped out, e.g, a TCP implementation]
 
 # ================================================================================
 #                        external declarations
 # ================================================================================
 
-include "python.pxi"
+#include "python.pxi"
 # Note that this cimports libc.
 include "pyrex_helpers.pyx"
 include "tsc_time_include.pyx"
@@ -75,8 +65,17 @@ cdef extern from "stdlib.h":
     ELSE:
         void srandomdev()
 
+from cpython.ref cimport Py_DECREF, Py_INCREF
+from cpython.mem cimport PyMem_Free, PyMem_Malloc
+from cpython.list cimport PyList_New
+from cpython.bytes cimport PyBytes_FromStringAndSize
+
 cdef extern from "Python.h":
-    # hack
+    ctypedef struct PyThreadState:
+        PyFrameObject * frame
+        int recursion_depth
+        void * curexc_type, * curexc_value, * curexc_traceback
+        void * exc_type, * exc_value, * exc_traceback
     PyThreadState * _PyThreadState_Current
 
 # ================================================================================
@@ -91,7 +90,7 @@ _ticks_per_sec = tsc_time_module.ticks_per_sec
 # to a Long.
 ticks_per_sec = _ticks_per_sec
 
-cdef object _all_threads
+cdef dict _all_threads
 _all_threads = {}
 all_threads = _all_threads
 
@@ -151,9 +150,6 @@ include "fifo.pyx"
 #                       coroutine/context object
 # ================================================================================
 
-cdef enum:
-    EVENT_SCALE = 16384
-
 import sys
 
 class ScheduleError (Exception):
@@ -194,7 +190,7 @@ cdef extern int coro_breakpoint()
 cdef extern int SHRAP_STACK_PAD
 
 # forward
-cdef public class sched  [ object sched_object, type sched_type ]
+#cdef public class sched  [ object sched_object, type sched_type ]
 cdef public class queue_poller [ object queue_poller_object, type queue_poller_type ]
 cdef sched the_scheduler "the_scheduler"
 cdef queue_poller the_poller "the_poller"
@@ -223,27 +219,6 @@ cdef public class coro [ object _coro_object, type _coro_type ]:
     :func:`spawn` to create one.
     """
 
-    cdef machine_state state
-    cdef object fun
-    cdef object args, kwargs
-    cdef readonly object name
-    # XXX think about doing these as a bitfield/property
-    cdef public int dead, started, id, scheduled
-    cdef public object value
-    cdef void * stack_copy
-    cdef size_t stack_size
-    cdef PyFrameObject * frame
-    cdef void * saved_exception_data[6]
-    # used only by the profiler, a call_stack object. NULL if the profiler is
-    # not enabled or if this is the first call of the coroutine.
-    cdef call_stack * top
-    cdef int saved_recursion_depth
-    cdef int selfish_acts, max_selfish_acts
-    cdef bint compress, compressed
-    cdef object waiting_joiners
-    # Used for thread-local-storage.
-    cdef object _tdict
-
     def __init__ (self, fun, args, kwargs, int id, name=None):
         global live_coros
         self.fun = fun
@@ -260,7 +235,7 @@ cdef public class coro [ object _coro_object, type _coro_type ]:
         self.selfish_acts = default_selfishness
         self.max_selfish_acts = default_selfishness
         if name is None:
-            self.name = 'coro %d' % (self.id,)
+            self.name = b'coro %d' % (self.id,)
         live_coros = live_coros + 1
 
     def __dealloc__ (self):
@@ -731,7 +706,7 @@ cdef int get_coro_id() except -1:
         next_coro_id = next_coro_id + 1
         if next_coro_id == libc.INT_MAX:
             next_coro_id = 1
-        if not PyDict_Contains(_all_threads, result):
+        if not _all_threads.has_key (result):
             return result
 
 def default_exception_notifier():
@@ -901,16 +876,6 @@ ELSE:
 # ================================================================================
 
 cdef public class sched [ object sched_object, type sched_type ]:
-    cdef machine_state state
-    # this is the stack that all coroutines run on
-    cdef void * stack_base
-    cdef int stack_size
-    cdef public object _current, pending, staging
-    cdef coro _last
-    cdef int profiling
-    cdef uint64_t latency_threshold
-    cdef zstack squish
-    cdef object events
 
     def __init__ (self, stack_size=4*1024*1024):
         self.stack_size = stack_size
@@ -1003,7 +968,7 @@ cdef public class sched [ object sched_object, type sched_type ]:
             raise ScheduleError, self
         else:
             co.scheduled = 1
-            PyList_Append (self.pending, (co, value))
+            self.pending.append ((co, value))
 
     cdef _unschedule (self, coro co):
         """Unschedule this coroutine.
@@ -1014,16 +979,16 @@ cdef public class sched [ object sched_object, type sched_type ]:
         """
         cdef int i
         for i from 0 <= i < len(self.pending):
-            co2, v2 = PySequence_GetItem (self.pending, i)
+            co2, v2 = self.pending[i]
             if co is co2:
-                PySequence_DelItem (self.pending, i)
+                del self.pending[i]
                 co.scheduled = 0
                 return True
         else:
             for i from 0 <= i < len(self.staging):
-                co2, v2 = PySequence_GetItem (self.staging, i)
+                co2, v2 = self.staging[i]
                 if co is co2:
-                    PySequence_SetItem (self.staging, i, (None, None))
+                    self.staging[i] = (None, None)
                     co.scheduled = 0
                     return True
             else:
@@ -1128,11 +1093,11 @@ cdef public class sched [ object sched_object, type sched_type ]:
         self.events.insert (e.t, e)
         try:
             try:
-                return PyObject_Call (function, args, kwargs)
+                return function (*args, **kwargs)
             except Interrupted, value:
                 # is this *my* timebomb?
                 args = value.args
-                if (PyTuple_Size(args) > 0) and (PySequence_GetItem (args, 0) is tb):
+                if len(args) > 0 and args[0] is tb:
                     raise TimeoutError
                 else:
                     raise
@@ -1258,6 +1223,7 @@ cdef public class sched [ object sched_object, type sched_type ]:
         cdef coro co
         cdef uint64_t _now
         cdef object _coro_package
+        cdef tuple x
 
         # Make a cdef reference to avoid __Pyx_GetName.
         _coro_package = coro_package
@@ -1270,12 +1236,11 @@ cdef public class sched [ object sched_object, type sched_type ]:
                 _coro_package.now_usec = c_ticks_to_usec(_now)
                 self.schedule_ready_events (_now)
                 while 1:
-                    if PyList_GET_SIZE (self.pending):
+                    if len(self.pending) > 0:
                         self.staging, self.pending = self.pending, self.staging
-                        for i from 0 <= i < PyList_GET_SIZE (self.staging):
-                            x = PyList_GET_ITEM_SAFE (self.staging, i)
-                            co = PyTuple_GET_ITEM_SAFE (x, 0)
-                            value = PyTuple_GET_ITEM_SAFE (x, 1)
+                        for i from 0 <= i < len (self.staging):
+                            x = self.staging[i]
+                            co, value = x
                             # co may be None if it was unscheduled.
                             if co is not None:
                                 #W ('resuming %d: #%d\n' % (i, co.id))
@@ -1465,17 +1430,19 @@ cdef void info(int sig):
     co = the_scheduler._current
     frame = _PyThreadState_Current.frame
     if co:
-        libc.fprintf(libc.stderr, 'coro %i "%s" at %s: %s %i\n',
+        libc.fprintf (
+            libc.stderr, 'coro %i "%s" at %s: %s %i\n',
             co.id,
-            PyString_AsString (co.name),
-            PyString_AsString (<object>frame.f_code.co_filename),
-            PyString_AsString (<object>frame.f_code.co_name),
+            co.name,
+            <bytes>frame.f_code.co_filename,
+            <bytes>frame.f_code.co_name,
             PyCode_Addr2Line  (frame.f_code, frame.f_lasti)
             )
     else:
-        libc.fprintf(libc.stderr, 'No current coro. %s: %s %i\n',
-            PyString_AsString (<object>frame.f_code.co_filename),
-            PyString_AsString (<object>frame.f_code.co_name),
+        libc.fprintf (
+            libc.stderr, 'No current coro. %s: %s %i\n',
+            <bytes>frame.f_code.co_filename,
+            <bytes>frame.f_code.co_name,
             PyCode_Addr2Line  (frame.f_code, frame.f_lasti)
             )
 

coro/local.pyx

@@ -130,12 +130,12 @@ cdef class ThreadLocal:
         cdef coro co
 
         # _all_threads.itervalues() might be better here.
-        for co in PyDict_Values(_all_threads):
+        for co in _all_threads.values():
             if co._tdict is not None:
                 # Avoiding exceptions for performance.
                 # Would be much better to just call PyDict_DelItem, but we've
                 # defined it with except -1.
-                if PyDict_Contains(co._tdict, self.key):
+                if co._tdict.has_key (self.key):
                     del co._tdict[self.key]
 
     property __dict__:

coro/poller.pyx

@@ -269,7 +269,7 @@ cdef public class queue_poller [ object queue_poller_object, type queue_poller_t
     cdef kevent * change_list
     cdef int change_list_index
     cdef int kq_fd
-    cdef object event_map
+    cdef dict event_map
 
     def __cinit__ (self):
         # XXX EVENT_SCALE should be a parameter.
@@ -298,7 +298,7 @@ cdef public class queue_poller [ object queue_poller_object, type queue_poller_t
         cdef coro me
         cdef kevent_target kt
         if self.change_list_index < EVENT_SCALE:
-            if PyDict_Contains (self.event_map, kk):
+            if self.event_map.has_key (kk):
                 # Should be impossible to have KeyError due to previous line.
                 kt = self.event_map[kk]
                 raise SimultaneousError (the_scheduler._current, kt.target, kk)
@@ -397,7 +397,7 @@ cdef public class queue_poller [ object queue_poller_object, type queue_poller_t
         if r < 0:
             raise_oserror()
 
-    def set_handler (self, object event, object handler, int flags=(EV_ADD|EV_ONESHOT), unsigned int fflags=0):
+    def set_handler (self, tuple event, object handler, int flags=(EV_ADD|EV_ONESHOT), unsigned int fflags=0):
         """Add a kevent handler.
 
         This is a low-level interface to register a kevent handler.
@@ -419,11 +419,10 @@ cdef public class queue_poller [ object queue_poller_object, type queue_poller_t
         cdef kevent_key kk
         assert callable(handler)
 
-        ident = PySequence_GetItem(event, 0)
-        filter = PySequence_GetItem(event, 1)
+        ident, filter = event
         kk = kevent_key (filter, ident)
         # for kqueue, event == (ident, filter)
-        if PyDict_Contains (self.event_map, kk):
+        if self.event_map.has_key (kk):
             # Should be impossible to have KeyError due to previous line.
             kt = self.event_map[kk]
             raise SimultaneousError (the_scheduler._current, kt.target, kk)
@@ -437,13 +436,12 @@ cdef public class queue_poller [ object queue_poller_object, type queue_poller_t
             else:
                 raise SystemError, "too many kevents in change_list"
 
-    cdef set_event_target (self, object event, kevent_target kt):
+    cdef set_event_target (self, tuple event, kevent_target kt):
         cdef short filter
         cdef uintptr_t ident
         cdef kevent_key kk
 
-        ident = PySequence_GetItem(event, 0)
-        filter = PySequence_GetItem(event, 1)
+        ident, filter = event
         kk = kevent_key (filter, ident)
         self.event_map[kk] = kt
 
@@ -453,7 +451,7 @@ cdef public class queue_poller [ object queue_poller_object, type queue_poller_t
         cdef kevent_key kk
 
         kk = kevent_key (EVFILT_READ, fd)
-        if PyDict_Contains(self.event_map, kk):
+        if self.event_map.has_key (kk):
             kt = self.event_map[kk]
             kt.flags = kt.flags | KTARGET_CLOSED
             del self.event_map[kk]
@@ -465,7 +463,7 @@ cdef public class queue_poller [ object queue_poller_object, type queue_poller_t
                 W ('notify_of_close (%d) [read]: unable to interrupt thread: %r\n' % (fd, co))
 
         kk = kevent_key (EVFILT_WRITE, fd)
-        if PyDict_Contains(self.event_map, kk):
+        if self.event_map.has_key (kk):
             kt = self.event_map[kk]
             kt.flags = kt.flags | KTARGET_CLOSED
             del self.event_map[kk]

coro/profile.pyx

@@ -133,7 +133,7 @@ cdef class internal_function(function):
     cdef object name
 
     def __hash__(self):
-        return PyObject_Hash(self.name)
+        return hash (self.name)
 
     def __cmp__(x, y):
         if not isinstance(x, function) or not isinstance(y, function):
@@ -169,7 +169,7 @@ cdef class python_function(function):
     cdef object code
 
     def __hash__(self):
-        return PyObject_Hash(self.code)
+        return hash (self.code)
 
     def __cmp__(x, y):
         if not isinstance(x, function) or not isinstance(y, function):
@@ -216,7 +216,7 @@ cdef class c_function(function):
     cdef uintptr_t func_addr
 
     def __hash__(self):
-        return PyObject_Hash(self.func_addr)
+        return hash (self.func_addr)
 
     def __cmp__(x, y):
         cdef uintptr_t a,b
@@ -559,7 +559,8 @@ cdef class _profiler:
     """
 
     cdef bench counter, main_bench, wait_bench
-    cdef public object charges, call_counts, bench_class, func_cache
+    cdef public dict charges, call_counts, func_cache
+    cdef public object bench_class
     cdef uint64_t start_ticks
 
     def __init__ (self, bench_class):
@@ -619,7 +620,7 @@ cdef class _profiler:
 
         self.counter.mark(<bench>top.b)
 
-        if PyDict_Contains (self.charges, <function>top.func):
+        if self.charges.has_key (<function>top.func):
             b = self.charges[<function>top.func]
         else:
             b = self.bench_class()
@@ -659,7 +660,7 @@ cdef class _profiler:
         :Return:
             Returns a `call_counts_object`.
         """
-        if PyDict_Contains (self.call_counts, func) == 0:
+        if not self.call_counts.has_key (func):
             cc = call_counts_object()
             self.call_counts[func] = cc
             return cc
@@ -686,7 +687,7 @@ cdef class _profiler:
             Returns a `python_function` object.
         """
         code_ptr = <intptr_t><void *> code
-        if PyDict_Contains (self.func_cache, code_ptr):
+        if self.func_cache.has_key (code_ptr):
             return self.func_cache[code_ptr]
         else:
             func = new_python_function(code)
@@ -705,7 +706,7 @@ cdef class _profiler:
             Returns a `c_function` object.
         """
         c_func_ptr = <intptr_t>PyCFunction_GetFunction(c_func)
-        if PyDict_Contains (self.func_cache, c_func_ptr):
+        if self.func_cache.has_key (c_func_ptr):
             return self.func_cache[c_func_ptr]
         else:
             func = new_c_function(c_func)

coro/socket.pxd

+# -*- Mode: Cython -*-
+
+IF UNAME_SYSNAME == "Linux":
+    cdef extern from "stdint.h":
+        ctypedef unsigned char uint8_t
+        ctypedef unsigned short uint16_t
+        ctypedef unsigned int uint32_t
+ELSE:
+    from libc cimport uint8_t, uint16_t, uint32_t
+
+cdef extern from "netinet/in.h":
+    IF UNAME_SYSNAME == "Linux":
+        cdef struct in_addr:
+            uint32_t s_addr
+        cdef struct sockaddr_in:
+            short sin_family
+            unsigned short sin_port
+            in_addr sin_addr
+            char sin_zero[8]
+    ELSE:
+        pass
+
+cdef extern from "sys/un.h":
+    IF UNAME_SYSNAME == "Linux":
+        cdef struct sockaddr_un:
+            short sun_family
+            char sun_path[104]
+    ELSE:
+        pass
+
+cdef extern from "arpa/inet.h":
+    cdef enum:
+        INET_ADDRSTRLEN
+        INET6_ADDRSTRLEN
+
+    int htons (int)
+    int htonl (int)
+    int ntohl (int)
+    int ntohs (int)
+
+cdef extern from "sys/socket.h":
+    int AF_UNSPEC, AF_INET, AF_INET6, AF_UNIX
+    int SOCK_STREAM, SOCK_DGRAM, SOL_SOCKET, INADDR_ANY
+    int SHUT_RD, SHUT_WR, SHUT_RDWR
+
+    int SO_DEBUG, SO_REUSEADDR, SO_KEEPALIVE, SO_DONTROUTE, SO_LINGER
+    int SO_BROADCAST, SO_OOBINLINE, SO_SNDBUF, SO_RCVBUF, SO_SNDLOWAT
+    int SO_RCVLOWAT, SO_SNDTIMEO, SO_RCVTIMEO, SO_TYPE, SO_ERROR
+    IF UNAME_SYSNAME == "FreeBSD":
+        int SO_REUSEPORT, SO_ACCEPTFILTER
+
+    int SO_DONTROUTE, SO_LINGER, SO_BROADCAST, SO_OOBINLINE, SO_SNDBUF
+    int SO_REUSEADDR, SO_DEBUG, SO_RCVBUF, SO_SNDLOWAT, SO_RCVLOWAT
+    int SO_SNDTIMEO, SO_RCVTIMEO, SO_KEEPALIVE, SO_TYPE, SO_ERROR
+
+    ctypedef unsigned int sa_family_t
+    ctypedef unsigned int in_port_t
+    ctypedef unsigned int in_addr_t
+    ctypedef unsigned int socklen_t
+
+    cdef struct in_addr:
+        in_addr_t s_addr
+
+    union ip__u6_addr:
+        uint8_t  __u6_addr8[16]
+        uint16_t __u6_addr16[8]
+        uint32_t __u6_addr32[4]
+
+    struct in6_addr:
+        ip__u6_addr __u6_addr
+
+    IF UNAME_SYSNAME == "FreeBSD" or UNAME_SYSNAME == "Darwin":
+        cdef struct sockaddr:
+            unsigned char sa_len
+            sa_family_t sa_family
+            char sa_data[250]
+
+        cdef struct sockaddr_in:
+            unsigned char sin_len
+            sa_family_t sin_family
+            in_port_t sin_port
+            in_addr sin_addr
+            char sin_zero[8]
+
+        cdef struct sockaddr_in6:
+            unsigned char sin6_len
+            sa_family_t sin6_family
+            in_port_t sin6_port
+            unsigned int sin6_flowinfo
+            in6_addr sin6_addr
+            unsigned int sin6_scope_id
+
+        cdef struct sockaddr_un:
+            unsigned char sun_len
+            sa_family_t sun_family
+            char sun_path[104]
+
+        cdef struct sockaddr_storage:
+            unsigned char sa_len
+            sa_family_t sa_family
+    ELSE:
+        cdef struct sockaddr:
+            sa_family_t sa_family
+            char sa_data[250]
+
+        cdef struct sockaddr_in:
+            sa_family_t sin_family
+            unsigned short sin_port
+            in_addr sin_addr
+            char sa_data[250]
+
+        cdef struct sockaddr_in6:
+            sa_family_t sin6_family
+            unsigned short sin6_port
+            in6_addr sin6_addr
+            char sa_data[250]
+
+        cdef struct sockaddr_storage:
+            sa_family_t sa_family
+            char sa_data[250]
+
+    int socket      (int domain, int type, int protocol)
+    int connect     (int fd, sockaddr * addr, socklen_t addr_len)
+    int accept      (int fd, sockaddr * addr, socklen_t * addr_len)
+    int bind        (int fd, sockaddr * addr, socklen_t addr_len)
+    int listen      (int fd, int backlog)
+    int shutdown    (int fd, int how)
+    int close       (int fd)
+    int getsockopt  (int fd, int level, int optname, void * optval, socklen_t * optlen)
+    int setsockopt  (int fd, int level, int optname, void * optval, socklen_t optlen)
+    int getpeername (int fd, sockaddr * name, socklen_t * namelen)
+    int getsockname (int fd, sockaddr * name, socklen_t * namelen)
+    int sendto      (int fd, void * buf, size_t len, int flags, sockaddr * addr, socklen_t addr_len)
+    int send        (int fd, void * buf, size_t len, int flags)
+    int recv        (int fd, void * buf, size_t len, int flags)
+    int recvfrom    (int fd, void * buf, size_t len, int flags, sockaddr * addr, socklen_t * addr_len)
+    int _c_socketpair "socketpair"  (int d, int type, int protocol, int *sv)
+    int inet_pton   (int af, char *src, void *dst)
+    char *inet_ntop (int af, void *src, char *dst, socklen_t size)
+    char * inet_ntoa (in_addr pin)
+    int inet_aton   (char * cp, in_addr * pin)
+
+cdef extern from "sys/uio.h":
+    cdef struct iovec:
+        void * iov_base
+        size_t iov_len
+
+cdef extern from "unistd.h":
+    size_t write (int fd, char * buf, size_t nbytes)
+    size_t read  (int fd, char * buf, size_t nbytes)
+    size_t writev(int d, iovec *iov, int iovcnt)
+    size_t readv (int d, iovec *iov, int iovcnt)
+
+cdef extern from "fcntl.h":
+    int fcntl (int fd, int cmd, ...)
+    int F_GETFL, O_NONBLOCK, F_SETFL
+
+cdef public class sock [ object sock_object, type sock_type ]:
+    cdef public int fd, orig_fd, domain, stype
+    #def __init__ (self, int domain=AF_INET, int stype=SOCK_STREAM, int protocol=0, int fd=-1)
+    cdef int _try_selfish(self) except -1
+    cdef _set_reuse_addr (self)
+    cdef set_nonblocking (self)
+    cdef parse_address (self, object address, sockaddr_storage * sa, socklen_t * addr_len, bint resolve=?)
+    cdef parse_address_inet (self, tuple address, sockaddr_storage * sa, socklen_t * addr_len, bint resolve)
+    cdef parse_address_inet6 (self, tuple address, sockaddr_storage * sa, socklen_t * addr_len, bint resolve)
+    cdef parse_address_unix (self, bytes address, sockaddr_storage * sa, socklen_t * addr_len, bint resolve)
+    cdef object unparse_address (self, sockaddr_storage *sa, socklen_t addr_len)
+    cdef _wait_for_read (self)
+    cdef _wait_for_write (self)
+    cpdef connect_addr (self, address, bint resolve=?)
+    cpdef connect (self, address)
+    cpdef recv (self, int buffer_size)
+    cpdef read (self, int buffer_size)
+    cpdef recvfrom (self, int buffer_size, int flags=?)
+    cpdef recv_exact (self, int bytes)
+    cpdef readv (self, list size_list)
+    cpdef writev (self, list data)
+    # XXX is there a cpython.type for buffer objects?
+    IF False:
+        cpdef recv_into (self, buffer, int nbytes=?, int flags=?)
+        cpdef recvfrom_into(self, buffer, int nbytes=?, int flags=?)
+    cpdef bind (self, address)
+    cpdef listen (self, int backlog)
+    cpdef accept (self)
+    cpdef accept_many (self, int max=?)
+    cpdef shutdown (self, int how)
+    cpdef getpeername (self)
+    cpdef getsockname (self)
+    cpdef dup(self)
+
+cdef class file_sock (sock):
+    cdef object _fileobj
+

coro/sync.pyx

@@ -25,6 +25,8 @@ __sync_version__ = "$Id: //prod/main/ap/shrapnel/coro/sync.pyx#29 $"
 
 # Note: this file is included by <coro.pyx>
 
+from cpython.list cimport PyList_New
+
 # ================================================================================
 #                      synchronization primitives
 # ================================================================================
@@ -757,11 +759,12 @@ cdef class fifo:
             empty.
         """
         cdef int i
+        cdef list result
         while self.fifo.size == 0:
             self.cv.wait()
         result = [None] * self.fifo.size
         i = 0
         while self.fifo.size:
-            PySequence_SetItem (result, i, self.fifo._pop())
+            result[i] = self.fifo._pop()
             i = i + 1
         return result

coro/zstack_zlib.pxd

+# -*- Mode: Cython -*-
+
+cimport zlib
+
+cdef class zstack:
+    cdef zlib.z_stream squish, unsquish
+    # this buffer is only used for compression
+    cdef unsigned char * buffer
+    cdef int buffer_size
+    cdef size_t deflate (self, void * base, size_t size)
+    cdef size_t inflate (self, void * dst, size_t dsize, void * src, size_t ssize)

coro/zstack_zlib.pyx

@@ -6,10 +6,6 @@ class ZlibError (Exception):
 cimport zlib
 
 cdef class zstack:
-    cdef zlib.z_stream squish, unsquish
-    # this buffer is only used for compression
-    cdef unsigned char * buffer
-    cdef int buffer_size
     def __init__ (self, int size=4*1024*1024):
         cdef int r
         self.squish.zalloc = NULL

setup.py

@@ -20,7 +20,6 @@ except ImportError:
         )
     sys.exit (-1)
 
-
 include_dir = os.getcwd()
 
 def newer(x, y):
@@ -47,6 +46,14 @@ def check_lio():
     status = os.system('test/build/test_lio')
     return exit_ok(status)
 
+compile_time_env = {
+    'COMPILE_LIO': check_lio(),
+    'COMPILE_NETDEV' : False,
+    'COMPILE_LZO' : False,
+    'COMPILE_LZ4' : False,
+    'CORO_DEBUG': False,
+    }
+
 setup (
     name='coro',
     version='1.0.2-000',
@@ -69,14 +76,14 @@ setup (
             'coro._coro',
             ['coro/_coro.pyx', 'coro/swap.c'],
             extra_compile_args = ['-Wno-unused-function'],
-            depends=(glob.glob('coro/*.pyx') +
+            depends = (
+                glob.glob('coro/*.pyx') +
                 glob.glob('coro/*.pxi') +
-                     glob.glob('coro/*.pxd') +
-                     [os.path.join(include_dir, 'pyrex', 'python.pxi'),
+                glob.glob('coro/*.pxd') + [
+                    os.path.join(include_dir, 'pyrex', 'python.pxi'),
                     os.path.join(include_dir, 'pyrex', 'pyrex_helpers.pyx'),
                     os.path.join(include_dir, 'include', 'pyrex_helpers.h'),
-                      os.path.join(include_dir, 'pyrex',
-                                   'tsc_time_include.pyx'),
+                    os.path.join(include_dir, 'pyrex', 'tsc_time_include.pyx'),
                     os.path.join(include_dir, 'include', 'tsc_time.h'),
                     os.path.join(include_dir, 'pyrex', 'libc.pxd'),
                     ]
@@ -85,24 +92,18 @@ setup (
                 os.path.join(include_dir, '.'),
                 os.path.join(include_dir, 'pyrex'),
                 ],
-            #include_dirs=[os.path.join(include_dir, 'pyrex')],
             include_dirs=[
                 os.path.join(include_dir, '.'),
                 os.path.join(include_dir, 'include'),
                 ],
-            #pyrex_compile_time_env={'COMPILE_LIO': check_lio(),
-            #                        'CORO_DEBUG': True,
-            #                       },
-            # to enable LZO|LZ4 for stack compression, set COMPILE_LZO|COMPILE_LZ4 in coro/_coro.pyx
+            pyrex_compile_time_env = compile_time_env,
+            # to enable LZO|LZ4 for stack compression, set COMPILE_LZO|COMPILE_LZ4 above
             #   and uncomment one of the following:
             #libraries=['lzo2', 'z']
             #libraries=['lz4', 'z'],
             libraries=['z']
             ),
-        Extension(
-            'coro.oserrors',
-            ['coro/oserrors.pyx', ],
-            ),
+        Extension ('coro.oserrors', ['coro/oserrors.pyx', ],),
         Extension ('coro.dns.packet', ['coro/dns/packet.pyx', ],),
         Extension ('coro.asn1.ber', ['coro/asn1/ber.pyx'],),
         Extension ('coro.ldap.query', ['coro/ldap/query.pyx'],),
@@ -118,7 +119,7 @@ setup (
         ],
     packages=['coro', 'coro.clocks', 'coro.http', 'coro.dns', 'coro.emulation'],
     package_dir = {
-#        '': 'coroutine',
+    #    '': 'coroutine',
         'coro': 'coro',
         'coro.clocks': 'coro/clocks',
         'coro.dns': 'coro/dns',