Turn pygopath into full pygolang

Not only we can import modules by full path, but now we can also spawn
threads/coroutines and exchange data in between them with the same
primitives and semantic as in Go.

The bulk of new functionality is copied from here:

	go123@9e1aa6ab

Original commit description follows:

"""
golang: New _Python_ package to provide Go-like features to Python language
- `go` spawns lightweight thread.
- `chan` and `select` provide channels with Go semantic.
- `method` allows to define methods separate from class.
- `gimport` allows to import python modules by full path in a Go workspace.

The focus of first draft was on usage interface and on correctness, not speed.
In particular select should be fully working.

If there is a chance I will maybe try to followup with gevent-based
implementation in the future.
Hide whitespace changes
"""

COPYING

-Pygopath is free software licensed under GPLv3+ with additional permission
+Pygolang is free software licensed under GPLv3+ with additional permission
 to link, combine and redistribute it with other free or open source software.
 
 Please see https://www.nexedi.com/licensing for rationale and options.

MANIFEST.in

-recursive-include testdata *.py
+recursive-include golang/testdata *.py

README.rst

-=================================================================
- Pygopath - Import python modules by full path in a Go workspace
-=================================================================
+========================================
+ Pygolang - Go-like features for Python
+========================================
 
-Module `gopath` provides way to import python modules by full path in a Go workspace.
+Package golang provides Go-like features for Python:
+
+- `go` spawns lightweight thread.
+- `chan` and `select` provide channels with Go semantic.
+- `method` allows to define methods separate from class.
+- `gimport` allows to import python modules by full path in a Go workspace.
+
+
+Goroutines and channels
+-----------------------
+
+`go` spawns a thread, or a coroutine if gevent was activated. It is possible to
+exchange data in between either threads or coroutines via channels. `chan`
+creates a new channel with Go semantic - either synchronous or buffered. Use
+`chan.recv`, `chan.send` and `chan.close` for communication. `select` can be
+used to multiplex on several channels. For example::
+
+    ch1 = chan()    # synchronous channel
+    ch2 = chan(3)   # channel with buffer of size 3
+
+    def _():
+        ch1.send('a')
+        ch2.send('b')
+    go(_)
+
+    ch1.recv()      # will give 'a'
+    ch2.recv_()     # will give ('b', True)
+
+    _, _rx = select(
+        ch1.recv,           # 0
+        ch2.recv_,          # 1
+        (ch2.send, obj2),   # 2
+        default,            # 3
+    )
+    if _ == 0:
+        # _rx is what was received from ch1
+        ...
+    if _ == 1:
+        # _rx is (rx, ok) of what was received from ch2
+        ...
+    if _ == 2:
+        # we know obj2 was sent to ch2
+        ...
+    if _ == 3:
+        # default case
+        ...
+
+Methods
+-------
+
+`method` decorator allows to define methods separate from class.
+
+For example::
+
+  @method(MyClass)
+  def my_method(self, ...):
+      ...
+
+will define `MyClass.my_method()`.
+
+
+Import
+------
+
+`gimport` provides way to import python modules by full path in a Go workspace.
 
 For example
 
 ::
 
-    lonet = gopath.gimport('lab.nexedi.com/kirr/go123/xnet/lonet')
+    lonet = gimport('lab.nexedi.com/kirr/go123/xnet/lonet')
 
 will import either
 

gopath.py → golang/_gopath.py

@@ -16,7 +16,7 @@
 #
 # See COPYING file for full licensing terms.
 # See https://www.nexedi.com/licensing for rationale and options.
-"""Module gopath provides way to import python modules by full path in a Go workspace.
+"""Module _gopath provides way to import python modules by full path in a Go workspace.
 
 For example
 
@@ -56,8 +56,8 @@ def _gimport(name):
     # reason: if we leave dots in place, python emits warning:
     #   RuntimeWarning: Parent module 'lab.nexedi' not found while handling absolute import
     #
-    # we put every imported module under `gopath.` namespace with '.' changed to '_'
-    modname = 'gopath.' + name.replace('.', '_')
+    # we put every imported module under `golang._gopath.` namespace with '.' changed to '_'
+    modname = 'golang._gopath.' + name.replace('.', '_')
 
     try:
         return sys.modules[modname]

gopath_test.py → golang/_gopath_test.py

@@ -18,7 +18,7 @@
 # See https://www.nexedi.com/licensing for rationale and options.
 
 import os, os.path
-from gopath import gimport
+from golang._gopath import gimport
 
 GOPATH_orig = None
 

golang/gcompat.py

+# -*- coding: utf-8 -*-
+# Copyright (C) 2018  Nexedi SA and Contributors.
+#                     Kirill Smelkov <kirr@nexedi.com>
+#
+# This program is free software: you can Use, Study, Modify and Redistribute
+# it under the terms of the GNU General Public License version 3, or (at your
+# option) any later version, as published by the Free Software Foundation.
+#
+# You can also Link and Combine this program with other software covered by
+# the terms of any of the Free Software licenses or any of the Open Source
+# Initiative approved licenses and Convey the resulting work. Corresponding
+# source of such a combination shall include the source code for all other
+# software used.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See COPYING file for full licensing terms.
+# See https://www.nexedi.com/licensing for rationale and options.
+"""Package gcompat provides Go-compatibility layer for Python"""
+
+# qq is substitute for %q, which is missing in python.
+#
+# (python's automatic escape uses smartquotes quoting with either ' or ").
+def qq(obj):
+    # go: like %s, %q automatically converts to string
+    if not isinstance(obj, basestring):
+        obj = str(obj)
+    return _quote(obj)
+
+# _quote quotes string into valid "..." string always quoted with ".
+def _quote(s):
+    # TODO also accept unicode as input.
+    # TODO output printable UTF-8 characters as-is, but escape non-printable UTF-8 and invalid UTF-8 bytes.
+    outv = []
+    # we don't want ' to be escaped
+    for _ in s.split("'"):
+        # this escape almost everything except " character
+        # NOTE string_escape does not do smartquotes and always uses ' for quoting
+        # (repr(str) is the same except it does smartquoting picking ' or " automatically)
+        q = _.encode("string_escape")
+        q = q.replace('"', r'\"')
+        outv.append(q)
+    return '"' + "'".join(outv) + '"'

golang/gcompat_test.py

+# -*- coding: utf-8 -*-
+# Copyright (C) 2018  Nexedi SA and Contributors.
+#                     Kirill Smelkov <kirr@nexedi.com>
+#
+# This program is free software: you can Use, Study, Modify and Redistribute
+# it under the terms of the GNU General Public License version 3, or (at your
+# option) any later version, as published by the Free Software Foundation.
+#
+# You can also Link and Combine this program with other software covered by
+# the terms of any of the Free Software licenses or any of the Open Source
+# Initiative approved licenses and Convey the resulting work. Corresponding
+# source of such a combination shall include the source code for all other
+# software used.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See COPYING file for full licensing terms.
+# See https://www.nexedi.com/licensing for rationale and options.
+
+from golang.gcompat import qq
+
+def test_qq():
+    testv = (
+        # in            want without leading/trailing "
+        ('',            r""),
+        ('\'',          r"'"),
+        ('"',           r"\""),
+        ('abc\ndef',    r"abc\ndef"),
+        ('a\'c\ndef',   r"a'c\ndef"),
+        ('a\"c\ndef',   r"a\"c\ndef"),
+        # ('привет',      r"привет"),       TODO
+    )
+
+    for tin, twant in testv:
+        twant = '"' + twant + '"'   # add lead/trail "
+        assert qq(tin) == twant

golang/golang_test.py

+# -*- coding: utf-8 -*-
+# Copyright (C) 2018  Nexedi SA and Contributors.
+#                     Kirill Smelkov <kirr@nexedi.com>
+#
+# This program is free software: you can Use, Study, Modify and Redistribute
+# it under the terms of the GNU General Public License version 3, or (at your
+# option) any later version, as published by the Free Software Foundation.
+#
+# You can also Link and Combine this program with other software covered by
+# the terms of any of the Free Software licenses or any of the Open Source
+# Initiative approved licenses and Convey the resulting work. Corresponding
+# source of such a combination shall include the source code for all other
+# software used.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See COPYING file for full licensing terms.
+# See https://www.nexedi.com/licensing for rationale and options.
+
+from golang import go, chan, select, default, _PanicError
+from pytest import raises
+import time, threading
+
+# tdelay delays a bit.
+#
+# XXX needed in situations when we need to start with known ordering but do not
+# have a way to wait properly for ordering event.
+def tdelay():
+    time.sleep(1E-3)    # 1ms
+
+
+def test_chan():
+    # sync: pre-close vs send/recv
+    ch = chan()
+    ch.close()
+    assert ch.recv()    == None
+    assert ch.recv_()   == (None, False)
+    assert ch.recv_()   == (None, False)
+    raises(_PanicError, "ch.send(0)")
+    raises(_PanicError, "ch.close()")
+
+    # sync: send vs recv
+    ch = chan()
+    def _():
+        ch.send(1)
+        assert ch.recv() == 2
+        ch.close()
+    go(_)
+    assert ch.recv() == 1
+    ch.send(2)
+    assert ch.recv_() == (None, False)
+    assert ch.recv_() == (None, False)
+
+    # sync: close vs send
+    ch = chan()
+    def _():
+        tdelay()
+        ch.close()
+    go(_)
+    raises(_PanicError, "ch.send(0)")
+
+    # close vs recv
+    ch = chan()
+    def _():
+        tdelay()
+        ch.close()
+    go(_)
+    assert ch.recv_() == (None, False)
+
+    # sync: close vs multiple recv
+    ch = chan()
+    done = chan()
+    mu = threading.Lock()
+    s  = set()
+    def _():
+        assert ch.recv_() == (None, False)
+        with mu:
+            x = len(s)
+            s.add(x)
+        done.send(x)
+    for i in range(3):
+        go(_)
+    ch.close()
+    for i in range(3):
+        done.recv()
+    assert s == {0,1,2}
+
+    # buffered
+    ch = chan(3)
+    done = chan()
+    for _ in range(2):
+        for i in range(3):
+            assert len(ch) == i
+            ch.send(i)
+            assert len(ch) == i+1
+        for i in range(3):
+            assert ch.recv_() == (i, True)
+
+    assert len(ch) == 0
+    for i in range(3):
+        ch.send(i)
+    assert len(ch) == 3
+    def _():
+        tdelay()
+        assert ch.recv_() == (0, True)
+        done.send('a')
+        for i in range(1,4):
+            assert ch.recv_() == (i, True)
+        assert ch.recv_() == (None, False)
+        done.send('b')
+    go(_)
+    ch.send(3)  # will block without receiver
+    assert done.recv() == 'a'
+    ch.close()
+    assert done.recv() == 'b'
+
+
+def test_select():
+    # non-blocking try send: not ok
+    ch = chan()
+    _, _rx = select(
+            (ch.send, 0),
+            default,
+    )
+    assert (_, _rx) == (1, None)
+
+    # non-blocking try recv: not ok
+    _, _rx = select(
+            ch.recv,
+            default,
+    )
+    assert (_, _rx) == (1, None)
+
+    _, _rx = select(
+            ch.recv_,
+            default,
+    )
+    assert (_, _rx) == (1, None)
+
+    # non-blocking try send: ok
+    ch = chan()
+    done = chan()
+    def _():
+        i = 0
+        while 1:
+            x = ch.recv()
+            if x == 'stop':
+                break
+            assert x == i
+            i += 1
+        done.close()
+    go(_)
+
+    for i in range(10):
+        tdelay()
+        _, _rx = select(
+                (ch.send, i),
+                default,
+        )
+        assert (_, _rx) == (0, None)
+    ch.send('stop')
+    done.recv()
+
+    # non-blocking try recv: ok
+    ch = chan()
+    done = chan()
+    def _():
+        for i in range(10):
+            ch.send(i)
+        done.close()
+    go(_)
+
+    for i in range(10):
+        tdelay()
+        if i % 2:
+            _, _rx = select(
+                    ch.recv,
+                    default,
+            )
+            assert (_, _rx) == (0, i)
+        else:
+            _, _rx = select(
+                    ch.recv_,
+                    default,
+            )
+            assert (_, _rx) == (0, (i, True))
+    done.recv()
+
+
+    # blocking 2·send
+    ch1 = chan()
+    ch2 = chan()
+    done = chan()
+    def _():
+        tdelay()
+        assert ch1.recv() == 'a'
+        done.close()
+    go(_)
+
+    _, _rx = select(
+        (ch1.send, 'a'),
+        (ch2.send, 'b'),
+    )
+    assert (_, _rx) == (0, None)
+    done.recv()
+    assert len(ch1._sendq) == len(ch1._recvq) == 0
+    assert len(ch2._sendq) == len(ch2._recvq) == 0
+
+
+    # blocking 2·recv
+    ch1 = chan()
+    ch2 = chan()
+    done = chan()
+    def _():
+        tdelay()
+        ch1.send('a')
+        done.close()
+    go(_)
+
+    _, _rx = select(
+        ch1.recv,
+        ch2.recv,
+    )
+    assert (_, _rx) == (0, 'a')
+    done.recv()
+    assert len(ch1._sendq) == len(ch1._recvq) == 0
+    assert len(ch2._sendq) == len(ch2._recvq) == 0
+
+
+    # blocking send/recv
+    ch1 = chan()
+    ch2 = chan()
+    done = chan()
+    def _():
+        tdelay()
+        assert ch1.recv() == 'a'
+        done.close()
+    go(_)
+
+    _, _rx = select(
+        (ch1.send, 'a'),
+        ch2.recv,
+    )
+    assert (_, _rx) == (0, None)
+    done.recv()
+    assert len(ch1._sendq) == len(ch1._recvq) == 0
+    assert len(ch2._sendq) == len(ch2._recvq) == 0
+
+
+    # blocking recv/send
+    ch1 = chan()
+    ch2 = chan()
+    done = chan()
+    def _():
+        tdelay()
+        ch1.send('a')
+        done.close()
+    go(_)
+
+    _, _rx = select(
+        ch1.recv,
+        (ch2.send, 'b'),
+    )
+    assert (_, _rx) == (0, 'a')
+    done.recv()
+    assert len(ch1._sendq) == len(ch1._recvq) == 0
+    assert len(ch2._sendq) == len(ch2._recvq) == 0
+
+
+    # buffered ping-pong
+    ch = chan(1)
+    for i in range(10):
+        _, _rx = select(
+            (ch.send, i),
+            ch.recv,
+        )
+        assert _    == (i % 2)
+        assert _rx  == (i - 1 if i % 2 else None)
+
+
+    # select vs select
+    for i in range(10):
+        ch1 = chan()
+        ch2 = chan()
+        done = chan()
+        def _():
+            _, _rx = select(
+                (ch1.send, 'a'),
+                (ch2.send, 'xxx2'),
+            )
+            assert (_, _rx) == (0, None)
+
+            _, _rx = select(
+                (ch1.send, 'yyy2'),
+                ch2.recv,
+            )
+            assert (_, _rx) == (1, 'b')
+
+            done.close()
+
+        go(_)
+
+        _, _rx = select(
+            ch1.recv,
+            (ch2.send, 'xxx1'),
+        )
+        assert (_, _rx) == (0, 'a')
+
+        _, _rx = select(
+            (ch1.send, 'yyy1'),
+            (ch2.send, 'b'),
+        )
+        assert (_, _rx) == (1, None)
+
+        done.recv()
+        assert len(ch1._sendq) == len(ch1._recvq) == 0
+        assert len(ch2._sendq) == len(ch2._recvq) == 0

setup.py

-# pygopath | pythonic package setup
+# pygolang | pythonic package setup
 from setuptools import setup, find_packages
 
 # read file content
@@ -7,20 +7,18 @@ def readfile(path):
         return f.read()
 
 setup(
-    name        = 'pygopath',
-    version     = '0.0.0.dev1',
-    description = 'Import python modules by full-path in Go workspace',
+    name        = 'pygolang',
+    version     = '0.0.0.dev2',
+    description = 'Go-like features for Python',
     long_description = readfile('README.rst'),
-    url         = 'https://lab.nexedi.com/kirr/pygopath',
+    url         = 'https://lab.nexedi.com/kirr/pygolang',
     license     = 'GPLv3+ with wide exception for Open-Source',
     author      = 'Kirill Smelkov',
     author_email= 'kirr@nexedi.com',
 
-    keywords    = 'go GOPATH python import',
+    keywords    = 'go channel goroutine GOPATH python import',
 
-    # XXX find_packages does not find top-level *.py
-    #packages    = find_packages(),
-    packages    = [''],
+    packages    = find_packages(),
 
     extras_require = {
                   'test': ['pytest'],