Commit 5f16012d authored by Kirill Smelkov's avatar Kirill Smelkov

xnet/lonet: Draft _Python_ counterpart

This patch brings lonet implementation in Python with the idea that Go
and Python programs could interoperate via lonet network and thus mixed
Go/Python application cluster could be tested.

Implementation quality is lower compared to Go version, but still it
should be more or less ok.
parent fa0f9048
This diff is collapsed.
# Copyright (C) 2018 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
# option) any later version, as published by the Free Software Foundation.
#
# You can also Link and Combine this program with other software covered by
# the terms of any of the Free Software licenses or any of the Open Source
# Initiative approved licenses and Convey the resulting work. Corresponding
# source of such a combination shall include the source code for all other
# software used.
#
# This program is distributed WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See COPYING file for full licensing terms.
# See https://www.nexedi.com/licensing for rationale and options.
# pytest setup so that Go side could pass fixture parameters to Python side.
import pytest
# --network: for go side to pass network name to join to py side.
# --registry-dbpath: for go side to pass registry db location to py side.
def pytest_addoption(parser):
parser.addoption("--network", action="store")
parser.addoption("--registry-dbpath", action="store")
@pytest.fixture
def network(request):
network = request.config.getoption("--network")
if network is None:
raise RuntimeError("--network not set")
return network
@pytest.fixture
def registry_dbpath(request):
dbpath = request.config.getoption("--registry-dbpath")
if dbpath is None:
raise RuntimeError("--registry-dbpath not set")
return dbpath
...@@ -57,6 +57,13 @@ ...@@ -57,6 +57,13 @@
// Once again lonet is similar to pipenet, but since it works via OS TCP stack // Once again lonet is similar to pipenet, but since it works via OS TCP stack
// it could be handy for testing networked application when there are several // it could be handy for testing networked application when there are several
// OS-level processes involved. // OS-level processes involved.
//
// Package lonet also provides corresponding Python package for accessing lonet
// networks from Python(*).
//
// --------
//
// (*) use https://pypi.org/project/pygopath to import.
package lonet package lonet
// Lonet organization // Lonet organization
......
...@@ -24,12 +24,20 @@ import ( ...@@ -24,12 +24,20 @@ import (
"io/ioutil" "io/ioutil"
"log" "log"
"os" "os"
"os/exec"
"strings"
"testing" "testing"
"golang.org/x/sync/errgroup"
"lab.nexedi.com/kirr/go123/exc"
"lab.nexedi.com/kirr/go123/internal/xtesting"
"lab.nexedi.com/kirr/go123/xerr"
"lab.nexedi.com/kirr/go123/xnet/internal/virtnettest" "lab.nexedi.com/kirr/go123/xnet/internal/virtnettest"
"lab.nexedi.com/kirr/go123/xnet/virtnet"
) )
func TestLonet(t *testing.T) { func TestLonetGoGo(t *testing.T) {
subnet, err := Join(context.Background(), "") subnet, err := Join(context.Background(), "")
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
...@@ -39,13 +47,93 @@ func TestLonet(t *testing.T) { ...@@ -39,13 +47,93 @@ func TestLonet(t *testing.T) {
virtnettest.TestBasic(t, subnet) virtnettest.TestBasic(t, subnet)
} }
func TestLonetPyPy(t *testing.T) {
needPy(t)
err := pytest("-k", "test_lonet_py_basic", "lonet_test.py")
if err != nil {
t.Fatal(err)
}
}
func TestLonetGoPy(t *testing.T) {
needPy(t)
assert := xtesting.Assert(t)
subnet, err := Join(bg, ""); X(err)
defer func() {
err := subnet.Close(); X(err)
}()
xaddr := func(addr string) *virtnet.Addr {
a, err := virtnet.ParseAddr(subnet.Network(), addr); X(err)
return a
}
, err := subnet.NewHost(bg, "α"); X(err)
, err := .Listen(":1"); X(err)
wg := &errgroup.Group{}
wg.Go(exc.Funcx(func() {
c1, err := .Accept(); X(err)
assert.Eq(c1.LocalAddr(), xaddr("α:2"))
assert.Eq(c1.RemoteAddr(), xaddr("β:2"))
_, err = c1.Write([]byte("hello py")); X(err)
buf := make([]byte, 1024)
n, err := c1.Read(buf)
buf = buf[:n]
if want := "hello go"; string(buf) != want {
exc.Raisef("go<-py: got %q; want %q", buf, want)
}
err = c1.Close(); X(err)
c2, err := .Dial(bg, "β:1"); X(err)
assert.Eq(c2.LocalAddr(), xaddr("α:2"))
assert.Eq(c2.RemoteAddr(), xaddr("β:2"))
buf = make([]byte, 1024)
n, err = c2.Read(buf)
buf = buf[:n]
if want := "hello2 go"; string(buf) != want {
exc.Raisef("go<-py 2: got %q; want %q", buf, want)
}
_, err = c2.Write([]byte("hello2 py")); X(err)
err = c2.Close(); X(err)
}))
lonetwork := strings.TrimPrefix(subnet.Network(), "lonet")
err = pytest("-k", "test_lonet_py_go", "--network", lonetwork, "lonet_test.py")
X(err)
err = wg.Wait(); X(err)
}
var havePy = false
var workRoot string var workRoot string
// needPy skips test if python is not available
func needPy(t *testing.T) {
if havePy {
return
}
t.Skipf("skipping: python/pygopath/pytest are not available")
}
func TestMain(m *testing.M) { func TestMain(m *testing.M) {
// check whether we have python + infrastructure for tests
cmd := exec.Command("python", "-c", "import gopath, pytest")
err := cmd.Run()
if err == nil {
havePy = true
}
// setup workroot for all tests // setup workroot for all tests
workRoot, err := ioutil.TempDir("", "t-lonet") workRoot, err = ioutil.TempDir("", "t-lonet")
if err != nil { if err != nil {
log.Fatal(err) log.Fatal(err)
} }
...@@ -63,3 +151,23 @@ func xworkdir(t testing.TB) string { ...@@ -63,3 +151,23 @@ func xworkdir(t testing.TB) string {
} }
return work return work
} }
// pytest runs py.test with argv arguments.
func pytest(argv ...string) (err error) {
defer xerr.Contextf(&err, "pytest %s", argv)
cmd := exec.Command("python", "-m", "pytest",
// ex. with `--registry-dbpath /tmp/1.db` and existing /tmp/1.db,
// pytest tries to set cachedir=/ , fails and prints warning.
// Just disable the cache.
"-p", "no:cacheprovider")
if testing.Verbose() {
cmd.Args = append(cmd.Args, "-v", "-s", "--log-file=/dev/stderr")
} else {
cmd.Args = append(cmd.Args, "-q", "-q")
}
cmd.Args = append(cmd.Args, argv...)
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
return cmd.Run()
}
# -*- coding: utf-8 -*-
# Copyright (C) 2018 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
# option) any later version, as published by the Free Software Foundation.
#
# You can also Link and Combine this program with other software covered by
# the terms of any of the Free Software licenses or any of the Open Source
# Initiative approved licenses and Convey the resulting work. Corresponding
# source of such a combination shall include the source code for all other
# software used.
#
# This program is distributed WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See COPYING file for full licensing terms.
# See https://www.nexedi.com/licensing for rationale and options.
import gopath
xerr = gopath.gimport('lab.nexedi.com/kirr/go123/xerr')
lonet = gopath.gimport('lab.nexedi.com/kirr/go123/xnet/lonet')
from threading import Thread
from cStringIO import StringIO
import errno, logging as log
def xread(sk):
# XXX might get only part of sent data
return sk.recv(4096)
def xwrite(sk, data):
sk.sendall(data)
# TODO test that fd of listener can be used in select/epoll
# TODO non-blocking mode
# _test_virtnet_basic runs basic tests on a virtnet network implementation.
# (this follows virtnettest.TestBasic)
def _test_virtnet_basic(subnet):
# (verifying that error log stays empty)
errorlog = StringIO()
errorlogh = log.StreamHandler(errorlog)
l = log.getLogger()
l.addHandler(errorlogh)
try:
__test_virtnet_basic(subnet)
finally:
subnet.close()
l.removeHandler(errorlogh)
assert errorlog.getvalue() == ""
def __test_virtnet_basic(subnet):
def xaddr(addr):
return lonet.Addr.parse(subnet.network(), addr)
ha = subnet.new_host("α")
hb = subnet.new_host("β")
assert ha.network() == subnet.network()
assert hb.network() == subnet.network()
assert ha.name() == "α"
assert hb.name() == "β"
try:
ha.dial(":0")
except Exception as e:
assert xerr.cause(e) is lonet.ErrConnRefused
assert str(e) == "dial %s α:1->α:0: [Errno %d] connection refused" % (subnet.network(), errno.ECONNREFUSED)
else:
assert 0, "connection not refused"
l1 = ha.listen("")
assert l1.addr() == xaddr("α:1")
try:
ha.dial(":0")
except Exception as e:
assert xerr.cause(e) is lonet.ErrConnRefused
assert str(e) == "dial %s α:2->α:0: [Errno %d] connection refused" % (subnet.network(), errno.ECONNREFUSED)
else:
assert 0, "connection not refused"
def Tsrv():
c1s = l1.accept()
assert c1s.local_addr() == xaddr("α:2")
assert c1s.getsockname() == ("α", 2)
assert c1s.remote_addr() == xaddr("β:1")
assert c1s.getpeername() == ("β", 1)
assert xread(c1s) == "ping"
xwrite(c1s, "pong")
c2s = l1.accept()
assert c2s.local_addr() == xaddr("α:3")
assert c2s.getsockname() == ("α", 3)
assert c2s.remote_addr() == xaddr("β:2")
assert c2s.getpeername() == ("β", 2)
assert xread(c2s) == "hello"
xwrite(c2s, "world")
tsrv = Thread(target=Tsrv)
tsrv.start()
c1c = hb.dial("α:1")
assert c1c.local_addr() == xaddr("β:1")
assert c1c.getsockname() == ("β", 1)
assert c1c.remote_addr() == xaddr("α:2")
assert c1c.getpeername() == ("α", 2)
xwrite(c1c, "ping")
assert xread(c1c) == "pong"
c2c = hb.dial("α:1")
assert c2c.local_addr() == xaddr("β:2")
assert c2c.getsockname() == ("β", 2)
assert c2c.remote_addr() == xaddr("α:3")
assert c2c.getpeername() == ("α", 3)
xwrite(c2c, "hello")
assert xread(c2c) == "world"
tsrv.join()
l2 = ha.listen(":0")
assert l2.addr() == xaddr("α:4")
subnet.close()
def test_lonet_py_basic():
subnet = lonet.join("")
_test_virtnet_basic(subnet)
# test interaction with lonet.go
def test_lonet_py_go(network):
subnet = lonet.join(network)
try:
_test_lonet_py_go(subnet)
finally:
subnet.close()
def _test_lonet_py_go(subnet):
def xaddr(addr):
return lonet.Addr.parse(subnet.network(), addr)
hb = subnet.new_host("β")
lb = hb.listen(":1")
c1 = hb.dial("α:1")
assert c1.local_addr() == xaddr("β:2")
assert c1.remote_addr() == xaddr("α:2")
assert xread(c1) == "hello py"
xwrite(c1, "hello go")
c1.close()
c2 = lb.accept()
assert c2.local_addr() == xaddr("β:2")
assert c2.remote_addr() == xaddr("α:2")
xwrite(c2, "hello2 go")
assert xread(c2) == "hello2 py"
c2.close()
# go created a registry. verify we can read values from it and write something back too.
# go side will check what we wrote.
def test_registry_pygo(registry_dbpath):
try:
lonet.SQLiteRegistry(registry_dbpath, "ddd")
except lonet.RegistryError as e:
assert 'network name mismatch: want "ddd"; have "ccc"' in str(e)
else:
assert 0, 'network name mismatch not detected'
r = lonet.SQLiteRegistry(registry_dbpath, "ccc")
assert r.query("α") == "alpha:1234"
assert r.query("β") is None
r.announce("β", "beta:py")
assert r.query("β") == "beta:py"
try:
r.announce("β", "beta:py2")
except lonet.RegistryError as e:
# XXX py escapes utf-8 with \
#assert "announce ('β', 'beta:py2'): host already registered" in str(e)
assert ": host already registered" in str(e)
else:
assert 0, 'duplicate host announce not detected'
# ok - hand over checks back to go side.
...@@ -153,3 +153,31 @@ func TestRegistrySQLite(t *testing.T) { ...@@ -153,3 +153,31 @@ func TestRegistrySQLite(t *testing.T) {
t.Fatalf("network mismatch: error:\nhave: %q\nwant: %q", err.Error(), errWant) t.Fatalf("network mismatch: error:\nhave: %q\nwant: %q", err.Error(), errWant)
} }
} }
// verify that go and python implementations of sqlite registry understand each other.
func TestRegistrySQLitePyGo(t *testing.T) {
needPy(t)
work := xworkdir(t)
dbpath := work + "/1.db"
r1, err := openRegistrySQLite(bg, dbpath, "ccc")
X(err)
t1 := &registryTester{t, r1}
t1.Query("α", ø)
t1.Announce("α", "alpha:1234")
t1.Announce("α", "alpha:1234", DUP)
t1.Announce("α", "alpha:1235", DUP)
t1.Query("α", "alpha:1234")
t1.Query("β", ø)
// in python: check/modify the registry
err = pytest("-k", "test_registry_pygo", "--registry-dbpath", dbpath, "lonet_test.py")
X(err)
// back in go: python must have set β + α should stay the same
t1.Query("β", "beta:py")
t1.Query("α", "alpha:1234")
}
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment