Commit a6b993c8 authored by Kirill Smelkov's avatar Kirill Smelkov

gpython: Add way to run it with threads runtime

Until now gpython was always activating gevent on startup + adding
goodness such as "always UTF-8 default encoding"; go, chan, b/u and
friends available from builtin namespace, etc... While those goodness
are sometimes useful on their own, it is not always appropriate to force
a project to switch from threads to gevent.

For this reason add a flag to select which runtime - either gevent
or threads - gpython should use.

	gpython -Xgpython.runtime=gevent	selects gevent,

while

	gpython -Xgpython.runtime=threads	selects threads.

Gevent remains the default.

It is also possible to specify desired runtime via $GPYTHON_RUNTIME
environment variable.

/reviewed-on !5
parent c0282565
...@@ -53,6 +53,8 @@ Additionally GPython sets UTF-8 to be default encoding always, and puts `go`, ...@@ -53,6 +53,8 @@ Additionally GPython sets UTF-8 to be default encoding always, and puts `go`,
GPython is optional and the rest of Pygolang can be used from under standard Python too. GPython is optional and the rest of Pygolang can be used from under standard Python too.
However without gevent integration `go` spawns full - not lightweight - OS thread. However without gevent integration `go` spawns full - not lightweight - OS thread.
GPython can be also used with threads - not gevent - runtime. Please see
`GPython options`_ for details.
Goroutines and channels Goroutines and channels
...@@ -502,3 +504,22 @@ such benchmarking data in Python. ...@@ -502,3 +504,22 @@ such benchmarking data in Python.
.. _golang.x.perf.benchlib: https://lab.nexedi.com/nexedi/pygolang/tree/master/golang/x/perf/benchlib.py .. _golang.x.perf.benchlib: https://lab.nexedi.com/nexedi/pygolang/tree/master/golang/x/perf/benchlib.py
__ https://github.com/golang/proposal/blob/master/design/14313-benchmark-format.md __ https://github.com/golang/proposal/blob/master/design/14313-benchmark-format.md
__ https://godoc.org/golang.org/x/perf/cmd/benchstat __ https://godoc.org/golang.org/x/perf/cmd/benchstat
--------
GPython options
---------------
GPython mimics and supports most of Python command-line options, like `gpython
-c <commands>` to run Python statements from command line, or `gpython -m
<module>` to execute a module. Such options have the same meaning as in
standard Python and are not documented here.
GPython-specific options and environment variables are listed below:
`-X gpython.runtime=(gevent|threads)`
Specify which runtime GPython should use. `gevent` provides lightweight
coroutines, while with `threads` `go` spawns full OS thread. `gevent` is
default. The runtime to use can be also specified via `$GPYTHON_RUNTIME`
environment variable.
...@@ -26,6 +26,9 @@ differences: ...@@ -26,6 +26,9 @@ differences:
- gevent is pre-activated and stdlib is patched to be gevent aware; - gevent is pre-activated and stdlib is patched to be gevent aware;
- go, chan, select etc are put into builtin namespace; - go, chan, select etc are put into builtin namespace;
- default string encoding is always set to UTF-8. - default string encoding is always set to UTF-8.
Gevent activation can be disabled via `-X gpython.runtime=threads`, or
$GPYTHON_RUNTIME=threads.
""" """
# NOTE gpython is kept out of golang/ , since even just importing e.g. golang.cmd.gpython, # NOTE gpython is kept out of golang/ , since even just importing e.g. golang.cmd.gpython,
...@@ -75,8 +78,12 @@ def pymain(argv): ...@@ -75,8 +78,12 @@ def pymain(argv):
ver = [] ver = []
if 'GPython' in sys.version: if 'GPython' in sys.version:
golang = sys.modules['golang'] # must be already imported golang = sys.modules['golang'] # must be already imported
gevent = sys.modules['gevent'] # must be already imported gevent = sys.modules.get('gevent', None)
gpyver = 'GPython %s [gevent %s]' % (golang.__version__, gevent.__version__) gpyver = 'GPython %s' % golang.__version__
if gevent is not None:
gpyver += ' [gevent %s]' % gevent.__version__
else:
gpyver += ' [threads]'
ver.append(gpyver) ver.append(gpyver)
import platform import platform
...@@ -213,20 +220,65 @@ def main(): ...@@ -213,20 +220,65 @@ def main():
exe = exe + '.exe' exe = exe + '.exe'
sys.executable = exe sys.executable = exe
# make gevent pre-available & stdlib patched # import os to get access to environment.
import gevent # it is practically ok to import os before gevent, because os is always
from gevent import monkey # imported by site. Yes, `import site` can be disabled by -S, but there is
# XXX workaround for gevent vs pypy2 crash. # no harm wrt gevent monkey-patching even if we import os first.
# XXX remove when gevent-1.4.1 is relased (https://github.com/gevent/gevent/pull/1357). import os
patch_thread=True
if pypy and sys.version_info.major == 2: # extract and process -X
_ = monkey.patch_thread(existing_locks=False) # -X gpython.runtime=(gevent|threads) + $GPYTHON_RUNTIME
assert _ in (True, None) sys._xoptions = getattr(sys, '_xoptions', {})
patch_thread=False argv_ = []
_ = monkey.patch_all(thread=patch_thread) # XXX sys=True ? gpy_runtime = os.getenv('GPYTHON_RUNTIME', 'gevent')
if _ not in (True, None): # patched or nothing to do while len(argv) > 0:
# XXX provide details arg = argv[0]
raise RuntimeError('gevent monkey-patching failed') argv = argv[1:]
if not arg.startswith('-X'):
argv_.append(arg)
# continue looking for -X only until options end
if not arg.startswith('-'):
break
continue
# -X <opt>
opt = arg[2:] # -X<opt>
if opt == '':
opt = argv[0] # -X <opt>
argv = argv[1:]
if opt.startswith('gpython.runtime='):
gpy_runtime = opt[len('gpython.runtime='):]
sys._xoptions['gpython.runtime'] = gpy_runtime
else:
raise RuntimeError('gpython: unknown -X option %s' % opt)
argv = argv_ + argv
# initialize according to selected runtime
if gpy_runtime == 'gevent':
# make gevent pre-available & stdlib patched
import gevent
from gevent import monkey
# XXX workaround for gevent vs pypy2 crash.
# XXX remove when gevent-1.4.1 is relased (https://github.com/gevent/gevent/pull/1357).
patch_thread=True
if pypy and sys.version_info.major == 2:
_ = monkey.patch_thread(existing_locks=False)
assert _ in (True, None)
patch_thread=False
_ = monkey.patch_all(thread=patch_thread) # XXX sys=True ?
if _ not in (True, None): # patched or nothing to do
# XXX provide details
raise RuntimeError('gevent monkey-patching failed')
gpy_verextra = 'gevent %s' % gevent.__version__
elif gpy_runtime == 'threads':
gpy_verextra = 'threads'
else:
raise RuntimeError('gpython: invalid runtime %s' % gpy_runtime)
# put go, chan, select, ... into builtin namespace # put go, chan, select, ... into builtin namespace
import golang import golang
...@@ -235,7 +287,7 @@ def main(): ...@@ -235,7 +287,7 @@ def main():
setattr(builtins, k, getattr(golang, k)) setattr(builtins, k, getattr(golang, k))
# sys.version # sys.version
sys.version += (' [GPython %s] [gevent %s]' % (golang.__version__, gevent.__version__)) sys.version += (' [GPython %s] [%s]' % (golang.__version__, gpy_verextra))
# tail to pymain # tail to pymain
pymain(argv) pymain(argv)
......
...@@ -33,6 +33,24 @@ is_cpython = (platform.python_implementation() == 'CPython') ...@@ -33,6 +33,24 @@ is_cpython = (platform.python_implementation() == 'CPython')
# @gpython_only is marker to run a test only under gpython # @gpython_only is marker to run a test only under gpython
gpython_only = pytest.mark.skipif('GPython' not in sys.version, reason="gpython-only test") gpython_only = pytest.mark.skipif('GPython' not in sys.version, reason="gpython-only test")
# runtime is pytest fixture that yields all variants of should be supported gpython runtimes:
# '' - not specified (gpython should autoselect)
# 'gevent'
# 'threads'
@pytest.fixture(scope="function", params=['', 'gevent', 'threads'])
def runtime(request):
yield request.param
# gpyenv returns environment appropriate for spawning gpython with
# specified runtime.
def gpyenv(runtime): # -> env
env = os.environ.copy()
if runtime != '':
env['GPYTHON_RUNTIME'] = runtime
else:
env.pop('GPYTHON_RUNTIME', None)
return env
@gpython_only @gpython_only
def test_defaultencoding_utf8(): def test_defaultencoding_utf8():
...@@ -54,6 +72,12 @@ def test_golang_builtins(): ...@@ -54,6 +72,12 @@ def test_golang_builtins():
@gpython_only @gpython_only
def test_gevent_activated(): def test_gevent_activated():
# gpython, by default, acticates gevent.
# handling of various runtime modes is explicitly tested in test_Xruntime.
assert_gevent_activated()
def assert_gevent_activated():
assert 'gevent' in sys.modules
from gevent.monkey import is_module_patched as patched, is_object_patched as obj_patched from gevent.monkey import is_module_patched as patched, is_object_patched as obj_patched
# builtin (gevent: only on py2 - on py3 __import__ uses fine-grained locking) # builtin (gevent: only on py2 - on py3 __import__ uses fine-grained locking)
...@@ -91,15 +115,38 @@ def test_gevent_activated(): ...@@ -91,15 +115,38 @@ def test_gevent_activated():
if sys.hexversion >= 0x03070000: # >= 3.7.0 if sys.hexversion >= 0x03070000: # >= 3.7.0
assert patched('queue') assert patched('queue')
def assert_gevent_not_activated():
assert 'gevent' not in sys.modules
from gevent.monkey import is_module_patched as patched, is_object_patched as obj_patched
assert not patched('socket')
assert not patched('time')
assert not patched('select')
assert not patched('os')
assert not patched('signal')
assert not patched('thread' if PY2 else '_thread')
assert not patched('threading')
assert not patched('_threading_local')
assert not patched('ssl')
assert not patched('subprocess')
assert not patched('sys')
@gpython_only @gpython_only
def test_executable(): def test_executable(runtime):
# sys.executable must point to gpython and we must be able to execute it. # sys.executable must point to gpython and we must be able to execute it.
import gevent import gevent
assert 'gpython' in sys.executable assert 'gpython' in sys.executable
out = pyout(['-c', 'import sys; print(sys.version)']) ver = pyout(['-c', 'import sys; print(sys.version)'], env=gpyenv(runtime))
assert ('[GPython %s]' % golang.__version__) in str(out) ver = str(ver)
assert ('[gevent %s]' % gevent.__version__) in str(out) assert ('[GPython %s]' % golang.__version__) in ver
if runtime != 'threads':
assert ('[gevent %s]' % gevent.__version__) in ver
assert ('[threads]') not in ver
else:
assert ('[gevent ') not in ver
assert ('[threads]') in ver
# verify pymain. # verify pymain.
# #
...@@ -138,11 +185,15 @@ def test_pymain(): ...@@ -138,11 +185,15 @@ def test_pymain():
# pymain -V/--version # pymain -V/--version
# gpython_only because output differs from !gpython. # gpython_only because output differs from !gpython.
@gpython_only @gpython_only
def test_pymain_ver(): def test_pymain_ver(runtime):
from golang import b from golang import b
from gpython import _version_info_str as V from gpython import _version_info_str as V
import gevent import gevent
vok = 'GPython %s [gevent %s]' % (golang.__version__, gevent.__version__) vok = 'GPython %s' % golang.__version__
if runtime != 'threads':
vok += ' [gevent %s]' % gevent.__version__
else:
vok += ' [threads]'
if is_cpython: if is_cpython:
vok += ' / CPython %s' % platform.python_version() vok += ' / CPython %s' % platform.python_version()
...@@ -153,8 +204,29 @@ def test_pymain_ver(): ...@@ -153,8 +204,29 @@ def test_pymain_ver():
vok += '\n' vok += '\n'
ret, out, err = _pyrun(['-V'], stdout=PIPE, stderr=PIPE) ret, out, err = _pyrun(['-V'], stdout=PIPE, stderr=PIPE, env=gpyenv(runtime))
assert (ret, out, b(err)) == (0, b'', b(vok)) assert (ret, out, b(err)) == (0, b'', b(vok))
ret, out, err = _pyrun(['--version'], stdout=PIPE, stderr=PIPE) ret, out, err = _pyrun(['--version'], stdout=PIPE, stderr=PIPE, env=gpyenv(runtime))
assert (ret, out, b(err)) == (0, b'', b(vok)) assert (ret, out, b(err)) == (0, b'', b(vok))
# verify -X gpython.runtime=...
@gpython_only
def test_Xruntime(runtime):
env = os.environ.copy()
env.pop('GPYTHON_RUNTIME', None) # del
argv = []
if runtime != '':
argv += ['-X', 'gpython.runtime='+runtime]
prog = 'from gpython import gpython_test as t; '
if runtime != 'threads':
prog += 't.assert_gevent_activated(); '
else:
prog += 't.assert_gevent_not_activated(); '
prog += 'print("ok")'
argv += ['-c', prog]
out = pyout(argv, env=env)
assert out == b'ok\n'
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment