Commit 4d64fd0f authored by Kirill Smelkov's avatar Kirill Smelkov

X update (sync with master + ustr.translate fixes)

parent ac87a2ed
[submodule "3rdparty/ratas"]
path = 3rdparty/ratas
url = https://github.com/jsnell/ratas.git
[submodule "3rdparty/funchook"] [submodule "3rdparty/funchook"]
path = 3rdparty/funchook path = 3rdparty/funchook
url = https://github.com/kubo/funchook.git url = https://github.com/kubo/funchook.git
......
# .lsan-ignore.txt lists memory leak events that LeakSanitizer should not
# report when running pygolang tests.
#
# Many python allocations, whose lifetime coincides with python interpreter
# lifetime, and which are not explicitly freed on python shutdown, are
# reported as leaks by default. Disable leak reporting for those to avoid
# non-pygolang related printouts.
# >>> Everything created when initializing python, e.g. sys.stderr
# #0 0x7f21e74f3bd7 in malloc .../asan_malloc_linux.cpp:69
# #1 0x555f361ff9a4 in PyThread_allocate_lock Python/thread_pthread.h:385
# #2 0x555f3623f72a in _buffered_init Modules/_io/bufferedio.c:725
# #3 0x555f3623ff7e in _io_BufferedWriter___init___impl Modules/_io/bufferedio.c:1803
# #4 0x555f3623ff7e in _io_BufferedWriter___init__ Modules/_io/clinic/bufferedio.c.h:489
# #5 0x555f3610c086 in type_call Objects/typeobject.c:1103
# #6 0x555f3609cdcc in _PyObject_MakeTpCall Objects/call.c:214
# #7 0x555f3609d6a8 in _PyObject_VectorcallTstate Include/internal/pycore_call.h:90
# #8 0x555f3609d6a8 in _PyObject_VectorcallTstate Include/internal/pycore_call.h:77
# #9 0x555f3609d6a8 in _PyObject_CallFunctionVa Objects/call.c:536
# #10 0x555f3609e89c in _PyObject_CallFunction_SizeT Objects/call.c:590
# #11 0x555f3623a0df in _io_open_impl Modules/_io/_iomodule.c:407
# #12 0x555f3623a0df in _io_open Modules/_io/clinic/_iomodule.c.h:264
# #13 0x555f360f17da in cfunction_vectorcall_FASTCALL_KEYWORDS Objects/methodobject.c:443
# #14 0x555f3609d54c in _PyObject_VectorcallTstate Include/internal/pycore_call.h:92
# #15 0x555f3609d54c in _PyObject_CallFunctionVa Objects/call.c:536
# #16 0x555f3609ec34 in callmethod Objects/call.c:608
# #17 0x555f3609ec34 in _PyObject_CallMethod Objects/call.c:677
# #18 0x555f361e60cf in create_stdio Python/pylifecycle.c:2244
# #19 0x555f361e6523 in init_sys_streams Python/pylifecycle.c:2431
# #20 0x555f361e6523 in init_interp_main Python/pylifecycle.c:1154
# #21 0x555f361e7204 in pyinit_main Python/pylifecycle.c:1230
# #22 0x555f361e85ba in Py_InitializeFromConfig Python/pylifecycle.c:1261
# #23 0x555f3621010a in pymain_init Modules/main.c:67
# #24 0x555f362113de in pymain_main Modules/main.c:701
# #25 0x555f362113de in Py_BytesMain Modules/main.c:734
leak:^pymain_init$
# >>> Everything created when importing py modules, e.g.
# #0 0x7f18c86f3bd7 in malloc .../asan_malloc_linux.cpp:69
# #1 0x55b971430acf in PyMem_RawMalloc Objects/obmalloc.c:586
# #2 0x55b971430acf in _PyObject_Malloc Objects/obmalloc.c:2003
# #3 0x55b971430acf in _PyObject_Malloc Objects/obmalloc.c:1996
# #4 0x55b971415696 in new_keys_object Objects/dictobject.c:632
# #5 0x55b971415716 in dictresize Objects/dictobject.c:1429
# #6 0x55b97141961a in insertion_resize Objects/dictobject.c:1183
# #7 0x55b97141961a in insertdict Objects/dictobject.c:1248
# #8 0x55b97143eb7b in add_subclass Objects/typeobject.c:6547
# #9 0x55b97144ca52 in type_ready_add_subclasses Objects/typeobject.c:6345
# #10 0x55b97144ca52 in type_ready Objects/typeobject.c:6476
# #11 0x55b971451a1f in PyType_Ready Objects/typeobject.c:6508
# #12 0x55b971451a1f in type_new_impl Objects/typeobject.c:3189
# #13 0x55b971451a1f in type_new Objects/typeobject.c:3323
# #14 0x55b971443014 in type_call Objects/typeobject.c:1091
# #15 0x55b9713d3dcc in _PyObject_MakeTpCall Objects/call.c:214
# #16 0x55b9713d47bd in _PyObject_FastCallDictTstate Objects/call.c:141
# #17 0x55b9713d47bd in PyObject_VectorcallDict Objects/call.c:165
# #18 0x55b9714d14c2 in builtin___build_class__ Python/bltinmodule.c:209
# #19 0x55b9714287da in cfunction_vectorcall_FASTCALL_KEYWORDS Objects/methodobject.c:443
# #20 0x55b9713d4a7b in _PyObject_VectorcallTstate Include/internal/pycore_call.h:92
# #21 0x55b9713d4a7b in PyObject_Vectorcall Objects/call.c:299
# #22 0x55b97137666e in _PyEval_EvalFrameDefault Python/ceval.c:4769
# #23 0x55b9714d7e6b in _PyEval_EvalFrame Include/internal/pycore_ceval.h:73
# #24 0x55b9714d7e6b in _PyEval_Vector Python/ceval.c:6434
# #25 0x55b9714d7e6b in PyEval_EvalCode Python/ceval.c:1148
# #26 0x55b9714d2e1f in builtin_exec_impl Python/bltinmodule.c:1077
# #27 0x55b9714d2e1f in builtin_exec Python/clinic/bltinmodule.c.h:465
# #28 0x55b9714287da in cfunction_vectorcall_FASTCALL_KEYWORDS Objects/methodobject.c:443
# #29 0x55b971376dcb in do_call_core Python/ceval.c:7349
# #30 0x55b971376dcb in _PyEval_EvalFrameDefault Python/ceval.c:5376
# #31 0x55b9714d7faf in _PyEval_EvalFrame Include/internal/pycore_ceval.h:73
# #32 0x55b9714d7faf in _PyEval_Vector Python/ceval.c:6434
# #33 0x55b9713d436e in _PyObject_VectorcallTstate Include/internal/pycore_call.h:92
# #34 0x55b9713d436e in object_vacall Objects/call.c:819
# #35 0x55b9713d63cf in PyObject_CallMethodObjArgs Objects/call.c:879
# #36 0x55b9715080e1 in import_find_and_load Python/import.c:1748
# #37 0x55b9715080e1 in PyImport_ImportModuleLevelObject Python/import.c:1847
# #38 0x55b97137de9c in import_name Python/ceval.c:7422
# #39 0x55b97137de9c in _PyEval_EvalFrameDefault Python/ceval.c:3946
# #40 0x55b9714d7e6b in _PyEval_EvalFrame Include/internal/pycore_ceval.h:73
# #41 0x55b9714d7e6b in _PyEval_Vector Python/ceval.c:6434
# #42 0x55b9714d7e6b in PyEval_EvalCode Python/ceval.c:1148
# #43 0x55b9714d2e1f in builtin_exec_impl Python/bltinmodule.c:1077
# #44 0x55b9714d2e1f in builtin_exec Python/clinic/bltinmodule.c.h:465
# #45 0x55b9714287da in cfunction_vectorcall_FASTCALL_KEYWORDS Objects/methodobject.c:443
# #46 0x55b971376dcb in do_call_core Python/ceval.c:7349
# #47 0x55b971376dcb in _PyEval_EvalFrameDefault Python/ceval.c:5376
leak:^PyImport_Import
# importlib.import_module leads to
# #0 0x7f1951ef3bd7 in malloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:69
# #1 0x55f399e8cacf in PyMem_RawMalloc Objects/obmalloc.c:586
# #2 0x55f399e8cacf in _PyObject_Malloc Objects/obmalloc.c:2003
# #3 0x55f399e8cacf in _PyObject_Malloc Objects/obmalloc.c:1996
# #4 0x55f399e86344 in PyModule_ExecDef Objects/moduleobject.c:400
# #5 0x55f399f6178a in exec_builtin_or_dynamic Python/import.c:2345
# #6 0x55f399f6178a in _imp_exec_dynamic_impl Python/import.c:2419
# #7 0x55f399f6178a in _imp_exec_dynamic Python/clinic/import.c.h:474
# #8 0x55f399e8438a in cfunction_vectorcall_O Objects/methodobject.c:514
leak:^_imp_exec_dynamic
# >>> Everything allocated at DSO initialization, e.g.
# #0 0x7f35d2af46c8 in operator new(unsigned long) .../asan_new_delete.cpp:95
# #1 0x7f35ce897e9f in __static_initialization_and_destruction_0 golang/context.cpp:61
# #2 0x7f35ce8982ef in _GLOBAL__sub_I_context.cpp golang/context.cpp:380
# #3 0x7f35d32838bd in call_init elf/dl-init.c:90
# #4 0x7f35d32838bd in call_init elf/dl-init.c:27
# #5 0x7f35d32839a3 in _dl_init elf/dl-init.c:137
# #6 0x7f35d256e023 in __GI__dl_catch_exception elf/dl-error-skeleton.c:182
# #7 0x7f35d328a09d in dl_open_worker elf/dl-open.c:808
# #8 0x7f35d256dfc9 in __GI__dl_catch_exception elf/dl-error-skeleton.c:208
# #9 0x7f35d328a437 in _dl_open elf/dl-open.c:884
# #10 0x7f35d24a4437 in dlopen_doit dlfcn/dlopen.c:56
# #11 0x7f35d256dfc9 in __GI__dl_catch_exception elf/dl-error-skeleton.c:208
# #12 0x7f35d256e07e in __GI__dl_catch_error elf/dl-error-skeleton.c:227
# #13 0x7f35d24a3f26 in _dlerror_run dlfcn/dlerror.c:138
# #14 0x7f35d24a44e8 in dlopen_implementation dlfcn/dlopen.c:71
# #15 0x7f35d24a44e8 in ___dlopen dlfcn/dlopen.c:81
# #16 0x7f35d2a77ff9 in dlopen .../sanitizer_common_interceptors.inc:6341
leak:^_GLOBAL_
# global<> does not deallocate its reference on purpose
leak:^_test_global()$
Subproject commit becd5fc5c1e9ea600cd8b3b1c24d564794fedac4
include COPYING README.rst CHANGELOG.rst tox.ini pyproject.toml trun .nxdtest include COPYING README.rst CHANGELOG.rst tox.ini pyproject.toml trun .lsan-ignore.txt .nxdtest conftest.py
include golang/libgolang.h include golang/libgolang.h
include golang/runtime/libgolang.cpp include golang/runtime/libgolang.cpp
include golang/runtime/libpyxruntime.cpp include golang/runtime/libpyxruntime.cpp
......
# pygolang | pytest config
# Copyright (C) 2021-2024 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
# option) any later version, as published by the Free Software Foundation.
#
# You can also Link and Combine this program with other software covered by
# the terms of any of the Free Software licenses or any of the Open Source
# Initiative approved licenses and Convey the resulting work. Corresponding
# source of such a combination shall include the source code for all other
# software used.
#
# This program is distributed WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See COPYING file for full licensing terms.
# See https://www.nexedi.com/licensing for rationale and options.
from __future__ import print_function, absolute_import
import gc
# Do full GC before pytest exits, to avoid false positives in the leak detector.
def pytest_unconfigure():
gc.collect()
# ignore tests in distorm - else it breaks as e.g. # ignore tests in distorm - else it breaks as e.g.
# #
# 3rdparty/funchook/distorm/python/test_distorm3.py:15: in <module> # 3rdparty/funchook/distorm/python/test_distorm3.py:15: in <module>
......
...@@ -173,11 +173,18 @@ cdef void __goviac(void *arg) nogil: ...@@ -173,11 +173,18 @@ cdef void __goviac(void *arg) nogil:
# ---- channels ---- # ---- channels ----
# _frompyx indicates that a constructor is called from pyx code
cdef object _frompyx = object()
@final @final
cdef class pychan: cdef class pychan:
def __cinit__(pychan pych, size=0, dtype=object): def __cinit__(pychan pych, size=0, dtype=object):
pych.dtype = parse_dtype(dtype) if dtype is _frompyx:
pych._ch = _makechan_pyexc(dtypeRegistry[<int>pych.dtype].size, size) pych.dtype = DTYPE_STRUCTZ # anything
pych._ch = NULL
else:
pych.dtype = parse_dtype(dtype)
pych._ch = _makechan_pyexc(dtypeRegistry[<int>pych.dtype].size, size)
# pychan.nil(X) creates new nil pychan with specified dtype. # pychan.nil(X) creates new nil pychan with specified dtype.
# TODO try to avoid exposing .nil on pychan instances, and expose only pychan.nil # TODO try to avoid exposing .nil on pychan instances, and expose only pychan.nil
...@@ -370,7 +377,7 @@ cdef void pychan_asserttype(pychan pych, DType dtype) nogil: ...@@ -370,7 +377,7 @@ cdef void pychan_asserttype(pychan pych, DType dtype) nogil:
panic("pychan: channel type mismatch") panic("pychan: channel type mismatch")
cdef pychan pychan_from_raw(_chan *_ch, DType dtype): cdef pychan pychan_from_raw(_chan *_ch, DType dtype):
cdef pychan pych = pychan.__new__(pychan) cdef pychan pych = pychan.__new__(pychan, dtype=_frompyx)
pych.dtype = dtype pych.dtype = dtype
pych._ch = _ch; _chanxincref(_ch) pych._ch = _ch; _chanxincref(_ch)
return pych return pych
...@@ -626,9 +633,7 @@ cdef object c_to_py(DType dtype, const chanElemBuf *cfrom): ...@@ -626,9 +633,7 @@ cdef object c_to_py(DType dtype, const chanElemBuf *cfrom):
# mkpynil creates pychan instance that represents nil[dtype]. # mkpynil creates pychan instance that represents nil[dtype].
cdef PyObject *mkpynil(DType dtype): cdef PyObject *mkpynil(DType dtype):
cdef pychan pynil = pychan.__new__(pychan) cdef pychan pynil = pychan_from_raw(NULL, dtype)
pynil.dtype = dtype
pynil._ch = NULL # should be already NULL
Py_INCREF(pynil) Py_INCREF(pynil)
return <PyObject *>pynil return <PyObject *>pynil
...@@ -818,9 +823,6 @@ from libcpp.typeinfo cimport type_info ...@@ -818,9 +823,6 @@ from libcpp.typeinfo cimport type_info
from cython.operator cimport typeid from cython.operator cimport typeid
from libc.string cimport strcmp from libc.string cimport strcmp
# _frompyx indicates that a constructor is called from pyx code
cdef object _frompyx = object()
cdef class pyerror(Exception): cdef class pyerror(Exception):
# pyerror <- error # pyerror <- error
@staticmethod @staticmethod
......
...@@ -580,6 +580,7 @@ cdef class _pybstr(bytes): # https://github.com/cython/cython/issues/711 ...@@ -580,6 +580,7 @@ cdef class _pybstr(bytes): # https://github.com/cython/cython/issues/711
def title(self): return pyb(pyu(self).title()) def title(self): return pyb(pyu(self).title())
def translate(self, table, delete=None): def translate(self, table, delete=None):
# bytes mode (compatibility with str/py2) # bytes mode (compatibility with str/py2)
# XXX isinstance(zbytes) -> isinstance(bytes) ?
if table is None or isinstance(table, zbytes) or delete is not None: if table is None or isinstance(table, zbytes) or delete is not None:
if delete is None: delete = b'' if delete is None: delete = b''
return pyb(zbytes.translate(self, table, delete)) return pyb(zbytes.translate(self, table, delete))
...@@ -905,12 +906,7 @@ cdef class _pyustr(unicode): ...@@ -905,12 +906,7 @@ cdef class _pyustr(unicode):
def translate(self, table): def translate(self, table):
# unicode.translate does not accept bstr values # unicode.translate does not accept bstr values
t = {} return pyu(zunicode.translate(self, _pyustrTranslateTab(table)))
for k,v in table.items():
if not isinstance(v, int): # either unicode ordinal,
v = _xpyu_coerce(v) # character or None
t[k] = v
return pyu(zunicode.translate(self, t))
def upper(self): return pyu(zunicode.upper(self)) def upper(self): return pyu(zunicode.upper(self))
def zfill(self, width): return pyu(zunicode.zfill(self, width)) def zfill(self, width): return pyu(zunicode.zfill(self, width))
...@@ -983,6 +979,18 @@ cdef class _pyustrIter: ...@@ -983,6 +979,18 @@ cdef class _pyustrIter:
x = next(self.uiter) x = next(self.uiter)
return pyu(x) return pyu(x)
# _pyustrTranslateTab wraps table for .translate to return bstr as unicode
# because unicode.translate does not accept bstr values.
cdef class _pyustrTranslateTab:
cdef object tab
def __init__(self, tab):
self.tab = tab
def __getitem__(self, k):
v = self.tab[k]
if not isinstance(v, int): # either unicode ordinal,
v = _xpyu_coerce(v) # character or None
return v
# _bdata/_udata retrieve raw data from bytes/unicode. # _bdata/_udata retrieve raw data from bytes/unicode.
def _bdata(obj): # -> bytes def _bdata(obj): # -> bytes
......
...@@ -2,7 +2,7 @@ ...@@ -2,7 +2,7 @@
# cython: language_level=2 # cython: language_level=2
# distutils: language=c++ # distutils: language=c++
# #
# Copyright (C) 2018-2020 Nexedi SA and Contributors. # Copyright (C) 2018-2024 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com> # Kirill Smelkov <kirr@nexedi.com>
# #
# This program is free software: you can Use, Study, Modify and Redistribute # This program is free software: you can Use, Study, Modify and Redistribute
...@@ -344,6 +344,25 @@ cdef nogil: ...@@ -344,6 +344,25 @@ cdef nogil:
pych.chan_double().close() pych.chan_double().close()
# verify that pychan_from_raw is not leaking C channel.
def test_pychan_from_raw_noleak():
# pychan_from_raw used to create another channel and leak it
#
# this test _implicitly_ verifies that it is no longer the case - if it is,
# LSAN will report a memory leak after running the test.
#
# TODO consider adding explicit verification effective even under regular
# builds. Possible options:
#
# * verify malloc totals before and after tested code
# see e.g. https://stackoverflow.com/q/1761125/9456786
# * hook _makechan and verify that it is not invoked from under
# pychan_from_raw. Depends on funchook integration.
cdef chan[int] ch = makechan[int]()
cdef pychan pych = pychan.from_chan_int(ch) # uses pychan_from_raw internally
# pych and ch are freed automatically
# ---- benchmarks ---- # ---- benchmarks ----
# bench_go_nogil mirrors golang_test.py:bench_go # bench_go_nogil mirrors golang_test.py:bench_go
......
...@@ -1567,7 +1567,10 @@ def test_strings_methods(): ...@@ -1567,7 +1567,10 @@ def test_strings_methods():
# checkop verifies that `s.meth(*argv, **kw)` gives the same result for s, # checkop verifies that `s.meth(*argv, **kw)` gives the same result for s,
# argv and kw being various combinations of unicode,bstr,ustr, bytes/bytearray. # argv and kw being various combinations of unicode,bstr,ustr, bytes/bytearray.
def checkop(s, meth, *argv, **kw): def checkop(s, meth, *argv, **kw):
assert type(s) is str if six.PY3:
assert type(s) is str
else:
assert type(s) in (str, unicode) # some tests use unicode because \u does not work in str literals
ok = kw.pop('ok') ok = kw.pop('ok')
if six.PY2: if six.PY2:
ok = deepReplaceStr(ok, xunicode) ok = deepReplaceStr(ok, xunicode)
...@@ -1738,7 +1741,7 @@ def test_strings_methods(): ...@@ -1738,7 +1741,7 @@ def test_strings_methods():
_("123").isnumeric( ok=True) _("123").isnumeric( ok=True)
_("0x123").isnumeric( ok=False) _("0x123").isnumeric( ok=False)
_("мир").isprintable( ok=True, optional=True) # py3.0 _("мир").isprintable( ok=True, optional=True) # py3.0
_("\u2009").isspace( ok=x32(True,False)) # thin space _(u"\u2009").isspace( ok=True) # thin space
_(" ").isspace( ok=True) _(" ").isspace( ok=True)
_("мир").isspace( ok=False) _("мир").isspace( ok=False)
_("мир").istitle( ok=False) _("мир").istitle( ok=False)
...@@ -1748,8 +1751,8 @@ def test_strings_methods(): ...@@ -1748,8 +1751,8 @@ def test_strings_methods():
_("мир").ljust(10, ok="мир ") _("мир").ljust(10, ok="мир ")
_("мир").ljust(10, 'ж', ok="миржжжжжжж") _("мир").ljust(10, 'ж', ok="миржжжжжжж")
_("МиР").lower( ok="мир") _("МиР").lower( ok="мир")
_("\u2009 мир").lstrip( ok=x32("мир", "\u2009 мир")) _(u"\u2009 мир").lstrip( ok="мир")
_("\u2009 мир\u2009 ").lstrip( ok=x32("мир\u2009 ", "\u2009 мир\u2009 ")) _(u"\u2009 мир\u2009 ").lstrip( ok=u"мир\u2009 ")
_("мммир").lstrip('ми', ok="р") _("мммир").lstrip('ми', ok="р")
_("миру мир").partition('ру', ok=("ми", "ру", " мир")) _("миру мир").partition('ру', ok=("ми", "ру", " мир"))
_("миру мир").partition('ж', ok=("миру мир", "", "")) _("миру мир").partition('ж', ok=("миру мир", "", ""))
...@@ -1764,15 +1767,15 @@ def test_strings_methods(): ...@@ -1764,15 +1767,15 @@ def test_strings_methods():
_("миру мир").rpartition('ж', ok=("", "", "миру мир")) _("миру мир").rpartition('ж', ok=("", "", "миру мир"))
_("мир").rsplit( ok=["мир"]) _("мир").rsplit( ok=["мир"])
_("привет мир").rsplit( ok=["привет", "мир"]) _("привет мир").rsplit( ok=["привет", "мир"])
_("привет\u2009мир").rsplit( ok=x32(["привет", "мир"], ["привет\u2009мир"])) _(u"привет\u2009мир").rsplit( ok=["привет", "мир"])
_("привет мир").rsplit("и", ok=["пр", "вет м", "р"]) _("привет мир").rsplit("и", ok=["пр", "вет м", "р"])
_("привет мир").rsplit("и", 1, ok=["привет м", "р"]) _("привет мир").rsplit("и", 1, ok=["привет м", "р"])
_("мир \u2009").rstrip( ok=x32("мир", "мир \u2009")) _(u"мир \u2009").rstrip( ok="мир")
_(" мир \u2009").rstrip( ok=x32(" мир", " мир \u2009")) _(u" мир \u2009").rstrip( ok=" мир")
_("мируу").rstrip('ру', ok="ми") _("мируу").rstrip('ру', ok="ми")
_("мир").split( ok=["мир"]) _("мир").split( ok=["мир"])
_("привет мир").split( ok=["привет", "мир"]) _("привет мир").split( ok=["привет", "мир"])
_("привет\u2009мир").split( ok=x32(['привет', 'мир'], ["привет\u2009мир"])) _(u"привет\u2009мир").split( ok=['привет', 'мир'])
_("привет мир").split("и", ok=["пр", "вет м", "р"]) _("привет мир").split("и", ok=["пр", "вет м", "р"])
_("привет мир").split("и", 1, ok=["пр", "вет мир"]) _("привет мир").split("и", 1, ok=["пр", "вет мир"])
_("мир").splitlines( ok=["мир"]) _("мир").splitlines( ok=["мир"])
...@@ -1782,11 +1785,12 @@ def test_strings_methods(): ...@@ -1782,11 +1785,12 @@ def test_strings_methods():
_("мир\nтруд\nмай\n").splitlines( ok=["мир", "труд", "май"]) _("мир\nтруд\nмай\n").splitlines( ok=["мир", "труд", "май"])
_("мир\nтруд\nмай\n").splitlines(True, ok=["мир\n", "труд\n", "май\n"]) _("мир\nтруд\nмай\n").splitlines(True, ok=["мир\n", "труд\n", "май\n"])
# startswith - tested in test_strings_index # startswith - tested in test_strings_index
_("\u2009 мир \u2009").strip( ok=x32("мир", "\u2009 мир \u2009")) _(u"\u2009 мир \u2009").strip( ok="мир")
_("миру мир").strip('мир', ok="у ") _("миру мир").strip('мир', ok="у ")
_("МиР").swapcase( ok="мИр") _("МиР").swapcase( ok="мИр")
_("МиР").title( ok="Мир") _("МиР").title( ok="Мир")
_("мир").translate({ord(u'м'):ord(u'и'), ord(u'и'):'я', ord(u'р'):None}, ok="ия") _("мир").translate({ord(u'м'):ord(u'и'), ord(u'и'):'я', ord(u'р'):None}, ok="ия")
_(u"\u0000\u0001\u0002.").translate([u'м', ord(u'и'), None], ok="ми.")
_("МиР").upper( ok="МИР") _("МиР").upper( ok="МИР")
_("мир").zfill(10, ok="0000000мир") _("мир").zfill(10, ok="0000000мир")
_("123").zfill(10, ok="0000000123") _("123").zfill(10, ok="0000000123")
......
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# Copyright (C) 2018-2023 Nexedi SA and Contributors. # Copyright (C) 2018-2024 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com> # Kirill Smelkov <kirr@nexedi.com>
# #
# This program is free software: you can Use, Study, Modify and Redistribute # This program is free software: you can Use, Study, Modify and Redistribute
...@@ -74,7 +74,8 @@ import_pyx_tests("golang._golang_test") ...@@ -74,7 +74,8 @@ import_pyx_tests("golang._golang_test")
# leaked goroutine behaviour check: done in separate process because we need # leaked goroutine behaviour check: done in separate process because we need
# to test process termination exit there. # to test process termination exit there.
def test_go_leaked(): def test_go_leaked():
pyrun([dir_testprog + "/golang_test_goleaked.py"]) pyrun([dir_testprog + "/golang_test_goleaked.py"],
lsan=False) # there are on-purpose leaks in this test
# benchmark go+join a thread/coroutine. # benchmark go+join a thread/coroutine.
# pyx/nogil mirror is in _golang_test.pyx # pyx/nogil mirror is in _golang_test.pyx
...@@ -1756,6 +1757,11 @@ def _pyrun(argv, stdin=None, stdout=None, stderr=None, **kw): # -> retcode, st ...@@ -1756,6 +1757,11 @@ def _pyrun(argv, stdin=None, stdout=None, stderr=None, **kw): # -> retcode, st
assert len(enc) == 1 assert len(enc) == 1
env['PYTHONIOENCODING'] = enc.pop() env['PYTHONIOENCODING'] = enc.pop()
# disable LeakSanitizer if requested, e.g. when test is known to leak something on purpose
lsan = kw.pop('lsan', True)
if not lsan:
env['ASAN_OPTIONS'] = env.get('ASAN_OPTIONS', '') + ',detect_leaks=0'
p = Popen(argv, stdin=(PIPE if stdin else None), stdout=stdout, stderr=stderr, env=env, **kw) p = Popen(argv, stdin=(PIPE if stdin else None), stdout=stdout, stderr=stderr, env=env, **kw)
stdout, stderr = p.communicate(stdin) stdout, stderr = p.communicate(stdin)
......
...@@ -345,8 +345,13 @@ typedef struct _libgolang_runtime_ops { ...@@ -345,8 +345,13 @@ typedef struct _libgolang_runtime_ops {
// previously successfully allocated via sema_alloc. // previously successfully allocated via sema_alloc.
void (*sema_free) (_libgolang_sema*); void (*sema_free) (_libgolang_sema*);
// sema_acquire/sema_release should acquire/release live semaphore allocated via sema_alloc. // sema_acquire should try to acquire live semaphore allocated via sema_alloc during given time.
void (*sema_acquire)(_libgolang_sema*); // it returns whether acquisition succeeded or timed out.
// the timeout is specified in nanoseconds.
// UINT64_MAX means no timeout.
bool (*sema_acquire)(_libgolang_sema*, uint64_t timeout_ns);
// sema_release should release live semaphore allocated via sema_alloc.
void (*sema_release)(_libgolang_sema*); void (*sema_release)(_libgolang_sema*);
// nanosleep should pause current goroutine for at least dt nanoseconds. // nanosleep should pause current goroutine for at least dt nanoseconds.
......
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# Copyright (C) 2019 Nexedi SA and Contributors. # Copyright (C) 2019-2024 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com> # Kirill Smelkov <kirr@nexedi.com>
# #
# This program is free software: you can Use, Study, Modify and Redistribute # This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your # it under the terms of the GNU General Public License version 3, or (at your
...@@ -28,7 +28,8 @@ testprog = dirname(__file__) + "/testprog" ...@@ -28,7 +28,8 @@ testprog = dirname(__file__) + "/testprog"
# verify that we can build/run external package that uses pygolang in pyx mode. # verify that we can build/run external package that uses pygolang in pyx mode.
def test_pyx_build(): def test_pyx_build():
pyxuser = testprog + "/golang_pyx_user" pyxuser = testprog + "/golang_pyx_user"
pyrun(["setup.py", "build_ext", "-i"], cwd=pyxuser) pyrun(["setup.py", "build_ext", "-i"], cwd=pyxuser,
lsan=False) # gcc leaks
# run built test. # run built test.
_ = pyout(["-c", _ = pyout(["-c",
...@@ -44,8 +45,8 @@ def test_pyx_build(): ...@@ -44,8 +45,8 @@ def test_pyx_build():
# verify that we can build/run external dso that uses libgolang. # verify that we can build/run external dso that uses libgolang.
def test_dso_build(): def test_dso_build():
dsouser = testprog + "/golang_dso_user" dsouser = testprog + "/golang_dso_user"
pyrun(["setup.py", "build_dso", "-i"], cwd=dsouser) pyrun(["setup.py", "build_dso", "-i"], cwd=dsouser, lsan=False) # gcc leaks
pyrun(["setup.py", "build_ext", "-i"], cwd=dsouser) pyrun(["setup.py", "build_ext", "-i"], cwd=dsouser, lsan=False) # gcc leaks
# run built test. # run built test.
_ = pyout(["-c", _ = pyout(["-c",
......
# cython: language_level=2 # cython: language_level=2
# Copyright (C) 2019-2022 Nexedi SA and Contributors. # Copyright (C) 2019-2024 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com> # Kirill Smelkov <kirr@nexedi.com>
# #
# This program is free software: you can Use, Study, Modify and Redistribute # This program is free software: you can Use, Study, Modify and Redistribute
...@@ -36,7 +36,7 @@ cdef extern from "golang/libgolang.h" namespace "golang" nogil: ...@@ -36,7 +36,7 @@ cdef extern from "golang/libgolang.h" namespace "golang" nogil:
_libgolang_sema* (*sema_alloc) () _libgolang_sema* (*sema_alloc) ()
void (*sema_free) (_libgolang_sema*) void (*sema_free) (_libgolang_sema*)
void (*sema_acquire)(_libgolang_sema*) bint (*sema_acquire)(_libgolang_sema*, uint64_t timeout_ns)
void (*sema_release)(_libgolang_sema*) void (*sema_release)(_libgolang_sema*)
void (*nanosleep)(uint64_t) void (*nanosleep)(uint64_t)
......
# cython: language_level=2 # cython: language_level=2
# Copyright (C) 2019-2023 Nexedi SA and Contributors. # Copyright (C) 2019-2024 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com> # Kirill Smelkov <kirr@nexedi.com>
# #
# This program is free software: you can Use, Study, Modify and Redistribute # This program is free software: you can Use, Study, Modify and Redistribute
...@@ -40,7 +40,10 @@ ELSE: ...@@ -40,7 +40,10 @@ ELSE:
from gevent import sleep as pygsleep from gevent import sleep as pygsleep
from libc.stdint cimport uint64_t from libc.stdint cimport uint8_t, uint64_t, UINT64_MAX
cdef extern from *:
ctypedef bint cbool "bool"
from cpython cimport PyObject, Py_INCREF, Py_DECREF from cpython cimport PyObject, Py_INCREF, Py_DECREF
from cython cimport final from cython cimport final
...@@ -95,9 +98,12 @@ cdef: ...@@ -95,9 +98,12 @@ cdef:
Py_DECREF(pygsema) Py_DECREF(pygsema)
return True return True
bint _sema_acquire(_libgolang_sema *gsema): bint _sema_acquire(_libgolang_sema *gsema, uint64_t timeout_ns, cbool* pacq):
pygsema = <PYGSema>gsema pygsema = <PYGSema>gsema
pygsema.acquire() timeout = None
if timeout_ns != UINT64_MAX:
timeout = float(timeout_ns) * 1e-9
pacq[0] = pygsema.acquire(timeout=timeout)
return True return True
bint _sema_release(_libgolang_sema *gsema): bint _sema_release(_libgolang_sema *gsema):
...@@ -142,14 +148,16 @@ cdef nogil: ...@@ -142,14 +148,16 @@ cdef nogil:
if not ok: if not ok:
panic("pyxgo: gevent: sema: free: failed") panic("pyxgo: gevent: sema: free: failed")
void sema_acquire(_libgolang_sema *gsema): cbool sema_acquire(_libgolang_sema *gsema, uint64_t timeout_ns):
cdef PyExc exc cdef PyExc exc
cdef cbool acq
with gil: with gil:
pyexc_fetch(&exc) pyexc_fetch(&exc)
ok = _sema_acquire(gsema) ok = _sema_acquire(gsema, timeout_ns, &acq)
pyexc_restore(exc) pyexc_restore(exc)
if not ok: if not ok:
panic("pyxgo: gevent: sema: acquire: failed") panic("pyxgo: gevent: sema: acquire: failed")
return acq
void sema_release(_libgolang_sema *gsema): void sema_release(_libgolang_sema *gsema):
cdef PyExc exc cdef PyExc exc
......
# cython: language_level=2 # cython: language_level=2
# Copyright (C) 2019-2022 Nexedi SA and Contributors. # Copyright (C) 2019-2024 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com> # Kirill Smelkov <kirr@nexedi.com>
# #
# This program is free software: you can Use, Study, Modify and Redistribute # This program is free software: you can Use, Study, Modify and Redistribute
...@@ -35,7 +35,12 @@ from __future__ import print_function, absolute_import ...@@ -35,7 +35,12 @@ from __future__ import print_function, absolute_import
# #
# NOTE Cython declares PyThread_acquire_lock/PyThread_release_lock as nogil # NOTE Cython declares PyThread_acquire_lock/PyThread_release_lock as nogil
from cpython.pythread cimport PyThread_acquire_lock, PyThread_release_lock, \ from cpython.pythread cimport PyThread_acquire_lock, PyThread_release_lock, \
PyThread_type_lock, WAIT_LOCK PyThread_type_lock, WAIT_LOCK, NOWAIT_LOCK, PyLockStatus, PY_LOCK_ACQUIRED, PY_LOCK_FAILURE
cdef extern from * nogil:
ctypedef int PY_TIMEOUT_T # long long there
PyLockStatus PyThread_acquire_lock_timed(PyThread_type_lock, PY_TIMEOUT_T timeout_us, int intr_flag)
# NOTE On Darwin, even though this is considered as POSIX, Python uses # NOTE On Darwin, even though this is considered as POSIX, Python uses
# mutex+condition variable to implement its lock, and, as of 20190828, Py2.7 # mutex+condition variable to implement its lock, and, as of 20190828, Py2.7
...@@ -98,6 +103,9 @@ from libc.errno cimport errno, EINTR, EBADF ...@@ -98,6 +103,9 @@ from libc.errno cimport errno, EINTR, EBADF
from posix.fcntl cimport mode_t from posix.fcntl cimport mode_t
from posix.stat cimport struct_stat from posix.stat cimport struct_stat
from posix.strings cimport bzero from posix.strings cimport bzero
cdef extern from *:
ctypedef bint cbool "bool"
IF POSIX: IF POSIX:
from posix.time cimport clock_gettime, nanosleep as posix_nanosleep, timespec, CLOCK_REALTIME from posix.time cimport clock_gettime, nanosleep as posix_nanosleep, timespec, CLOCK_REALTIME
ELSE: ELSE:
...@@ -138,11 +146,46 @@ cdef nogil: ...@@ -138,11 +146,46 @@ cdef nogil:
pysema = <PyThread_type_lock>gsema pysema = <PyThread_type_lock>gsema
PyThread_free_lock(pysema) PyThread_free_lock(pysema)
void sema_acquire(_libgolang_sema *gsema): cbool sema_acquire(_libgolang_sema *gsema, uint64_t timeout_ns):
pysema = <PyThread_type_lock>gsema pysema = <PyThread_type_lock>gsema
ok = PyThread_acquire_lock(pysema, WAIT_LOCK) IF PY3:
if ok == 0: cdef PY_TIMEOUT_T timeout_us
panic("pyxgo: thread: sema_acquire: PyThread_acquire_lock failed") ELSE:
cdef uint64_t tprev, t, tsleep
if timeout_ns == UINT64_MAX:
ok = PyThread_acquire_lock(pysema, WAIT_LOCK)
if ok == 0:
panic("pyxgo: thread: sema_acquire: PyThread_acquire_lock failed")
return 1
else:
IF PY3:
timeout_us = timeout_ns // 1000
lkok = PyThread_acquire_lock_timed(pysema, timeout_us, 0)
if lkok == PY_LOCK_FAILURE:
return 0
elif lkok == PY_LOCK_ACQUIRED:
return 1
else:
panic("pyxgo: thread: sema_acquire: PyThread_acquire_lock_timed failed")
ELSE:
# py2 misses PyThread_acquire_lock_timed - provide fallback ourselves
tprev = nanotime()
while 1:
ok = PyThread_acquire_lock(pysema, NOWAIT_LOCK)
if ok:
return 1
tsleep = min(timeout_ns, 50*1000) # poll every 50 μs = 20 Hz
if tsleep == 0:
break
nanosleep(tsleep)
t = nanotime()
if t < tprev:
break # clock skew
if t - tprev >= timeout_ns:
break
timeout_ns -= t - tprev
tprev = t
return 0
void sema_release(_libgolang_sema *gsema): void sema_release(_libgolang_sema *gsema):
pysema = <PyThread_type_lock>gsema pysema = <PyThread_type_lock>gsema
......
...@@ -131,6 +131,7 @@ using internal::_runtime; ...@@ -131,6 +131,7 @@ using internal::_runtime;
namespace internal { namespace atomic { extern void _init(); } } namespace internal { namespace atomic { extern void _init(); } }
namespace os { namespace signal { extern void _init(); } } namespace os { namespace signal { extern void _init(); } }
namespace time { extern void _init(); }
void _libgolang_init(const _libgolang_runtime_ops *runtime_ops) { void _libgolang_init(const _libgolang_runtime_ops *runtime_ops) {
if (_runtime != nil) // XXX better check atomically if (_runtime != nil) // XXX better check atomically
panic("libgolang: double init"); panic("libgolang: double init");
...@@ -138,6 +139,7 @@ void _libgolang_init(const _libgolang_runtime_ops *runtime_ops) { ...@@ -138,6 +139,7 @@ void _libgolang_init(const _libgolang_runtime_ops *runtime_ops) {
internal::atomic::_init(); internal::atomic::_init();
os::signal::_init(); os::signal::_init();
time::_init();
} }
void _taskgo(void (*f)(void *), void *arg) { void _taskgo(void (*f)(void *), void *arg) {
...@@ -166,7 +168,15 @@ void _semafree(_sema *sema) { ...@@ -166,7 +168,15 @@ void _semafree(_sema *sema) {
} }
void _semaacquire(_sema *sema) { void _semaacquire(_sema *sema) {
_runtime->sema_acquire((_libgolang_sema *)sema); bool ok;
ok = _runtime->sema_acquire((_libgolang_sema *)sema, UINT64_MAX);
if (!ok)
panic("semaacquire: failed");
}
// NOTE not currently exposed in public API
bool _semaacquire_timed(_sema *sema, uint64_t timeout_ns) {
return _runtime->sema_acquire((_libgolang_sema *)sema, timeout_ns);
} }
void _semarelease(_sema *sema) { void _semarelease(_sema *sema) {
......
This diff is collapsed.
#ifndef _NXD_LIBGOLANG_TIME_H #ifndef _NXD_LIBGOLANG_TIME_H
#define _NXD_LIBGOLANG_TIME_H #define _NXD_LIBGOLANG_TIME_H
// Copyright (C) 2019-2023 Nexedi SA and Contributors. // Copyright (C) 2019-2024 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com> // Kirill Smelkov <kirr@nexedi.com>
// //
// This program is free software: you can Use, Study, Modify and Redistribute // This program is free software: you can Use, Study, Modify and Redistribute
...@@ -118,6 +118,7 @@ private: ...@@ -118,6 +118,7 @@ private:
double _dt; double _dt;
sync::Mutex _mu; sync::Mutex _mu;
bool _stop; bool _stop;
Timer _timer;
// don't new - create only via new_ticker() // don't new - create only via new_ticker()
private: private:
...@@ -147,18 +148,12 @@ LIBGOLANG_API Timer new_timer(double dt); ...@@ -147,18 +148,12 @@ LIBGOLANG_API Timer new_timer(double dt);
struct _Timer : object { struct _Timer : object {
chan<double> c; chan<double> c;
private:
func<void()> _f;
sync::Mutex _mu;
double _dt; // +inf - stopped, otherwise - armed
int _ver; // current timer was armed by n'th reset
// don't new - create only via new_timer() & co // don't new - create only via new_timer() & co
private: private:
_Timer(); _Timer();
~_Timer(); ~_Timer();
friend Timer _new_timer(double dt, func<void()> f); friend Timer _new_timer(double dt, func<void()> f);
friend class _TimerImpl;
public: public:
LIBGOLANG_API void decref(); LIBGOLANG_API void decref();
...@@ -182,9 +177,6 @@ public: ...@@ -182,9 +177,6 @@ public:
// //
// the timer must be either already stopped or expired. // the timer must be either already stopped or expired.
LIBGOLANG_API void reset(double dt); LIBGOLANG_API void reset(double dt);
private:
void _fire(double dt, int ver);
}; };
......
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# Copyright (C) 2019 Nexedi SA and Contributors. # Copyright (C) 2019-2024 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com> # Kirill Smelkov <kirr@nexedi.com>
# #
# This program is free software: you can Use, Study, Modify and Redistribute # This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your # it under the terms of the GNU General Public License version 3, or (at your
...@@ -20,9 +20,10 @@ ...@@ -20,9 +20,10 @@
from __future__ import print_function, absolute_import from __future__ import print_function, absolute_import
from golang import select from golang import select, func, defer
from golang import time from golang import time, sync
from golang.golang_test import panics from golang.golang_test import panics
from six.moves import range as xrange
# all timer tests operate in dt units # all timer tests operate in dt units
dt = 10*time.millisecond dt = 10*time.millisecond
...@@ -65,6 +66,7 @@ def test_ticker_time(): ...@@ -65,6 +66,7 @@ def test_ticker_time():
# test_timer verifies that Timer/Ticker fire as expected. # test_timer verifies that Timer/Ticker fire as expected.
@func
def test_timer(): def test_timer():
# start timers at x5, x7 and x11 intervals an verify that the timers fire # start timers at x5, x7 and x11 intervals an verify that the timers fire
# in expected sequence. The times when the timers fire do not overlap in # in expected sequence. The times when the timers fire do not overlap in
...@@ -73,15 +75,15 @@ def test_timer(): ...@@ -73,15 +75,15 @@ def test_timer():
tv = [] # timer events tv = [] # timer events
Tstart = time.now() Tstart = time.now()
t23 = time.Timer(23*dt) t23 = time.Timer(23*dt); defer(t23.stop)
t5 = time.Timer( 5*dt) t5 = time.Timer( 5*dt); defer(t5 .stop)
def _(): def _():
tv.append(7) tv.append(7)
t7f.reset(7*dt) t7f.reset(7*dt)
t7f = time.Timer( 7*dt, f=_) t7f = time.Timer( 7*dt, f=_); defer(t7f.stop)
tx11 = time.Ticker(11*dt) tx11 = time.Ticker(11*dt); defer(tx11.stop)
while 1: while 1:
_, _rx = select( _, _rx = select(
...@@ -108,19 +110,20 @@ def test_timer(): ...@@ -108,19 +110,20 @@ def test_timer():
# test_timer_misc, similarly to test_timer, verifies misc timer convenience functions. # test_timer_misc, similarly to test_timer, verifies misc timer convenience functions.
@func
def test_timer_misc(): def test_timer_misc():
tv = [] tv = []
Tstart = time.now() Tstart = time.now()
c23 = time.after(23*dt) c23 = time.after(23*dt) # cannot stop
c5 = time.after( 5*dt) c5 = time.after( 5*dt) # cannot stop
def _(): def _():
tv.append(7) tv.append(7)
t7f.reset(7*dt) t7f.reset(7*dt)
t7f = time.after_func(7*dt, _) t7f = time.after_func(7*dt, _); defer(t7f.stop)
cx11 = time.tick(11*dt) cx11 = time.tick(11*dt) # cannot stop
while 1: while 1:
_, _rx = select( _, _rx = select(
...@@ -148,13 +151,14 @@ def test_timer_misc(): ...@@ -148,13 +151,14 @@ def test_timer_misc():
# test_timer_stop verifies that .stop() cancels Timer or Ticker. # test_timer_stop verifies that .stop() cancels Timer or Ticker.
@func
def test_timer_stop(): def test_timer_stop():
tv = [] tv = []
t10 = time.Timer (10*dt) t10 = time.Timer (10*dt); defer(t10.stop)
t2 = time.Timer ( 2*dt) # will fire and cancel t3, tx5 t2 = time.Timer ( 2*dt); defer(t2 .stop) # will fire and cancel t3, tx5
t3 = time.Timer ( 3*dt) # will be canceled t3 = time.Timer ( 3*dt); defer(t3 .stop) # will be canceled
tx5 = time.Ticker( 5*dt) # will be canceled tx5 = time.Ticker( 5*dt); defer(tx5.stop) # will be canceled
while 1: while 1:
_, _rx = select( _, _rx = select(
...@@ -180,9 +184,10 @@ def test_timer_stop(): ...@@ -180,9 +184,10 @@ def test_timer_stop():
# test_timer_stop_drain verifies that Timer/Ticker .stop() drains timer channel. # test_timer_stop_drain verifies that Timer/Ticker .stop() drains timer channel.
@func
def test_timer_stop_drain(): def test_timer_stop_drain():
t = time.Timer (1*dt) t = time.Timer (1*dt); defer(t.stop)
tx = time.Ticker(1*dt) tx = time.Ticker(1*dt); defer(tx.stop)
time.sleep(2*dt) time.sleep(2*dt)
assert len(t.c) == 1 assert len(t.c) == 1
...@@ -195,9 +200,45 @@ def test_timer_stop_drain(): ...@@ -195,9 +200,45 @@ def test_timer_stop_drain():
assert len(tx.c) == 0 assert len(tx.c) == 0
# test_timer_stop_vs_func verifies that Timer .stop() works correctly with func-timer.
@func
def test_timer_stop_vs_func():
tv = []
def _1(): tv.append(1)
def _2(): tv.append(2)
t1 = time.after_func(1e6*dt, _1); defer(t1.stop)
t2 = time.after_func( 1*dt, _2); defer(t2.stop)
time.sleep(2*dt)
assert t1.stop() == True
assert t2.stop() == False
assert tv == [2]
# test_timer_reset_armed verifies that .reset() panics if called on armed timer. # test_timer_reset_armed verifies that .reset() panics if called on armed timer.
@func
def test_timer_reset_armed(): def test_timer_reset_armed():
# reset while armed # reset while armed
t = time.Timer(10*dt) t = time.Timer(10*dt); defer(t.stop)
with panics("Timer.reset: the timer is armed; must be stopped or expired"): with panics("Timer.reset: the timer is armed; must be stopped or expired"):
t.reset(5*dt) t.reset(5*dt)
# bench_timer_arm_cancel benchmarks arming timers that do not fire.
# it shows how cheap or expensive it is to use timers to implement timeouts.
def bench_timer_arm_cancel(b):
for i in xrange(b.N):
t = time.Timer(10*time.second)
_ = t.stop()
assert _ is True
# bench_timer_arm_fire benchmarks arming timers that do fire.
# it shows what it costs to go through all steps related to timer loop and firing timers.
def bench_timer_arm_fire(b):
wg = sync.WaitGroup()
wg.add(b.N)
for i in xrange(b.N):
t = time.after_func(1*time.millisecond, wg.done)
wg.wait()
...@@ -188,7 +188,7 @@ class develop(XInstallGPython, _develop): ...@@ -188,7 +188,7 @@ class develop(XInstallGPython, _develop):
# requirements of packages under "golang." namespace # requirements of packages under "golang." namespace
R = { R = {
'cmd.pybench': {'pytest', 'py'}, 'cmd.pybench': {'pytest', 'py ; python_version >= "3"'},
'pyx.build': {'setuptools', 'wheel', 'cython < 3', 'setuptools_dso >= 2.8'}, 'pyx.build': {'setuptools', 'wheel', 'cython < 3', 'setuptools_dso >= 2.8'},
'x.perf.benchlib': {'numpy'}, 'x.perf.benchlib': {'numpy'},
} }
...@@ -467,8 +467,11 @@ setup( ...@@ -467,8 +467,11 @@ setup(
'golang/os/signal.h', 'golang/os/signal.h',
'golang/strings.h', 'golang/strings.h',
'golang/sync.h', 'golang/sync.h',
'golang/time.h'], 'golang/time.h',
include_dirs = ['3rdparty/include'], '3rdparty/ratas/src/timer-wheel.h'],
include_dirs = [
'3rdparty/include',
'3rdparty/ratas/src'],
define_macros = [('BUILDING_LIBGOLANG', None)], define_macros = [('BUILDING_LIBGOLANG', None)],
soversion = '0.1'), soversion = '0.1'),
...@@ -604,9 +607,6 @@ setup( ...@@ -604,9 +607,6 @@ setup(
Programming Language :: Python :: 2 Programming Language :: Python :: 2
Programming Language :: Python :: 2.7 Programming Language :: Python :: 2.7
Programming Language :: Python :: 3 Programming Language :: Python :: 3
Programming Language :: Python :: 3.5
Programming Language :: Python :: 3.6
Programming Language :: Python :: 3.7
Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.8
Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.9
Programming Language :: Python :: 3.10 Programming Language :: Python :: 3.10
......
[tox] [tox]
envlist = envlist =
{py27d,py27,py37,py38,py39d,py39,py310d,py310,py311d,py311,py312,pypy,pypy3}-{thread,gevent} {py27d,py27,py38,py39d,py39,py310d,py310,py311d,py311,py312,pypy,pypy3}-{thread,gevent}
# ThreadSanitizer # ThreadSanitizer
...@@ -10,24 +10,23 @@ envlist = ...@@ -10,24 +10,23 @@ envlist =
# (*) PyPy locks its GIL (see RPyGilAcquire) by manually doing atomic cmpxchg # (*) PyPy locks its GIL (see RPyGilAcquire) by manually doing atomic cmpxchg
# and other games, which TSAN cannot see if PyPy itself was not compiled with # and other games, which TSAN cannot see if PyPy itself was not compiled with
# -fsanitize=thread. # -fsanitize=thread.
{py27d,py27,py37,py38,py39d,py39,py310d,py310,py311d,py311,py312 }-{thread }-tsan {py27d,py27,py38,py39d,py39,py310d,py310,py311d,py311,py312 }-{thread }-tsan
# XXX py*-gevent-tsan would be nice to have, but at present TSAN is not # XXX py*-gevent-tsan would be nice to have, but at present TSAN is not
# effective with gevent, because it does not understand greenlet "thread" # effective with gevent, because it does not understand greenlet "thread"
# switching and so perceives the program as having only one thread where races # switching and so perceives the program as having only one thread where races
# are impossible. Disabled to save time. # are impossible. Disabled to save time.
# {py27d,py27,py37,py38,py39d,py39,py310d,py310,py311d,py311,py312 }-{ gevent}-tsan # {py27d,py27,py38,py39d,py39,py310d,py310,py311d,py311,py312 }-{ gevent}-tsan
# AddressSanitizer # AddressSanitizer
# XXX asan does not work with gevent: https://github.com/python-greenlet/greenlet/issues/113 # XXX asan does not work with gevent: https://github.com/python-greenlet/greenlet/issues/113
{py27d,py27,py37,py38,py39d,py39,py310d,py310,py311d,py311,py312,pypy,pypy3}-{thread }-asan {py27d,py27,py38,py39d,py39,py310d,py310,py311d,py311,py312,pypy,pypy3}-{thread }-asan
[testenv] [testenv]
basepython = basepython =
py27d: python2.7-dbg py27d: python2.7-dbg
py27: python2.7 py27: python2.7
py37: python3.7
py38: python3.8 py38: python3.8
py39d: python3.9-dbg py39d: python3.9-dbg
py39: python3.9 py39: python3.9
...@@ -43,16 +42,16 @@ basepython = ...@@ -43,16 +42,16 @@ basepython =
setenv = setenv =
# distutils take CFLAGS for both C and C++. # distutils take CFLAGS for both C and C++.
# distutils use CFLAGS also at link stage -> we don't need to set LDFLAGS separately. # distutils use CFLAGS also at link stage -> we don't need to set LDFLAGS separately.
tsan: CFLAGS=-g -fsanitize=thread tsan: CFLAGS=-g -fsanitize=thread -fno-omit-frame-pointer
asan: CFLAGS=-g -fsanitize=address asan: CFLAGS=-g -fsanitize=address -fno-omit-frame-pointer
# XXX however distutils' try_link, which is used by numpy.distutils use only CC # XXX however distutils' try_link, which is used by numpy.distutils use only CC
# as linker without CFLAGS and _without_ LDFLAGS, which fails if *.o were # as linker without CFLAGS and _without_ LDFLAGS, which fails if *.o were
# compiled with -fsanitize=X and linked without that option. Work it around # compiled with -fsanitize=X and linked without that option. Work it around
# with also adjusting CC. # with also adjusting CC.
# XXX better arrange to pass CFLAGS to pygolang only, e.g. by adding --race or # XXX better arrange to pass CFLAGS to pygolang only, e.g. by adding --race or
# --sanitize=thread to `setup.py build_ext`. # --sanitize=thread to `setup.py build_ext`.
tsan: CC=cc -fsanitize=thread tsan: CC=cc -fsanitize=thread -fno-omit-frame-pointer
asan: CC=cc -fsanitize=address asan: CC=cc -fsanitize=address -fno-omit-frame-pointer
# always compile pygolang from source and don't reuse binary pygolang wheels as # always compile pygolang from source and don't reuse binary pygolang wheels as
# we compile each case with different CFLAGS. # we compile each case with different CFLAGS.
...@@ -76,3 +75,5 @@ commands= ...@@ -76,3 +75,5 @@ commands=
# likewise for python debug builds. # likewise for python debug builds.
asan,tsan,py{27,39,310,311,312}d: -s \ asan,tsan,py{27,39,310,311,312}d: -s \
gpython/ golang/ gpython/ golang/
allowlist_externals={toxinidir}/trun
#!/usr/bin/env python #!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright (C) 2019-2024 Nexedi SA and Contributors. # Copyright (C) 2019-2024 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com> # Kirill Smelkov <kirr@nexedi.com>
# #
...@@ -34,7 +35,7 @@ trun cares to run python with LD_PRELOAD set appropriately to /path/to/libtsan.s ...@@ -34,7 +35,7 @@ trun cares to run python with LD_PRELOAD set appropriately to /path/to/libtsan.s
from __future__ import print_function, absolute_import from __future__ import print_function, absolute_import
import os, sys, re, subprocess, types import os, os.path, sys, re, subprocess, platform, types
PY3 = (bytes is not str) PY3 = (bytes is not str)
if PY3: if PY3:
from importlib import machinery as imp_machinery from importlib import machinery as imp_machinery
...@@ -87,6 +88,7 @@ def main(): ...@@ -87,6 +88,7 @@ def main():
# determine if _golang.so is linked to a sanitizer, and if yes, to which # determine if _golang.so is linked to a sanitizer, and if yes, to which
# particular sanitizer DSO. Set LD_PRELOAD appropriately. # particular sanitizer DSO. Set LD_PRELOAD appropriately.
libxsan = None
ld_preload = None ld_preload = None
if 'linux' in sys.platform: if 'linux' in sys.platform:
p = subprocess.Popen(["ldd", _golang_so.path], stdout=subprocess.PIPE) p = subprocess.Popen(["ldd", _golang_so.path], stdout=subprocess.PIPE)
...@@ -127,7 +129,8 @@ def main(): ...@@ -127,7 +129,8 @@ def main():
_ = grep1("DYLD_INSERT_LIBRARIES=(.*)$", err) _ = grep1("DYLD_INSERT_LIBRARIES=(.*)$", err)
if _ is not None: if _ is not None:
ld_preload = ("DYLD_INSERT_LIBRARIES", _.group(1)) libxsan = _.group(1)
ld_preload = ("DYLD_INSERT_LIBRARIES", libxsan)
else: else:
print("trun %r: `import golang` failed with unexpected error:" % sys.argv[1:], file=sys.stderr) print("trun %r: `import golang` failed with unexpected error:" % sys.argv[1:], file=sys.stderr)
print(err, file=sys.stderr) print(err, file=sys.stderr)
...@@ -144,7 +147,7 @@ def main(): ...@@ -144,7 +147,7 @@ def main():
env_prepend("TSAN_OPTIONS", "halt_on_error=1") env_prepend("TSAN_OPTIONS", "halt_on_error=1")
env_prepend("ASAN_OPTIONS", "halt_on_error=1") env_prepend("ASAN_OPTIONS", "halt_on_error=1")
# tweak TSAN/ASAN defaults: # tweak TSAN/ASAN/LSAN defaults:
# enable TSAN deadlock detector # enable TSAN deadlock detector
# (unfortunately it caughts only few _potential_ deadlocks and actually # (unfortunately it caughts only few _potential_ deadlocks and actually
...@@ -152,15 +155,49 @@ def main(): ...@@ -152,15 +155,49 @@ def main():
env_prepend("TSAN_OPTIONS", "detect_deadlocks=1") env_prepend("TSAN_OPTIONS", "detect_deadlocks=1")
env_prepend("TSAN_OPTIONS", "second_deadlock_stack=1") env_prepend("TSAN_OPTIONS", "second_deadlock_stack=1")
# many python allocations, whose lifetime coincides with python interpreter
# lifetime and which are not explicitly freed on python shutdown, are
# reported as leaks. Disable leak reporting to avoid huge non-pygolang
# related printouts.
env_prepend("ASAN_OPTIONS", "detect_leaks=0")
# tune ASAN to check more aggressively by default # tune ASAN to check more aggressively by default
env_prepend("ASAN_OPTIONS", "detect_stack_use_after_return=1") env_prepend("ASAN_OPTIONS", "detect_stack_use_after_return=1")
# enable ASAN/LSAN leak detector.
#
# Do it only on CPython ≥ 3.11 because on py2 and on earlier py3 versions
# there are many many python allocations, whose lifetime coincide with
# python interpreter lifetime, and which are not explicitly freed on python
# shutdown. For py3 they significantly improved this step by step and
# starting from 3.11 it becomes practical to silence some still-leaks with
# suppressions, while for earlier py3 versions and especially for py2 it
# is, unfortunately, not manageable. Do not spend engineering time with
# activating LSAN on PyPy as that is tier 2 platform and bug tail history
# of memory leaks is very long even only on cpython.
if sys.version_info < (3,11):
env_prepend("ASAN_OPTIONS", "detect_leaks=0")
if libxsan is not None:
if 'asan' in libxsan.lower():
print("W: trun %r: asan: leak detection deactivated on %s %s" % (
sys.argv[1:], platform.python_implementation(), platform.python_version()),
file=sys.stderr)
else:
env_prepend("ASAN_OPTIONS", "detect_leaks=1")
env_prepend("LSAN_OPTIONS", "suppressions=%s" % os.path.abspath(os.path.join(
os.path.dirname(__file__), ".lsan-ignore.txt")))
# do not print statistics for suppressed leaks - else it breaks tests that verify program output
env_prepend("LSAN_OPTIONS", "print_suppressions=0")
# enable DWARF-based unwinding.
# else, if python is not compiled with -fno-omit-frame-pointer, it can show
# the whole traceback as e.g. just
# Direct leak of 32 byte(s) in 1 object(s) allocated from:
# #0 0x7f88522f3bd7 in malloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:69
# #1 0x55f910a3d9a4 in PyThread_allocate_lock Python/thread_pthread.h:385
# and our leak suppressions won't work.
# this is slower compared to default frame-pointer based unwinding, but
# still works reasonably timely when run with just tests.
env_prepend("ASAN_OPTIONS", "fast_unwind_on_malloc=0")
# leak suppression also needs full tracebacks to work correctly, since with
# python there are many levels of call nesting at C level, and to filter-out e.g.
# top-level PyImport_Import we need to go really deep.
env_prepend("ASAN_OPTIONS", "malloc_context_size=255")
# exec `...` # exec `...`
os.execvp(sys.argv[1], sys.argv[1:]) os.execvp(sys.argv[1], sys.argv[1:])
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment