Skip to content

T
typon-compiler
Commit d6b9c3fe authored by Tom Niget's avatar Tom Niget
Browse files
Options

Move stuff around

parent 90409113
Show whitespace changes
Inline Side-by-side
Showing with 389 additions and 384 deletions
typon/trans/test_runner.py
View file @ d6b9c3fe
......@@ -14,12 +14,13 @@ import colorama
import colorful as cf
import signal
from transpiler import transpile
from transpiler.format import format_code
# load .env file
from dotenv import load_dotenv
from transpiler.transpiler import transpile
colorama.init()
load_dotenv()
......
typon/trans/tests/user_generics.py
View file @ d6b9c3fe
......@@ -23,4 +23,4 @@
if __name__ == "__main__":
x = 5
\ No newline at end of file
print(5)
\ No newline at end of file
typon/trans/transpiler/__init__.py
View file @ d6b9c3fe
# coding: utf-8
import ast
import builtins
import importlib
import inspect
import os
import traceback
from pathlib import Path
#os.environ["TERM"] = "xterm-256"
import colorama
from transpiler.phases.desugar_compare import DesugarCompare
from transpiler.phases.desugar_op import DesugarOp
from transpiler.phases.emit_cpp.module import emit_module
from transpiler.phases.typing import PRELUDE
from transpiler.phases.typing.modules import parse_module
from transpiler.phases.typing.stdlib import StdlibVisitor
colorama.init()
from transpiler.exceptions import CompileError
from transpiler.phases.desugar_with import DesugarWith
#from transpiler.phases.emit_cpp.file import FileVisitor
from transpiler.phases.if_main import IfMainVisitor
#from transpiler.phases.typing.block import ScoperBlockVisitor
from transpiler.phases.typing.scope import Scope, ScopeKind
from transpiler.utils import highlight
import sys
import colorful as cf
from logging import debug
def exception_hook(exc_type, exc_value, tb):
print = lambda *args, **kwargs: builtins.print(*args, **kwargs, file=sys.stderr)
last_node = None
last_file = None
orig_tb = tb
while tb:
local_vars = tb.tb_frame.f_locals
name = tb.tb_frame.f_code.co_name
if name in ("transpile", "parse_module"):
last_file = local_vars["path"]
if name == "visit" and (node := local_vars["node"]) and isinstance(node, ast.AST):
last_node = node
if node := local_vars.get("__TB_NODE__", None):
last_node = node
if local_vars.get("__TB_SKIP__", None) and tb.tb_next:
tb = tb.tb_next
continue
filename = tb.tb_frame.f_code.co_filename
line_no = tb.tb_lineno
print(cf.red(f"File {filename}:{line_no}, in {cf.green(name)}"), end="")
if info := local_vars.get("__TB__", None):
print(f": {cf.magenta(info)}\x1b[24m")
else:
print()
tb = tb.tb_next
if last_node is not None and last_file is not None:
print()
if not hasattr(last_node, "lineno"):
print(cf.red("Error: "), cf.white("No line number available"))
last_node.lineno = 1
print(ast.unparse(last_node))
return
print(f"In file {cf.white(last_file)}:{last_node.lineno}")
#print(f"From {last_node.lineno}:{last_node.col_offset} to {last_node.end_lineno}:{last_node.end_col_offset}")
with open(last_file, "r", encoding="utf-8") as f:
code = f.read()
hg = (str(highlight(code, True))
.replace("\x1b[04m", "")
.replace("\x1b[24m", "")
.replace("\x1b[39;24m", "\x1b[39m")
.splitlines())
if last_node.lineno == last_node.end_lineno:
old = hg[last_node.lineno - 1]
start, end = find_indices(old, [last_node.col_offset, last_node.end_col_offset])
hg[last_node.lineno - 1] = old[:start] + "\x1b[4m" + old[start:end] + "\x1b[24m" + old[end:]
else:
old = hg[last_node.lineno - 1]
[start] = find_indices(old, [last_node.col_offset])
hg[last_node.lineno - 1] = old[:start] + "\x1b[4m" + old[start:]
for lineid in range(last_node.lineno, last_node.end_lineno - 1):
old = hg[lineid]
first_nonspace = len(old) - len(old.lstrip())
hg[lineid] = old[:first_nonspace] + "\x1b[4m" + old[first_nonspace:] + "\x1b[24m"
old = hg[last_node.end_lineno - 1]
first_nonspace = len(old) - len(old.lstrip())
[end] = find_indices(old, [last_node.end_col_offset])
hg[last_node.end_lineno - 1] = old[:first_nonspace] + "\x1b[4m" + old[first_nonspace:end] + "\x1b[24m" + old[end:]
CONTEXT_SIZE = 2
start = max(0, last_node.lineno - CONTEXT_SIZE - 1)
offset = start + 1
for i, line in enumerate(hg[start:last_node.end_lineno + CONTEXT_SIZE]):
erroneous = last_node.lineno <= offset + i <= last_node.end_lineno
indicator = cf.white(" →") if erroneous else " "
bar = " ▎"
# bar = "│" if erroneous else "┊"
disp = f"\x1b[24m{indicator}{cf.white}{(offset + i):>4}{cf.red if erroneous else cf.reset}{bar}{cf.reset} {line}\x1b[24m"
print(disp)
# print(repr(disp))
print()
if isinstance(exc_value, CompileError):
print(cf.red("Error:"), exc_value)
detail = inspect.cleandoc(exc_value.detail(last_node))
if detail:
print()
print(detail)
else:
print(cf.red("Internal Compiler Error:"), exc_value)
print()
print("Please report this error to the Typon maintainers.")
traceback.print_tb(orig_tb, limit=-1)
print()
def find_indices(s, indices: list[int]) -> list[int]:
"""
Matches indices to an ANSI-colored string.
:param s: An input string. This will usually be a line from a Python file that has been highlighted using Pygments.
:param indices: A list of indices to match. These will come from the `ast` parser and are UTF-8 *byte* offsets,
not *characters*!
:return: A list of *character* offsets that match the given indices, such text can be inserted at these indices and
end up in the expected place.
"""
results = set()
i = 0
j = 0
it = iter(sorted(list(set(indices))))
current = next(it)
while i <= len(s):
if i != len(s) and s[i] == "\x1b":
i += 1
while s[i] != "m":
i += 1
i += 1
continue
if j == current:
results.add(i)
try:
current = next(it)
except StopIteration:
break
j += len(s[i].encode("utf-8"))
i += 1
assert len(results) == len(indices), (results, indices, s)
return sorted(list(results))
assert find_indices("\x1b[48;5;237mmath.abcd\x1b[37m\x1b[39m\x1b[49m", [0, 9]) == [11, 35], find_indices("\x1b[48;5;237mmath.abcd\x1b[37m\x1b[39m\x1b[49m", [0, 9])
assert find_indices("abcdef", [2, 5]) == [2, 5]
assert find_indices("abc\x1b[32mdef", [2, 5]) == [2, 10], find_indices("abc\x1b[32mdef", [2, 5])
assert find_indices("math.abcd\x1b[37m\x1b[39m", [0, 9]) == [0, 19], find_indices("math.abcd\x1b[37m\x1b[39m", [0, 9])
assert find_indices(' \x1b[36mprint\x1b[39m(x, y, z)\x1b[37m\x1b[39m', [4, 18]) == [9, 38], find_indices(' \x1b[36mprint\x1b[39m(x, y, z)\x1b[37m\x1b[39m', [4, 18])
sys.excepthook = exception_hook
try:
pydevd = importlib.import_module("_pydevd_bundle.pydevd_breakpoints")
except ImportError:
pass
else:
pydevd._fallback_excepthook = sys.excepthook
pydevd.original_excepthook = sys.excepthook
typon_std = Path(__file__).parent.parent / "stdlib"
#discover_module(typon_std, PRELUDE.child(ScopeKind.GLOBAL))
parse_module("builtins", typon_std, PRELUDE)
def transpile(source, name: str, path: Path):
__TB__ = f"transpiling module {cf.white(name)}"
def preprocess(node):
IfMainVisitor().visit(node)
node = DesugarWith().visit(node)
node = DesugarCompare().visit(node)
node = DesugarOp().visit(node)
return node
module = parse_module(path.stem, path.parent, preprocess=preprocess)
def disp_scope(scope, indent=0):
debug(" " * indent, scope.kind)
for child in scope.children:
disp_scope(child, indent + 1)
for var in scope.vars.items():
debug(" " * (indent + 1), var)
def main_module():
yield from emit_module(module)
yield "#ifdef TYPON_EXTENSION"
# yield f"PYBIND11_MODULE({self.module_name}, m) {{"
# yield f"m.doc() = \"Typon extension module '{self.module_name}'\";"
# visitor = ModuleVisitorExt(self.scope)
# code = [line for stmt in node.body for line in visitor.visit(stmt)]
# yield from code
# yield "}"
yield "#else"
yield "typon::Root root() const {"
yield f"co_await dot(PROGRAMNS::{module.name()}, main)();"
yield "}"
yield "int main(int argc, char* argv[]) {"
yield "py_sys::all.argv = typon::PyList<PyStr>(std::vector<PyStr>(argv, argv + argc));"
yield f"root().call();"
yield "}"
yield "#endif"
code = "\n".join(filter(None, main_module()))
return code
exit()
assert isinstance(res, ast.Module)
res.name = "__main__"
code = "\n".join(filter(None, map(str, FileVisitor(Scope(), name).visit(res))))
return code
typon/trans/transpiler/error_display.py 0 → 100644
View file @ d6b9c3fe
# coding: utf-8
import ast
import builtins
import importlib
import sys
import traceback
import colorful as cf
def exception_hook(exc_type, exc_value, tb):
print = lambda *args, **kwargs: builtins.print(*args, **kwargs, file=sys.stderr)
last_node = None
last_file = None
orig_tb = tb
while tb:
local_vars = tb.tb_frame.f_locals
name = tb.tb_frame.f_code.co_name
if name in ("transpile", "parse_module"):
last_file = local_vars["path"]
if name == "visit" and (node := local_vars["node"]) and isinstance(node, ast.AST):
last_node = node
if node := local_vars.get("__TB_NODE__", None):
last_node = node
if local_vars.get("__TB_SKIP__", None) and tb.tb_next:
tb = tb.tb_next
continue
filename = tb.tb_frame.f_code.co_filename
line_no = tb.tb_lineno
print(cf.red(f"File {filename}:{line_no}, in {cf.green(name)}"), end="")
if info := local_vars.get("__TB__", None):
print(f": {cf.magenta(info)}\x1b[24m")
else:
print()
tb = tb.tb_next
if last_node is not None and last_file is not None:
print()
if not hasattr(last_node, "lineno"):
print(cf.red("Error: "), cf.white("No line number available"))
last_node.lineno = 1
print(ast.unparse(last_node))
return
print(f"In file {cf.white(last_file)}:{last_node.lineno}")
#print(f"From {last_node.lineno}:{last_node.col_offset} to {last_node.end_lineno}:{last_node.end_col_offset}")
with open(last_file, "r", encoding="utf-8") as f:
code = f.read()
hg = (str(highlight(code, True))
.replace("\x1b[04m", "")
.replace("\x1b[24m", "")
.replace("\x1b[39;24m", "\x1b[39m")
.splitlines())
if last_node.lineno == last_node.end_lineno:
old = hg[last_node.lineno - 1]
start, end = find_indices(old, [last_node.col_offset, last_node.end_col_offset])
hg[last_node.lineno - 1] = old[:start] + "\x1b[4m" + old[start:end] + "\x1b[24m" + old[end:]
else:
old = hg[last_node.lineno - 1]
[start] = find_indices(old, [last_node.col_offset])
hg[last_node.lineno - 1] = old[:start] + "\x1b[4m" + old[start:]
for lineid in range(last_node.lineno, last_node.end_lineno - 1):
old = hg[lineid]
first_nonspace = len(old) - len(old.lstrip())
hg[lineid] = old[:first_nonspace] + "\x1b[4m" + old[first_nonspace:] + "\x1b[24m"
old = hg[last_node.end_lineno - 1]
first_nonspace = len(old) - len(old.lstrip())
[end] = find_indices(old, [last_node.end_col_offset])
hg[last_node.end_lineno - 1] = old[:first_nonspace] + "\x1b[4m" + old[first_nonspace:end] + "\x1b[24m" + old[end:]
CONTEXT_SIZE = 2
start = max(0, last_node.lineno - CONTEXT_SIZE - 1)
offset = start + 1
for i, line in enumerate(hg[start:last_node.end_lineno + CONTEXT_SIZE]):
erroneous = last_node.lineno <= offset + i <= last_node.end_lineno
indicator = cf.white(" →") if erroneous else " "
bar = " ▎"
# bar = "│" if erroneous else "┊"
disp = f"\x1b[24m{indicator}{cf.white}{(offset + i):>4}{cf.red if erroneous else cf.reset}{bar}{cf.reset} {line}\x1b[24m"
print(disp)
# print(repr(disp))
print()
if isinstance(exc_value, CompileError):
print(cf.red("Error:"), exc_value)
detail = inspect.cleandoc(exc_value.detail(last_node))
if detail:
print()
print(detail)
else:
print(cf.red("Internal Compiler Error:"), exc_value)
print()
print("Please report this error to the Typon maintainers.")
traceback.print_tb(orig_tb, limit=-1)
print()
def find_indices(s, indices: list[int]) -> list[int]:
"""
Matches indices to an ANSI-colored string.
:param s: An input string. This will usually be a line from a Python file that has been highlighted using Pygments.
:param indices: A list of indices to match. These will come from the `ast` parser and are UTF-8 *byte* offsets,
not *characters*!
:return: A list of *character* offsets that match the given indices, such text can be inserted at these indices and
end up in the expected place.
"""
results = set()
i = 0
j = 0
it = iter(sorted(list(set(indices))))
current = next(it)
while i <= len(s):
if i != len(s) and s[i] == "\x1b":
i += 1
while s[i] != "m":
i += 1
i += 1
continue
if j == current:
results.add(i)
try:
current = next(it)
except StopIteration:
break
j += len(s[i].encode("utf-8"))
i += 1
assert len(results) == len(indices), (results, indices, s)
return sorted(list(results))
assert find_indices("\x1b[48;5;237mmath.abcd\x1b[37m\x1b[39m\x1b[49m", [0, 9]) == [11, 35], find_indices("\x1b[48;5;237mmath.abcd\x1b[37m\x1b[39m\x1b[49m", [0, 9])
assert find_indices("abcdef", [2, 5]) == [2, 5]
assert find_indices("abc\x1b[32mdef", [2, 5]) == [2, 10], find_indices("abc\x1b[32mdef", [2, 5])
assert find_indices("math.abcd\x1b[37m\x1b[39m", [0, 9]) == [0, 19], find_indices("math.abcd\x1b[37m\x1b[39m", [0, 9])
assert find_indices(' \x1b[36mprint\x1b[39m(x, y, z)\x1b[37m\x1b[39m', [4, 18]) == [9, 38], find_indices(' \x1b[36mprint\x1b[39m(x, y, z)\x1b[37m\x1b[39m', [4, 18])
def init():
sys.excepthook = exception_hook
try:
pydevd = importlib.import_module("_pydevd_bundle.pydevd_breakpoints")
except ImportError:
pass
else:
pydevd._fallback_excepthook = sys.excepthook
pydevd.original_excepthook = sys.excepthook
\ No newline at end of file
typon/trans/transpiler/phases/emit_cpp/function.py
View file @ d6b9c3fe
import ast
from dataclasses import dataclass
from typing import Iterable
from typing import Iterable, Optional
from transpiler import Scope
from transpiler.phases.typing.scope import Scope
from transpiler.phases.emit_cpp.visitors import NodeVisitor, flatmap
from transpiler.phases.typing.types import CallableInstanceType
from transpiler.phases.typing.types import CallableInstanceType, BaseType
def emit_function(name: str, func: CallableInstanceType) -> Iterable[str]:
yield f"struct : function {{"
yield "typon::Task<void> operator()() const {"
yield "typon::Task<void> operator()("
for arg, ty in zip(func.block_data.node.args.args, func.parameters):
yield "auto "
yield arg
yield ") const {"
yield "}"
yield f"}} static constexpr {name} {{}};"
yield f"static_assert(sizeof {name} == 1);"
@dataclass
class BlockVisitor(NodeVisitor):
if False:
@dataclass
class BlockVisitor(NodeVisitor):
scope: Scope
#generator: CoroutineMode = field(default=CoroutineMode.SYNC, kw_only=True)
......
typon/trans/transpiler/phases/emit_cpp/visitors.py
View file @ d6b9c3fe
......@@ -2,7 +2,6 @@ import ast
from itertools import chain
from typing import Iterable
from transpiler import highlight
import transpiler.phases.typing.types as types
from transpiler.phases.typing.exceptions import UnresolvedTypeVariableError
from transpiler.phases.typing.types import BaseType
......
typon/trans/transpiler/phases/typing/modules.py
View file @ d6b9c3fe
import ast
from pathlib import Path
from logging import debug
from transpiler.phases.typing import PRELUDE
from transpiler.phases.typing.scope import Scope, VarKind, VarDecl, ScopeKind
from transpiler.phases.typing.types import MemberDef, ResolvedConcreteType, UniqueTypeMixin
......@@ -39,7 +41,6 @@ def parse_module(mod_name: str, python_path: Path, scope=None, preprocess=None):
if mod := visited_modules.get(real_path.as_posix()):
return mod
from transpiler import PRELUDE
mod_scope = scope or PRELUDE.child(ScopeKind.GLOBAL)
if real_path.suffix == ".py":
......
typon/trans/transpiler/transpiler.py 0 → 100644
View file @ d6b9c3fe
# coding: utf-8
from pathlib import Path
import colorama
from logging import debug
import colorful as cf
from transpiler import error_display
from transpiler.phases.desugar_compare import DesugarCompare
from transpiler.phases.desugar_op import DesugarOp
from transpiler.phases.desugar_with import DesugarWith
from transpiler.phases.emit_cpp.module import emit_module
from transpiler.phases.if_main import IfMainVisitor
from transpiler.phases.typing import PRELUDE
from transpiler.phases.typing.modules import parse_module
def init():
error_display.init()
colorama.init()
typon_std = Path(__file__).parent.parent / "stdlib"
#discover_module(typon_std, PRELUDE.child(ScopeKind.GLOBAL))
parse_module("builtins", typon_std, PRELUDE)
def transpile(source, name: str, path: Path):
__TB__ = f"transpiling module {cf.white(name)}"
def preprocess(node):
IfMainVisitor().visit(node)
node = DesugarWith().visit(node)
node = DesugarCompare().visit(node)
node = DesugarOp().visit(node)
return node
module = parse_module(path.stem, path.parent, preprocess=preprocess)
def disp_scope(scope, indent=0):
debug(" " * indent, scope.kind)
for child in scope.children:
disp_scope(child, indent + 1)
for var in scope.vars.items():
debug(" " * (indent + 1), var)
def main_module():
yield from emit_module(module)
yield "#ifdef TYPON_EXTENSION"
# yield f"PYBIND11_MODULE({self.module_name}, m) {{"
# yield f"m.doc() = \"Typon extension module '{self.module_name}'\";"
# visitor = ModuleVisitorExt(self.scope)
# code = [line for stmt in node.body for line in visitor.visit(stmt)]
# yield from code
# yield "}"
yield "#else"
yield "typon::Root root() const {"
yield f"co_await dot(PROGRAMNS::{module.name()}, main)();"
yield "}"
yield "int main(int argc, char* argv[]) {"
yield "py_sys::all.argv = typon::PyList<PyStr>(std::vector<PyStr>(argv, argv + argc));"
yield f"root().call();"
yield "}"
yield "#endif"
code = "\n".join(filter(None, main_module()))
return code
exit()
assert isinstance(res, ast.Module)
res.name = "__main__"
code = "\n".join(filter(None, map(str, FileVisitor(Scope(), name).visit(res))))
return code
init()
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7