Skip to content

T
typon-compiler
Commit d6b9c3fe authored by Tom Niget's avatar Tom Niget
Browse files
Options

Move stuff around

parent 90409113
Show whitespace changes
Inline Side-by-side
Showing with 389 additions and 384 deletions
typon/trans/test_runner.py
View file @ d6b9c3fe
...@@ -14,12 +14,13 @@ import colorama ...@@ -14,12 +14,13 @@ import colorama
import colorful as cf import colorful as cf
import signal import signal
from transpiler import transpile
from transpiler.format import format_code from transpiler.format import format_code
# load .env file # load .env file
from dotenv import load_dotenv from dotenv import load_dotenv
from transpiler.transpiler import transpile
colorama.init() colorama.init()
load_dotenv() load_dotenv()
......
typon/trans/tests/user_generics.py
View file @ d6b9c3fe
...@@ -23,4 +23,4 @@ ...@@ -23,4 +23,4 @@
if __name__ == "__main__": if __name__ == "__main__":
x = 5 print(5)
\ No newline at end of file \ No newline at end of file
typon/trans/transpiler/__init__.py
View file @ d6b9c3fe
# coding: utf-8 # coding: utf-8
import ast
import builtins
import importlib
import inspect
import os
import traceback
from pathlib import Path
#os.environ["TERM"] = "xterm-256"
import colorama
from transpiler.phases.desugar_compare import DesugarCompare
from transpiler.phases.desugar_op import DesugarOp
from transpiler.phases.emit_cpp.module import emit_module
from transpiler.phases.typing import PRELUDE
from transpiler.phases.typing.modules import parse_module
from transpiler.phases.typing.stdlib import StdlibVisitor
colorama.init()
from transpiler.exceptions import CompileError
from transpiler.phases.desugar_with import DesugarWith
#from transpiler.phases.emit_cpp.file import FileVisitor
from transpiler.phases.if_main import IfMainVisitor
#from transpiler.phases.typing.block import ScoperBlockVisitor
from transpiler.phases.typing.scope import Scope, ScopeKind
from transpiler.utils import highlight
import sys
import colorful as cf
from logging import debug
def exception_hook(exc_type, exc_value, tb):
print = lambda *args, **kwargs: builtins.print(*args, **kwargs, file=sys.stderr)
last_node = None
last_file = None
orig_tb = tb
while tb:
local_vars = tb.tb_frame.f_locals
name = tb.tb_frame.f_code.co_name
if name in ("transpile", "parse_module"):
last_file = local_vars["path"]
if name == "visit" and (node := local_vars["node"]) and isinstance(node, ast.AST):
last_node = node
if node := local_vars.get("__TB_NODE__", None):
last_node = node
if local_vars.get("__TB_SKIP__", None) and tb.tb_next:
tb = tb.tb_next
continue
filename = tb.tb_frame.f_code.co_filename
line_no = tb.tb_lineno
print(cf.red(f"File {filename}:{line_no}, in {cf.green(name)}"), end="")
if info := local_vars.get("__TB__", None):
print(f": {cf.magenta(info)}\x1b[24m")
else:
print()
tb = tb.tb_next
if last_node is not None and last_file is not None:
print()
if not hasattr(last_node, "lineno"):
print(cf.red("Error: "), cf.white("No line number available"))
last_node.lineno = 1
print(ast.unparse(last_node))
return
print(f"In file {cf.white(last_file)}:{last_node.lineno}")
#print(f"From {last_node.lineno}:{last_node.col_offset} to {last_node.end_lineno}:{last_node.end_col_offset}")
with open(last_file, "r", encoding="utf-8") as f:
code = f.read()
hg = (str(highlight(code, True))
.replace("\x1b[04m", "")
.replace("\x1b[24m", "")
.replace("\x1b[39;24m", "\x1b[39m")
.splitlines())
if last_node.lineno == last_node.end_lineno:
old = hg[last_node.lineno - 1]
start, end = find_indices(old, [last_node.col_offset, last_node.end_col_offset])
hg[last_node.lineno - 1] = old[:start] + "\x1b[4m" + old[start:end] + "\x1b[24m" + old[end:]
else:
old = hg[last_node.lineno - 1]
[start] = find_indices(old, [last_node.col_offset])
hg[last_node.lineno - 1] = old[:start] + "\x1b[4m" + old[start:]
for lineid in range(last_node.lineno, last_node.end_lineno - 1):
old = hg[lineid]
first_nonspace = len(old) - len(old.lstrip())
hg[lineid] = old[:first_nonspace] + "\x1b[4m" + old[first_nonspace:] + "\x1b[24m"
old = hg[last_node.end_lineno - 1]
first_nonspace = len(old) - len(old.lstrip())
[end] = find_indices(old, [last_node.end_col_offset])
hg[last_node.end_lineno - 1] = old[:first_nonspace] + "\x1b[4m" + old[first_nonspace:end] + "\x1b[24m" + old[end:]
CONTEXT_SIZE = 2
start = max(0, last_node.lineno - CONTEXT_SIZE - 1)
offset = start + 1
for i, line in enumerate(hg[start:last_node.end_lineno + CONTEXT_SIZE]):
erroneous = last_node.lineno <= offset + i <= last_node.end_lineno
indicator = cf.white(" →") if erroneous else " "
bar = " ▎"
# bar = "│" if erroneous else "┊"
disp = f"\x1b[24m{indicator}{cf.white}{(offset + i):>4}{cf.red if erroneous else cf.reset}{bar}{cf.reset} {line}\x1b[24m"
print(disp)
# print(repr(disp))
print()
if isinstance(exc_value, CompileError):
print(cf.red("Error:"), exc_value)
detail = inspect.cleandoc(exc_value.detail(last_node))
if detail:
print()
print(detail)
else:
print(cf.red("Internal Compiler Error:"), exc_value)
print()
print("Please report this error to the Typon maintainers.")
traceback.print_tb(orig_tb, limit=-1)
print()
def find_indices(s, indices: list[int]) -> list[int]:
"""
Matches indices to an ANSI-colored string.
:param s: An input string. This will usually be a line from a Python file that has been highlighted using Pygments.
:param indices: A list of indices to match. These will come from the `ast` parser and are UTF-8 *byte* offsets,
not *characters*!
:return: A list of *character* offsets that match the given indices, such text can be inserted at these indices and
end up in the expected place.
"""
results = set()
i = 0
j = 0
it = iter(sorted(list(set(indices))))
current = next(it)
while i <= len(s):
if i != len(s) and s[i] == "\x1b":
i += 1
while s[i] != "m":
i += 1
i += 1
continue
if j == current:
results.add(i)
try:
current = next(it)
except StopIteration:
break
j += len(s[i].encode("utf-8"))
i += 1
assert len(results) == len(indices), (results, indices, s)
return sorted(list(results))
assert find_indices("\x1b[48;5;237mmath.abcd\x1b[37m\x1b[39m\x1b[49m", [0, 9]) == [11, 35], find_indices("\x1b[48;5;237mmath.abcd\x1b[37m\x1b[39m\x1b[49m", [0, 9])
assert find_indices("abcdef", [2, 5]) == [2, 5]
assert find_indices("abc\x1b[32mdef", [2, 5]) == [2, 10], find_indices("abc\x1b[32mdef", [2, 5])
assert find_indices("math.abcd\x1b[37m\x1b[39m", [0, 9]) == [0, 19], find_indices("math.abcd\x1b[37m\x1b[39m", [0, 9])
assert find_indices(' \x1b[36mprint\x1b[39m(x, y, z)\x1b[37m\x1b[39m', [4, 18]) == [9, 38], find_indices(' \x1b[36mprint\x1b[39m(x, y, z)\x1b[37m\x1b[39m', [4, 18])
sys.excepthook = exception_hook
try:
pydevd = importlib.import_module("_pydevd_bundle.pydevd_breakpoints")
except ImportError:
pass
else:
pydevd._fallback_excepthook = sys.excepthook
pydevd.original_excepthook = sys.excepthook
typon_std = Path(__file__).parent.parent / "stdlib"
#discover_module(typon_std, PRELUDE.child(ScopeKind.GLOBAL))
parse_module("builtins", typon_std, PRELUDE)
def transpile(source, name: str, path: Path):
__TB__ = f"transpiling module {cf.white(name)}"
def preprocess(node):
IfMainVisitor().visit(node)
node = DesugarWith().visit(node)
node = DesugarCompare().visit(node)
node = DesugarOp().visit(node)
return node
module = parse_module(path.stem, path.parent, preprocess=preprocess)
def disp_scope(scope, indent=0):
debug(" " * indent, scope.kind)
for child in scope.children:
disp_scope(child, indent + 1)
for var in scope.vars.items():
debug(" " * (indent + 1), var)
def main_module():
yield from emit_module(module)
yield "#ifdef TYPON_EXTENSION"
# yield f"PYBIND11_MODULE({self.module_name}, m) {{"
# yield f"m.doc() = \"Typon extension module '{self.module_name}'\";"
# visitor = ModuleVisitorExt(self.scope)
# code = [line for stmt in node.body for line in visitor.visit(stmt)]
# yield from code
# yield "}"
yield "#else"
yield "typon::Root root() const {"
yield f"co_await dot(PROGRAMNS::{module.name()}, main)();"
yield "}"
yield "int main(int argc, char* argv[]) {"
yield "py_sys::all.argv = typon::PyList<PyStr>(std::vector<PyStr>(argv, argv + argc));"
yield f"root().call();"
yield "}"
yield "#endif"
code = "\n".join(filter(None, main_module()))
return code
exit()
assert isinstance(res, ast.Module)
res.name = "__main__"
code = "\n".join(filter(None, map(str, FileVisitor(Scope(), name).visit(res))))
return code
typon/trans/transpiler/error_display.py 0 → 100644
View file @ d6b9c3fe
# coding: utf-8
import ast
import builtins
import importlib
import sys
import traceback
import colorful as cf
def exception_hook(exc_type, exc_value, tb):
print = lambda *args, **kwargs: builtins.print(*args, **kwargs, file=sys.stderr)
last_node = None
last_file = None
orig_tb = tb
while tb:
local_vars = tb.tb_frame.f_locals
name = tb.tb_frame.f_code.co_name
if name in ("transpile", "parse_module"):
last_file = local_vars["path"]
if name == "visit" and (node := local_vars["node"]) and isinstance(node, ast.AST):
last_node = node
if node := local_vars.get("__TB_NODE__", None):
last_node = node
if local_vars.get("__TB_SKIP__", None) and tb.tb_next:
tb = tb.tb_next
continue
filename = tb.tb_frame.f_code.co_filename
line_no = tb.tb_lineno
print(cf.red(f"File {filename}:{line_no}, in {cf.green(name)}"), end="")
if info := local_vars.get("__TB__", None):
print(f": {cf.magenta(info)}\x1b[24m")
else:
print()
tb = tb.tb_next
if last_node is not None and last_file is not None:
print()
if not hasattr(last_node, "lineno"):
print(cf.red("Error: "), cf.white("No line number available"))
last_node.lineno = 1
print(ast.unparse(last_node))
return
print(f"In file {cf.white(last_file)}:{last_node.lineno}")
#print(f"From {last_node.lineno}:{last_node.col_offset} to {last_node.end_lineno}:{last_node.end_col_offset}")
with open(last_file, "r", encoding="utf-8") as f:
code = f.read()
hg = (str(highlight(code, True))
.replace("\x1b[04m", "")
.replace("\x1b[24m", "")
.replace("\x1b[39;24m", "\x1b[39m")
.splitlines())
if last_node.lineno == last_node.end_lineno:
old = hg[last_node.lineno - 1]
start, end = find_indices(old, [last_node.col_offset, last_node.end_col_offset])
hg[last_node.lineno - 1] = old[:start] + "\x1b[4m" + old[start:end] + "\x1b[24m" + old[end:]
else:
old = hg[last_node.lineno - 1]
[start] = find_indices(old, [last_node.col_offset])
hg[last_node.lineno - 1] = old[:start] + "\x1b[4m" + old[start:]
for lineid in range(last_node.lineno, last_node.end_lineno - 1):
old = hg[lineid]
first_nonspace = len(old) - len(old.lstrip())
hg[lineid] = old[:first_nonspace] + "\x1b[4m" + old[first_nonspace:] + "\x1b[24m"
old = hg[last_node.end_lineno - 1]
first_nonspace = len(old) - len(old.lstrip())
[end] = find_indices(old, [last_node.end_col_offset])
hg[last_node.end_lineno - 1] = old[:first_nonspace] + "\x1b[4m" + old[first_nonspace:end] + "\x1b[24m" + old[end:]
CONTEXT_SIZE = 2
start = max(0, last_node.lineno - CONTEXT_SIZE - 1)
offset = start + 1
for i, line in enumerate(hg[start:last_node.end_lineno + CONTEXT_SIZE]):
erroneous = last_node.lineno <= offset + i <= last_node.end_lineno
indicator = cf.white(" →") if erroneous else " "
bar = " ▎"
# bar = "│" if erroneous else "┊"
disp = f"\x1b[24m{indicator}{cf.white}{(offset + i):>4}{cf.red if erroneous else cf.reset}{bar}{cf.reset} {line}\x1b[24m"
print(disp)
# print(repr(disp))
print()
if isinstance(exc_value, CompileError):
print(cf.red("Error:"), exc_value)
detail = inspect.cleandoc(exc_value.detail(last_node))
if detail:
print()
print(detail)
else:
print(cf.red("Internal Compiler Error:"), exc_value)
print()
print("Please report this error to the Typon maintainers.")
traceback.print_tb(orig_tb, limit=-1)
print()
def find_indices(s, indices: list[int]) -> list[int]:
"""
Matches indices to an ANSI-colored string.
:param s: An input string. This will usually be a line from a Python file that has been highlighted using Pygments.
:param indices: A list of indices to match. These will come from the `ast` parser and are UTF-8 *byte* offsets,
not *characters*!
:return: A list of *character* offsets that match the given indices, such text can be inserted at these indices and
end up in the expected place.
"""
results = set()
i = 0
j = 0
it = iter(sorted(list(set(indices))))
current = next(it)
while i <= len(s):
if i != len(s) and s[i] == "\x1b":
i += 1
while s[i] != "m":
i += 1
i += 1
continue
if j == current:
results.add(i)
try:
current = next(it)
except StopIteration:
break
j += len(s[i].encode("utf-8"))
i += 1
assert len(results) == len(indices), (results, indices, s)
return sorted(list(results))
assert find_indices("\x1b[48;5;237mmath.abcd\x1b[37m\x1b[39m\x1b[49m", [0, 9]) == [11, 35], find_indices("\x1b[48;5;237mmath.abcd\x1b[37m\x1b[39m\x1b[49m", [0, 9])
assert find_indices("abcdef", [2, 5]) == [2, 5]
assert find_indices("abc\x1b[32mdef", [2, 5]) == [2, 10], find_indices("abc\x1b[32mdef", [2, 5])
assert find_indices("math.abcd\x1b[37m\x1b[39m", [0, 9]) == [0, 19], find_indices("math.abcd\x1b[37m\x1b[39m", [0, 9])
assert find_indices(' \x1b[36mprint\x1b[39m(x, y, z)\x1b[37m\x1b[39m', [4, 18]) == [9, 38], find_indices(' \x1b[36mprint\x1b[39m(x, y, z)\x1b[37m\x1b[39m', [4, 18])
def init():
sys.excepthook = exception_hook
try:
pydevd = importlib.import_module("_pydevd_bundle.pydevd_breakpoints")
except ImportError:
pass
else:
pydevd._fallback_excepthook = sys.excepthook
pydevd.original_excepthook = sys.excepthook
\ No newline at end of file
typon/trans/transpiler/phases/emit_cpp/function.py
View file @ d6b9c3fe
import ast import ast
from dataclasses import dataclass from dataclasses import dataclass
from typing import Iterable from typing import Iterable, Optional
from transpiler import Scope
from transpiler.phases.typing.scope import Scope
from transpiler.phases.emit_cpp.visitors import NodeVisitor, flatmap from transpiler.phases.emit_cpp.visitors import NodeVisitor, flatmap
from transpiler.phases.typing.types import CallableInstanceType from transpiler.phases.typing.types import CallableInstanceType, BaseType
def emit_function(name: str, func: CallableInstanceType) -> Iterable[str]: def emit_function(name: str, func: CallableInstanceType) -> Iterable[str]:
yield f"struct : function {{" yield f"struct : function {{"
yield "typon::Task<void> operator()() const {" yield "typon::Task<void> operator()("
for arg, ty in zip(func.block_data.node.args.args, func.parameters):
yield "auto "
yield arg
yield ") const {"
yield "}" yield "}"
yield f"}} static constexpr {name} {{}};" yield f"}} static constexpr {name} {{}};"
yield f"static_assert(sizeof {name} == 1);" yield f"static_assert(sizeof {name} == 1);"
@dataclass
class BlockVisitor(NodeVisitor):
if False:
@dataclass
class BlockVisitor(NodeVisitor):
scope: Scope scope: Scope
#generator: CoroutineMode = field(default=CoroutineMode.SYNC, kw_only=True) #generator: CoroutineMode = field(default=CoroutineMode.SYNC, kw_only=True)
......
typon/trans/transpiler/phases/emit_cpp/visitors.py
View file @ d6b9c3fe
...@@ -2,7 +2,6 @@ import ast ...@@ -2,7 +2,6 @@ import ast
from itertools import chain from itertools import chain
from typing import Iterable from typing import Iterable
from transpiler import highlight
import transpiler.phases.typing.types as types import transpiler.phases.typing.types as types
from transpiler.phases.typing.exceptions import UnresolvedTypeVariableError from transpiler.phases.typing.exceptions import UnresolvedTypeVariableError
from transpiler.phases.typing.types import BaseType from transpiler.phases.typing.types import BaseType
......
typon/trans/transpiler/phases/typing/modules.py
View file @ d6b9c3fe
import ast import ast
from pathlib import Path from pathlib import Path
from logging import debug from logging import debug
from transpiler.phases.typing import PRELUDE
from transpiler.phases.typing.scope import Scope, VarKind, VarDecl, ScopeKind from transpiler.phases.typing.scope import Scope, VarKind, VarDecl, ScopeKind
from transpiler.phases.typing.types import MemberDef, ResolvedConcreteType, UniqueTypeMixin from transpiler.phases.typing.types import MemberDef, ResolvedConcreteType, UniqueTypeMixin
...@@ -39,7 +41,6 @@ def parse_module(mod_name: str, python_path: Path, scope=None, preprocess=None): ...@@ -39,7 +41,6 @@ def parse_module(mod_name: str, python_path: Path, scope=None, preprocess=None):
if mod := visited_modules.get(real_path.as_posix()): if mod := visited_modules.get(real_path.as_posix()):
return mod return mod
from transpiler import PRELUDE
mod_scope = scope or PRELUDE.child(ScopeKind.GLOBAL) mod_scope = scope or PRELUDE.child(ScopeKind.GLOBAL)
if real_path.suffix == ".py": if real_path.suffix == ".py":
......
typon/trans/transpiler/transpiler.py 0 → 100644
View file @ d6b9c3fe
# coding: utf-8
from pathlib import Path
import colorama
from logging import debug
import colorful as cf
from transpiler import error_display
from transpiler.phases.desugar_compare import DesugarCompare
from transpiler.phases.desugar_op import DesugarOp
from transpiler.phases.desugar_with import DesugarWith
from transpiler.phases.emit_cpp.module import emit_module
from transpiler.phases.if_main import IfMainVisitor
from transpiler.phases.typing import PRELUDE
from transpiler.phases.typing.modules import parse_module
def init():
error_display.init()
colorama.init()
typon_std = Path(__file__).parent.parent / "stdlib"
#discover_module(typon_std, PRELUDE.child(ScopeKind.GLOBAL))
parse_module("builtins", typon_std, PRELUDE)
def transpile(source, name: str, path: Path):
__TB__ = f"transpiling module {cf.white(name)}"
def preprocess(node):
IfMainVisitor().visit(node)
node = DesugarWith().visit(node)
node = DesugarCompare().visit(node)
node = DesugarOp().visit(node)
return node
module = parse_module(path.stem, path.parent, preprocess=preprocess)
def disp_scope(scope, indent=0):
debug(" " * indent, scope.kind)
for child in scope.children:
disp_scope(child, indent + 1)
for var in scope.vars.items():
debug(" " * (indent + 1), var)
def main_module():
yield from emit_module(module)
yield "#ifdef TYPON_EXTENSION"
# yield f"PYBIND11_MODULE({self.module_name}, m) {{"
# yield f"m.doc() = \"Typon extension module '{self.module_name}'\";"
# visitor = ModuleVisitorExt(self.scope)
# code = [line for stmt in node.body for line in visitor.visit(stmt)]
# yield from code
# yield "}"
yield "#else"
yield "typon::Root root() const {"
yield f"co_await dot(PROGRAMNS::{module.name()}, main)();"
yield "}"
yield "int main(int argc, char* argv[]) {"
yield "py_sys::all.argv = typon::PyList<PyStr>(std::vector<PyStr>(argv, argv + argc));"
yield f"root().call();"
yield "}"
yield "#endif"
code = "\n".join(filter(None, main_module()))
return code
exit()
assert isinstance(res, ast.Module)
res.name = "__main__"
code = "\n".join(filter(None, map(str, FileVisitor(Scope(), name).visit(res))))
return code
init()
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7