Skip to content

P
Pyston
Commit 3fa45d83 authored by Kevin Modzelewski's avatar Kevin Modzelewski Committed by Kevin Modzelewski
Browse files
Options

Import a bunch of parser stuff from CPython

parent c8c31cb1
Hide whitespace changes
Inline Side-by-side
Showing with 21436 additions and 0 deletions
from_cpython/CMakeLists.txt
View file @ 3fa45d83
......@@ -94,6 +94,8 @@ file(GLOB_RECURSE STDOBJECT_SRCS Objects
# compile specified files in from_cpython/Python
file(GLOB_RECURSE STDPYTHON_SRCS Python
asdl.c
ast.c
dtoa.c
formatter_string.c
formatter_unicode.c
......@@ -102,6 +104,7 @@ file(GLOB_RECURSE STDPYTHON_SRCS Python
mystrtoul.c
pyctype.c
pystrtod.c
Python-ast.c
random.c
structmember.c
)
......
from_cpython/Include/Python-ast.h 0 → 100644
View file @ 3fa45d83
/* File automatically generated by Parser/asdl_c.py. */
#include "asdl.h"
typedef struct _mod *mod_ty;
typedef struct _stmt *stmt_ty;
typedef struct _expr *expr_ty;
typedef enum _expr_context { Load=1, Store=2, Del=3, AugLoad=4, AugStore=5,
Param=6 } expr_context_ty;
typedef struct _slice *slice_ty;
typedef enum _boolop { And=1, Or=2 } boolop_ty;
typedef enum _operator { Add=1, Sub=2, Mult=3, Div=4, Mod=5, Pow=6, LShift=7,
RShift=8, BitOr=9, BitXor=10, BitAnd=11, FloorDiv=12 }
operator_ty;
typedef enum _unaryop { Invert=1, Not=2, UAdd=3, USub=4 } unaryop_ty;
typedef enum _cmpop { Eq=1, NotEq=2, Lt=3, LtE=4, Gt=5, GtE=6, Is=7, IsNot=8,
In=9, NotIn=10 } cmpop_ty;
typedef struct _comprehension *comprehension_ty;
typedef struct _excepthandler *excepthandler_ty;
typedef struct _arguments *arguments_ty;
typedef struct _keyword *keyword_ty;
typedef struct _alias *alias_ty;
enum _mod_kind {Module_kind=1, Interactive_kind=2, Expression_kind=3,
Suite_kind=4};
struct _mod {
enum _mod_kind kind;
union {
struct {
asdl_seq *body;
} Module;
struct {
asdl_seq *body;
} Interactive;
struct {
expr_ty body;
} Expression;
struct {
asdl_seq *body;
} Suite;
} v;
};
enum _stmt_kind {FunctionDef_kind=1, ClassDef_kind=2, Return_kind=3,
Delete_kind=4, Assign_kind=5, AugAssign_kind=6, Print_kind=7,
For_kind=8, While_kind=9, If_kind=10, With_kind=11,
Raise_kind=12, TryExcept_kind=13, TryFinally_kind=14,
Assert_kind=15, Import_kind=16, ImportFrom_kind=17,
Exec_kind=18, Global_kind=19, Expr_kind=20, Pass_kind=21,
Break_kind=22, Continue_kind=23};
struct _stmt {
enum _stmt_kind kind;
union {
struct {
identifier name;
arguments_ty args;
asdl_seq *body;
asdl_seq *decorator_list;
} FunctionDef;
struct {
identifier name;
asdl_seq *bases;
asdl_seq *body;
asdl_seq *decorator_list;
} ClassDef;
struct {
expr_ty value;
} Return;
struct {
asdl_seq *targets;
} Delete;
struct {
asdl_seq *targets;
expr_ty value;
} Assign;
struct {
expr_ty target;
operator_ty op;
expr_ty value;
} AugAssign;
struct {
expr_ty dest;
asdl_seq *values;
bool nl;
} Print;
struct {
expr_ty target;
expr_ty iter;
asdl_seq *body;
asdl_seq *orelse;
} For;
struct {
expr_ty test;
asdl_seq *body;
asdl_seq *orelse;
} While;
struct {
expr_ty test;
asdl_seq *body;
asdl_seq *orelse;
} If;
struct {
expr_ty context_expr;
expr_ty optional_vars;
asdl_seq *body;
} With;
struct {
expr_ty type;
expr_ty inst;
expr_ty tback;
} Raise;
struct {
asdl_seq *body;
asdl_seq *handlers;
asdl_seq *orelse;
} TryExcept;
struct {
asdl_seq *body;
asdl_seq *finalbody;
} TryFinally;
struct {
expr_ty test;
expr_ty msg;
} Assert;
struct {
asdl_seq *names;
} Import;
struct {
identifier module;
asdl_seq *names;
int level;
} ImportFrom;
struct {
expr_ty body;
expr_ty globals;
expr_ty locals;
} Exec;
struct {
asdl_seq *names;
} Global;
struct {
expr_ty value;
} Expr;
} v;
int lineno;
int col_offset;
};
enum _expr_kind {BoolOp_kind=1, BinOp_kind=2, UnaryOp_kind=3, Lambda_kind=4,
IfExp_kind=5, Dict_kind=6, Set_kind=7, ListComp_kind=8,
SetComp_kind=9, DictComp_kind=10, GeneratorExp_kind=11,
Yield_kind=12, Compare_kind=13, Call_kind=14, Repr_kind=15,
Num_kind=16, Str_kind=17, Attribute_kind=18,
Subscript_kind=19, Name_kind=20, List_kind=21, Tuple_kind=22};
struct _expr {
enum _expr_kind kind;
union {
struct {
boolop_ty op;
asdl_seq *values;
} BoolOp;
struct {
expr_ty left;
operator_ty op;
expr_ty right;
} BinOp;
struct {
unaryop_ty op;
expr_ty operand;
} UnaryOp;
struct {
arguments_ty args;
expr_ty body;
} Lambda;
struct {
expr_ty test;
expr_ty body;
expr_ty orelse;
} IfExp;
struct {
asdl_seq *keys;
asdl_seq *values;
} Dict;
struct {
asdl_seq *elts;
} Set;
struct {
expr_ty elt;
asdl_seq *generators;
} ListComp;
struct {
expr_ty elt;
asdl_seq *generators;
} SetComp;
struct {
expr_ty key;
expr_ty value;
asdl_seq *generators;
} DictComp;
struct {
expr_ty elt;
asdl_seq *generators;
} GeneratorExp;
struct {
expr_ty value;
} Yield;
struct {
expr_ty left;
asdl_int_seq *ops;
asdl_seq *comparators;
} Compare;
struct {
expr_ty func;
asdl_seq *args;
asdl_seq *keywords;
expr_ty starargs;
expr_ty kwargs;
} Call;
struct {
expr_ty value;
} Repr;
struct {
object n;
} Num;
struct {
string s;
} Str;
struct {
expr_ty value;
identifier attr;
expr_context_ty ctx;
} Attribute;
struct {
expr_ty value;
slice_ty slice;
expr_context_ty ctx;
} Subscript;
struct {
identifier id;
expr_context_ty ctx;
} Name;
struct {
asdl_seq *elts;
expr_context_ty ctx;
} List;
struct {
asdl_seq *elts;
expr_context_ty ctx;
} Tuple;
} v;
int lineno;
int col_offset;
};
enum _slice_kind {Ellipsis_kind=1, Slice_kind=2, ExtSlice_kind=3, Index_kind=4};
struct _slice {
enum _slice_kind kind;
union {
struct {
expr_ty lower;
expr_ty upper;
expr_ty step;
} Slice;
struct {
asdl_seq *dims;
} ExtSlice;
struct {
expr_ty value;
} Index;
} v;
};
struct _comprehension {
expr_ty target;
expr_ty iter;
asdl_seq *ifs;
};
enum _excepthandler_kind {ExceptHandler_kind=1};
struct _excepthandler {
enum _excepthandler_kind kind;
union {
struct {
expr_ty type;
expr_ty name;
asdl_seq *body;
} ExceptHandler;
} v;
int lineno;
int col_offset;
};
struct _arguments {
asdl_seq *args;
identifier vararg;
identifier kwarg;
asdl_seq *defaults;
};
struct _keyword {
identifier arg;
expr_ty value;
};
struct _alias {
identifier name;
identifier asname;
};
#define Module(a0, a1) _Py_Module(a0, a1)
mod_ty _Py_Module(asdl_seq * body, PyArena *arena);
#define Interactive(a0, a1) _Py_Interactive(a0, a1)
mod_ty _Py_Interactive(asdl_seq * body, PyArena *arena);
#define Expression(a0, a1) _Py_Expression(a0, a1)
mod_ty _Py_Expression(expr_ty body, PyArena *arena);
#define Suite(a0, a1) _Py_Suite(a0, a1)
mod_ty _Py_Suite(asdl_seq * body, PyArena *arena);
#define FunctionDef(a0, a1, a2, a3, a4, a5, a6) _Py_FunctionDef(a0, a1, a2, a3, a4, a5, a6)
stmt_ty _Py_FunctionDef(identifier name, arguments_ty args, asdl_seq * body,
asdl_seq * decorator_list, int lineno, int col_offset,
PyArena *arena);
#define ClassDef(a0, a1, a2, a3, a4, a5, a6) _Py_ClassDef(a0, a1, a2, a3, a4, a5, a6)
stmt_ty _Py_ClassDef(identifier name, asdl_seq * bases, asdl_seq * body,
asdl_seq * decorator_list, int lineno, int col_offset,
PyArena *arena);
#define Return(a0, a1, a2, a3) _Py_Return(a0, a1, a2, a3)
stmt_ty _Py_Return(expr_ty value, int lineno, int col_offset, PyArena *arena);
#define Delete(a0, a1, a2, a3) _Py_Delete(a0, a1, a2, a3)
stmt_ty _Py_Delete(asdl_seq * targets, int lineno, int col_offset, PyArena
*arena);
#define Assign(a0, a1, a2, a3, a4) _Py_Assign(a0, a1, a2, a3, a4)
stmt_ty _Py_Assign(asdl_seq * targets, expr_ty value, int lineno, int
col_offset, PyArena *arena);
#define AugAssign(a0, a1, a2, a3, a4, a5) _Py_AugAssign(a0, a1, a2, a3, a4, a5)
stmt_ty _Py_AugAssign(expr_ty target, operator_ty op, expr_ty value, int
lineno, int col_offset, PyArena *arena);
#define Print(a0, a1, a2, a3, a4, a5) _Py_Print(a0, a1, a2, a3, a4, a5)
stmt_ty _Py_Print(expr_ty dest, asdl_seq * values, bool nl, int lineno, int
col_offset, PyArena *arena);
#define For(a0, a1, a2, a3, a4, a5, a6) _Py_For(a0, a1, a2, a3, a4, a5, a6)
stmt_ty _Py_For(expr_ty target, expr_ty iter, asdl_seq * body, asdl_seq *
orelse, int lineno, int col_offset, PyArena *arena);
#define While(a0, a1, a2, a3, a4, a5) _Py_While(a0, a1, a2, a3, a4, a5)
stmt_ty _Py_While(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno,
int col_offset, PyArena *arena);
#define If(a0, a1, a2, a3, a4, a5) _Py_If(a0, a1, a2, a3, a4, a5)
stmt_ty _Py_If(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno,
int col_offset, PyArena *arena);
#define With(a0, a1, a2, a3, a4, a5) _Py_With(a0, a1, a2, a3, a4, a5)
stmt_ty _Py_With(expr_ty context_expr, expr_ty optional_vars, asdl_seq * body,
int lineno, int col_offset, PyArena *arena);
#define Raise(a0, a1, a2, a3, a4, a5) _Py_Raise(a0, a1, a2, a3, a4, a5)
stmt_ty _Py_Raise(expr_ty type, expr_ty inst, expr_ty tback, int lineno, int
col_offset, PyArena *arena);
#define TryExcept(a0, a1, a2, a3, a4, a5) _Py_TryExcept(a0, a1, a2, a3, a4, a5)
stmt_ty _Py_TryExcept(asdl_seq * body, asdl_seq * handlers, asdl_seq * orelse,
int lineno, int col_offset, PyArena *arena);
#define TryFinally(a0, a1, a2, a3, a4) _Py_TryFinally(a0, a1, a2, a3, a4)
stmt_ty _Py_TryFinally(asdl_seq * body, asdl_seq * finalbody, int lineno, int
col_offset, PyArena *arena);
#define Assert(a0, a1, a2, a3, a4) _Py_Assert(a0, a1, a2, a3, a4)
stmt_ty _Py_Assert(expr_ty test, expr_ty msg, int lineno, int col_offset,
PyArena *arena);
#define Import(a0, a1, a2, a3) _Py_Import(a0, a1, a2, a3)
stmt_ty _Py_Import(asdl_seq * names, int lineno, int col_offset, PyArena
*arena);
#define ImportFrom(a0, a1, a2, a3, a4, a5) _Py_ImportFrom(a0, a1, a2, a3, a4, a5)
stmt_ty _Py_ImportFrom(identifier module, asdl_seq * names, int level, int
lineno, int col_offset, PyArena *arena);
#define Exec(a0, a1, a2, a3, a4, a5) _Py_Exec(a0, a1, a2, a3, a4, a5)
stmt_ty _Py_Exec(expr_ty body, expr_ty globals, expr_ty locals, int lineno, int
col_offset, PyArena *arena);
#define Global(a0, a1, a2, a3) _Py_Global(a0, a1, a2, a3)
stmt_ty _Py_Global(asdl_seq * names, int lineno, int col_offset, PyArena
*arena);
#define Expr(a0, a1, a2, a3) _Py_Expr(a0, a1, a2, a3)
stmt_ty _Py_Expr(expr_ty value, int lineno, int col_offset, PyArena *arena);
#define Pass(a0, a1, a2) _Py_Pass(a0, a1, a2)
stmt_ty _Py_Pass(int lineno, int col_offset, PyArena *arena);
#define Break(a0, a1, a2) _Py_Break(a0, a1, a2)
stmt_ty _Py_Break(int lineno, int col_offset, PyArena *arena);
#define Continue(a0, a1, a2) _Py_Continue(a0, a1, a2)
stmt_ty _Py_Continue(int lineno, int col_offset, PyArena *arena);
#define BoolOp(a0, a1, a2, a3, a4) _Py_BoolOp(a0, a1, a2, a3, a4)
expr_ty _Py_BoolOp(boolop_ty op, asdl_seq * values, int lineno, int col_offset,
PyArena *arena);
#define BinOp(a0, a1, a2, a3, a4, a5) _Py_BinOp(a0, a1, a2, a3, a4, a5)
expr_ty _Py_BinOp(expr_ty left, operator_ty op, expr_ty right, int lineno, int
col_offset, PyArena *arena);
#define UnaryOp(a0, a1, a2, a3, a4) _Py_UnaryOp(a0, a1, a2, a3, a4)
expr_ty _Py_UnaryOp(unaryop_ty op, expr_ty operand, int lineno, int col_offset,
PyArena *arena);
#define Lambda(a0, a1, a2, a3, a4) _Py_Lambda(a0, a1, a2, a3, a4)
expr_ty _Py_Lambda(arguments_ty args, expr_ty body, int lineno, int col_offset,
PyArena *arena);
#define IfExp(a0, a1, a2, a3, a4, a5) _Py_IfExp(a0, a1, a2, a3, a4, a5)
expr_ty _Py_IfExp(expr_ty test, expr_ty body, expr_ty orelse, int lineno, int
col_offset, PyArena *arena);
#define Dict(a0, a1, a2, a3, a4) _Py_Dict(a0, a1, a2, a3, a4)
expr_ty _Py_Dict(asdl_seq * keys, asdl_seq * values, int lineno, int
col_offset, PyArena *arena);
#define Set(a0, a1, a2, a3) _Py_Set(a0, a1, a2, a3)
expr_ty _Py_Set(asdl_seq * elts, int lineno, int col_offset, PyArena *arena);
#define ListComp(a0, a1, a2, a3, a4) _Py_ListComp(a0, a1, a2, a3, a4)
expr_ty _Py_ListComp(expr_ty elt, asdl_seq * generators, int lineno, int
col_offset, PyArena *arena);
#define SetComp(a0, a1, a2, a3, a4) _Py_SetComp(a0, a1, a2, a3, a4)
expr_ty _Py_SetComp(expr_ty elt, asdl_seq * generators, int lineno, int
col_offset, PyArena *arena);
#define DictComp(a0, a1, a2, a3, a4, a5) _Py_DictComp(a0, a1, a2, a3, a4, a5)
expr_ty _Py_DictComp(expr_ty key, expr_ty value, asdl_seq * generators, int
lineno, int col_offset, PyArena *arena);
#define GeneratorExp(a0, a1, a2, a3, a4) _Py_GeneratorExp(a0, a1, a2, a3, a4)
expr_ty _Py_GeneratorExp(expr_ty elt, asdl_seq * generators, int lineno, int
col_offset, PyArena *arena);
#define Yield(a0, a1, a2, a3) _Py_Yield(a0, a1, a2, a3)
expr_ty _Py_Yield(expr_ty value, int lineno, int col_offset, PyArena *arena);
#define Compare(a0, a1, a2, a3, a4, a5) _Py_Compare(a0, a1, a2, a3, a4, a5)
expr_ty _Py_Compare(expr_ty left, asdl_int_seq * ops, asdl_seq * comparators,
int lineno, int col_offset, PyArena *arena);
#define Call(a0, a1, a2, a3, a4, a5, a6, a7) _Py_Call(a0, a1, a2, a3, a4, a5, a6, a7)
expr_ty _Py_Call(expr_ty func, asdl_seq * args, asdl_seq * keywords, expr_ty
starargs, expr_ty kwargs, int lineno, int col_offset, PyArena
*arena);
#define Repr(a0, a1, a2, a3) _Py_Repr(a0, a1, a2, a3)
expr_ty _Py_Repr(expr_ty value, int lineno, int col_offset, PyArena *arena);
#define Num(a0, a1, a2, a3) _Py_Num(a0, a1, a2, a3)
expr_ty _Py_Num(object n, int lineno, int col_offset, PyArena *arena);
#define Str(a0, a1, a2, a3) _Py_Str(a0, a1, a2, a3)
expr_ty _Py_Str(string s, int lineno, int col_offset, PyArena *arena);
#define Attribute(a0, a1, a2, a3, a4, a5) _Py_Attribute(a0, a1, a2, a3, a4, a5)
expr_ty _Py_Attribute(expr_ty value, identifier attr, expr_context_ty ctx, int
lineno, int col_offset, PyArena *arena);
#define Subscript(a0, a1, a2, a3, a4, a5) _Py_Subscript(a0, a1, a2, a3, a4, a5)
expr_ty _Py_Subscript(expr_ty value, slice_ty slice, expr_context_ty ctx, int
lineno, int col_offset, PyArena *arena);
#define Name(a0, a1, a2, a3, a4) _Py_Name(a0, a1, a2, a3, a4)
expr_ty _Py_Name(identifier id, expr_context_ty ctx, int lineno, int
col_offset, PyArena *arena);
#define List(a0, a1, a2, a3, a4) _Py_List(a0, a1, a2, a3, a4)
expr_ty _Py_List(asdl_seq * elts, expr_context_ty ctx, int lineno, int
col_offset, PyArena *arena);
#define Tuple(a0, a1, a2, a3, a4) _Py_Tuple(a0, a1, a2, a3, a4)
expr_ty _Py_Tuple(asdl_seq * elts, expr_context_ty ctx, int lineno, int
col_offset, PyArena *arena);
#define Ellipsis(a0) _Py_Ellipsis(a0)
slice_ty _Py_Ellipsis(PyArena *arena);
#define Slice(a0, a1, a2, a3) _Py_Slice(a0, a1, a2, a3)
slice_ty _Py_Slice(expr_ty lower, expr_ty upper, expr_ty step, PyArena *arena);
#define ExtSlice(a0, a1) _Py_ExtSlice(a0, a1)
slice_ty _Py_ExtSlice(asdl_seq * dims, PyArena *arena);
#define Index(a0, a1) _Py_Index(a0, a1)
slice_ty _Py_Index(expr_ty value, PyArena *arena);
#define comprehension(a0, a1, a2, a3) _Py_comprehension(a0, a1, a2, a3)
comprehension_ty _Py_comprehension(expr_ty target, expr_ty iter, asdl_seq *
ifs, PyArena *arena);
#define ExceptHandler(a0, a1, a2, a3, a4, a5) _Py_ExceptHandler(a0, a1, a2, a3, a4, a5)
excepthandler_ty _Py_ExceptHandler(expr_ty type, expr_ty name, asdl_seq * body,
int lineno, int col_offset, PyArena *arena);
#define arguments(a0, a1, a2, a3, a4) _Py_arguments(a0, a1, a2, a3, a4)
arguments_ty _Py_arguments(asdl_seq * args, identifier vararg, identifier
kwarg, asdl_seq * defaults, PyArena *arena);
#define keyword(a0, a1, a2) _Py_keyword(a0, a1, a2)
keyword_ty _Py_keyword(identifier arg, expr_ty value, PyArena *arena);
#define alias(a0, a1, a2) _Py_alias(a0, a1, a2)
alias_ty _Py_alias(identifier name, identifier asname, PyArena *arena);
PyObject* PyAST_mod2obj(mod_ty t);
mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode);
int PyAST_Check(PyObject* obj);
from_cpython/Include/asdl.h 0 → 100644
View file @ 3fa45d83
#ifndef Py_ASDL_H
#define Py_ASDL_H
typedef PyObject * identifier;
typedef PyObject * string;
typedef PyObject * object;
#ifndef __cplusplus
typedef enum {false, true} bool;
#endif
/* It would be nice if the code generated by asdl_c.py was completely
independent of Python, but it is a goal the requires too much work
at this stage. So, for example, I'll represent identifiers as
interned Python strings.
*/
/* XXX A sequence should be typed so that its use can be typechecked. */
typedef struct {
int size;
void *elements[1];
} asdl_seq;
typedef struct {
int size;
int elements[1];
} asdl_int_seq;
asdl_seq *asdl_seq_new(int size, PyArena *arena);
asdl_int_seq *asdl_int_seq_new(int size, PyArena *arena);
#define asdl_seq_GET(S, I) (S)->elements[(I)]
#define asdl_seq_LEN(S) ((S) == NULL ? 0 : (S)->size)
#ifdef Py_DEBUG
#define asdl_seq_SET(S, I, V) { \
int _asdl_i = (I); \
assert((S) && _asdl_i < (S)->size); \
(S)->elements[_asdl_i] = (V); \
}
#else
#define asdl_seq_SET(S, I, V) (S)->elements[I] = (V)
#endif
#endif /* !Py_ASDL_H */
from_cpython/Include/ast.h 0 → 100644
View file @ 3fa45d83
#ifndef Py_AST_H
#define Py_AST_H
#ifdef __cplusplus
extern "C" {
#endif
PyAPI_FUNC(mod_ty) PyAST_FromNode(const node *, PyCompilerFlags *flags,
const char *, PyArena *);
#ifdef __cplusplus
}
#endif
#endif /* !Py_AST_H */
from_cpython/Include/bitset.h 0 → 100644
View file @ 3fa45d83
#ifndef Py_BITSET_H
#define Py_BITSET_H
#ifdef __cplusplus
extern "C" {
#endif
/* Bitset interface */
#define BYTE char
typedef BYTE *bitset;
bitset newbitset(int nbits);
void delbitset(bitset bs);
#define testbit(ss, ibit) (((ss)[BIT2BYTE(ibit)] & BIT2MASK(ibit)) != 0)
int addbit(bitset bs, int ibit); /* Returns 0 if already set */
int samebitset(bitset bs1, bitset bs2, int nbits);
void mergebitset(bitset bs1, bitset bs2, int nbits);
#define BITSPERBYTE (8*sizeof(BYTE))
#define NBYTES(nbits) (((nbits) + BITSPERBYTE - 1) / BITSPERBYTE)
#define BIT2BYTE(ibit) ((ibit) / BITSPERBYTE)
#define BIT2SHIFT(ibit) ((ibit) % BITSPERBYTE)
#define BIT2MASK(ibit) (1 << BIT2SHIFT(ibit))
#define BYTE2BIT(ibyte) ((ibyte) * BITSPERBYTE)
#ifdef __cplusplus
}
#endif
#endif /* !Py_BITSET_H */
from_cpython/Include/errcode.h 0 → 100644
View file @ 3fa45d83
#ifndef Py_ERRCODE_H
#define Py_ERRCODE_H
#ifdef __cplusplus
extern "C" {
#endif
/* Error codes passed around between file input, tokenizer, parser and
interpreter. This is necessary so we can turn them into Python
exceptions at a higher level. Note that some errors have a
slightly different meaning when passed from the tokenizer to the
parser than when passed from the parser to the interpreter; e.g.
the parser only returns E_EOF when it hits EOF immediately, and it
never returns E_OK. */
#define E_OK 10 /* No error */
#define E_EOF 11 /* End Of File */
#define E_INTR 12 /* Interrupted */
#define E_TOKEN 13 /* Bad token */
#define E_SYNTAX 14 /* Syntax error */
#define E_NOMEM 15 /* Ran out of memory */
#define E_DONE 16 /* Parsing complete */
#define E_ERROR 17 /* Execution error */
#define E_TABSPACE 18 /* Inconsistent mixing of tabs and spaces */
#define E_OVERFLOW 19 /* Node had too many children */
#define E_TOODEEP 20 /* Too many indentation levels */
#define E_DEDENT 21 /* No matching outer block for dedent */
#define E_DECODE 22 /* Error in decoding into Unicode */
#define E_EOFS 23 /* EOF in triple-quoted string */
#define E_EOLS 24 /* EOL in single-quoted string */
#define E_LINECONT 25 /* Unexpected characters after a line continuation */
#ifdef __cplusplus
}
#endif
#endif /* !Py_ERRCODE_H */
from_cpython/Include/eval.h 0 → 100644
View file @ 3fa45d83
/* Interface to execute compiled code */
#ifndef Py_EVAL_H
#define Py_EVAL_H
#ifdef __cplusplus
extern "C" {
#endif
PyAPI_FUNC(PyObject *) PyEval_EvalCode(PyCodeObject *, PyObject *, PyObject *);
PyAPI_FUNC(PyObject *) PyEval_EvalCodeEx(PyCodeObject *co,
PyObject *globals,
PyObject *locals,
PyObject **args, int argc,
PyObject **kwds, int kwdc,
PyObject **defs, int defc,
PyObject *closure);
PyAPI_FUNC(PyObject *) _PyEval_CallTracing(PyObject *func, PyObject *args);
#ifdef __cplusplus
}
#endif
#endif /* !Py_EVAL_H */
from_cpython/Include/graminit.h 0 → 100644
View file @ 3fa45d83
/* Generated by Parser/pgen */
#define single_input 256
#define file_input 257
#define eval_input 258
#define decorator 259
#define decorators 260
#define decorated 261
#define funcdef 262
#define parameters 263
#define varargslist 264
#define fpdef 265
#define fplist 266
#define stmt 267
#define simple_stmt 268
#define small_stmt 269
#define expr_stmt 270
#define augassign 271
#define print_stmt 272
#define del_stmt 273
#define pass_stmt 274
#define flow_stmt 275
#define break_stmt 276
#define continue_stmt 277
#define return_stmt 278
#define yield_stmt 279
#define raise_stmt 280
#define import_stmt 281
#define import_name 282
#define import_from 283
#define import_as_name 284
#define dotted_as_name 285
#define import_as_names 286
#define dotted_as_names 287
#define dotted_name 288
#define global_stmt 289
#define exec_stmt 290
#define assert_stmt 291
#define compound_stmt 292
#define if_stmt 293
#define while_stmt 294
#define for_stmt 295
#define try_stmt 296
#define with_stmt 297
#define with_item 298
#define except_clause 299
#define suite 300
#define testlist_safe 301
#define old_test 302
#define old_lambdef 303
#define test 304
#define or_test 305
#define and_test 306
#define not_test 307
#define comparison 308
#define comp_op 309
#define expr 310
#define xor_expr 311
#define and_expr 312
#define shift_expr 313
#define arith_expr 314
#define term 315
#define factor 316
#define power 317
#define atom 318
#define listmaker 319
#define testlist_comp 320
#define lambdef 321
#define trailer 322
#define subscriptlist 323
#define subscript 324
#define sliceop 325
#define exprlist 326
#define testlist 327
#define dictorsetmaker 328
#define classdef 329
#define arglist 330
#define argument 331
#define list_iter 332
#define list_for 333
#define list_if 334
#define comp_iter 335
#define comp_for 336
#define comp_if 337
#define testlist1 338
#define encoding_decl 339
#define yield_expr 340
from_cpython/Include/grammar.h 0 → 100644
View file @ 3fa45d83
/* Grammar interface */
#ifndef Py_GRAMMAR_H
#define Py_GRAMMAR_H
#ifdef __cplusplus
extern "C" {
#endif
#include "bitset.h" /* Sigh... */
/* A label of an arc */
typedef struct {
int lb_type;
char *lb_str;
} label;
#define EMPTY 0 /* Label number 0 is by definition the empty label */
/* A list of labels */
typedef struct {
int ll_nlabels;
label *ll_label;
} labellist;
/* An arc from one state to another */
typedef struct {
short a_lbl; /* Label of this arc */
short a_arrow; /* State where this arc goes to */
} arc;
/* A state in a DFA */
typedef struct {
int s_narcs;
arc *s_arc; /* Array of arcs */
/* Optional accelerators */
int s_lower; /* Lowest label index */
int s_upper; /* Highest label index */
int *s_accel; /* Accelerator */
int s_accept; /* Nonzero for accepting state */
} state;
/* A DFA */
typedef struct {
int d_type; /* Non-terminal this represents */
char *d_name; /* For printing */
int d_initial; /* Initial state */
int d_nstates;
state *d_state; /* Array of states */
bitset d_first;
} dfa;
/* A grammar */
typedef struct {
int g_ndfas;
dfa *g_dfa; /* Array of DFAs */
labellist g_ll;
int g_start; /* Start symbol of the grammar */
int g_accel; /* Set if accelerators present */
} grammar;
/* FUNCTIONS */
grammar *newgrammar(int start);
dfa *adddfa(grammar *g, int type, char *name);
int addstate(dfa *d);
void addarc(dfa *d, int from, int to, int lbl);
dfa *PyGrammar_FindDFA(grammar *g, int type);
int addlabel(labellist *ll, int type, char *str);
int findlabel(labellist *ll, int type, char *str);
char *PyGrammar_LabelRepr(label *lb);
void translatelabels(grammar *g);
void addfirstsets(grammar *g);
void PyGrammar_AddAccelerators(grammar *g);
void PyGrammar_RemoveAccelerators(grammar *);
void printgrammar(grammar *g, FILE *fp);
void printnonterminals(grammar *g, FILE *fp);
#ifdef __cplusplus
}
#endif
#endif /* !Py_GRAMMAR_H */
from_cpython/Include/node.h 0 → 100644
View file @ 3fa45d83
/* Parse tree node interface */
#ifndef Py_NODE_H
#define Py_NODE_H
#ifdef __cplusplus
extern "C" {
#endif
typedef struct _node {
short n_type;
char *n_str;
int n_lineno;
int n_col_offset;
int n_nchildren;
struct _node *n_child;
} node;
PyAPI_FUNC(node *) PyNode_New(int type);
PyAPI_FUNC(int) PyNode_AddChild(node *n, int type,
char *str, int lineno, int col_offset);
PyAPI_FUNC(void) PyNode_Free(node *n);
#ifndef Py_LIMITED_API
Py_ssize_t _PyNode_SizeOf(node *n);
#endif
/* Node access functions */
#define NCH(n) ((n)->n_nchildren)
#define CHILD(n, i) (&(n)->n_child[i])
#define RCHILD(n, i) (CHILD(n, NCH(n) + i))
#define TYPE(n) ((n)->n_type)
#define STR(n) ((n)->n_str)
/* Assert that the type of a node is what we expect */
#define REQ(n, type) assert(TYPE(n) == (type))
PyAPI_FUNC(void) PyNode_ListTree(node *);
#ifdef __cplusplus
}
#endif
#endif /* !Py_NODE_H */
from_cpython/Include/parsetok.h 0 → 100644
View file @ 3fa45d83
/* Parser-tokenizer link interface */
#ifndef Py_PARSETOK_H
#define Py_PARSETOK_H
#ifdef __cplusplus
extern "C" {
#endif
typedef struct {
int error;
const char *filename;
int lineno;
int offset;
char *text;
int token;
int expected;
} perrdetail;
#if 0
#define PyPARSE_YIELD_IS_KEYWORD 0x0001
#endif
#define PyPARSE_DONT_IMPLY_DEDENT 0x0002
#if 0
#define PyPARSE_WITH_IS_KEYWORD 0x0003
#endif
#define PyPARSE_PRINT_IS_FUNCTION 0x0004
#define PyPARSE_UNICODE_LITERALS 0x0008
PyAPI_FUNC(node *) PyParser_ParseString(const char *, grammar *, int,
perrdetail *);
PyAPI_FUNC(node *) PyParser_ParseFile (FILE *, const char *, grammar *, int,
char *, char *, perrdetail *);
PyAPI_FUNC(node *) PyParser_ParseStringFlags(const char *, grammar *, int,
perrdetail *, int);
PyAPI_FUNC(node *) PyParser_ParseFileFlags(FILE *, const char *, grammar *,
int, char *, char *,
perrdetail *, int);
PyAPI_FUNC(node *) PyParser_ParseFileFlagsEx(FILE *, const char *, grammar *,
int, char *, char *,
perrdetail *, int *);
PyAPI_FUNC(node *) PyParser_ParseStringFlagsFilename(const char *,
const char *,
grammar *, int,
perrdetail *, int);
PyAPI_FUNC(node *) PyParser_ParseStringFlagsFilenameEx(const char *,
const char *,
grammar *, int,
perrdetail *, int *);
/* Note that he following function is defined in pythonrun.c not parsetok.c. */
PyAPI_FUNC(void) PyParser_SetError(perrdetail *);
#ifdef __cplusplus
}
#endif
#endif /* !Py_PARSETOK_H */
from_cpython/Include/pgenheaders.h 0 → 100644
View file @ 3fa45d83
#ifndef Py_PGENHEADERS_H
#define Py_PGENHEADERS_H
#ifdef __cplusplus
extern "C" {
#endif
/* Include files and extern declarations used by most of the parser. */
#include "Python.h"
PyAPI_FUNC(void) PySys_WriteStdout(const char *format, ...)
Py_GCC_ATTRIBUTE((format(printf, 1, 2)));
PyAPI_FUNC(void) PySys_WriteStderr(const char *format, ...)
Py_GCC_ATTRIBUTE((format(printf, 1, 2)));
#define addarc _Py_addarc
#define addbit _Py_addbit
#define adddfa _Py_adddfa
#define addfirstsets _Py_addfirstsets
#define addlabel _Py_addlabel
#define addstate _Py_addstate
#define delbitset _Py_delbitset
#define dumptree _Py_dumptree
#define findlabel _Py_findlabel
#define mergebitset _Py_mergebitset
#define meta_grammar _Py_meta_grammar
#define newbitset _Py_newbitset
#define newgrammar _Py_newgrammar
#define pgen _Py_pgen
#define printgrammar _Py_printgrammar
#define printnonterminals _Py_printnonterminals
#define printtree _Py_printtree
#define samebitset _Py_samebitset
#define showtree _Py_showtree
#define tok_dump _Py_tok_dump
#define translatelabels _Py_translatelabels
#ifdef __cplusplus
}
#endif
#endif /* !Py_PGENHEADERS_H */
from_cpython/Include/pystrcmp.h 0 → 100644
View file @ 3fa45d83
#ifndef Py_STRCMP_H
#define Py_STRCMP_H
#ifdef __cplusplus
extern "C" {
#endif
PyAPI_FUNC(int) PyOS_mystrnicmp(const char *, const char *, Py_ssize_t);
PyAPI_FUNC(int) PyOS_mystricmp(const char *, const char *);
#if defined(MS_WINDOWS) || defined(PYOS_OS2)
#define PyOS_strnicmp strnicmp
#define PyOS_stricmp stricmp
#else
#define PyOS_strnicmp PyOS_mystrnicmp
#define PyOS_stricmp PyOS_mystricmp
#endif
#ifdef __cplusplus
}
#endif
#endif /* !Py_STRCMP_H */
from_cpython/Include/symtable.h 0 → 100644
View file @ 3fa45d83
#ifndef Py_SYMTABLE_H
#define Py_SYMTABLE_H
#ifdef __cplusplus
extern "C" {
#endif
typedef enum _block_type { FunctionBlock, ClassBlock, ModuleBlock }
_Py_block_ty;
struct _symtable_entry;
struct symtable {
const char *st_filename; /* name of file being compiled */
struct _symtable_entry *st_cur; /* current symbol table entry */
struct _symtable_entry *st_top; /* module entry */
PyObject *st_symbols; /* dictionary of symbol table entries */
PyObject *st_stack; /* stack of namespace info */
PyObject *st_global; /* borrowed ref to MODULE in st_symbols */
int st_nblocks; /* number of blocks */
PyObject *st_private; /* name of current class or NULL */
PyFutureFeatures *st_future; /* module's future features */
};
typedef struct _symtable_entry {
PyObject_HEAD
PyObject *ste_id; /* int: key in st_symbols */
PyObject *ste_symbols; /* dict: name to flags */
PyObject *ste_name; /* string: name of block */
PyObject *ste_varnames; /* list of variable names */
PyObject *ste_children; /* list of child ids */
_Py_block_ty ste_type; /* module, class, or function */
int ste_unoptimized; /* false if namespace is optimized */
int ste_nested; /* true if block is nested */
unsigned ste_free : 1; /* true if block has free variables */
unsigned ste_child_free : 1; /* true if a child block has free vars,
including free refs to globals */
unsigned ste_generator : 1; /* true if namespace is a generator */
unsigned ste_varargs : 1; /* true if block has varargs */
unsigned ste_varkeywords : 1; /* true if block has varkeywords */
unsigned ste_returns_value : 1; /* true if namespace uses return with
an argument */
int ste_lineno; /* first line of block */
int ste_opt_lineno; /* lineno of last exec or import * */
int ste_tmpname; /* counter for listcomp temp vars */
struct symtable *ste_table;
} PySTEntryObject;
PyAPI_DATA(PyTypeObject) PySTEntry_Type;
#define PySTEntry_Check(op) (Py_TYPE(op) == &PySTEntry_Type)
PyAPI_FUNC(int) PyST_GetScope(PySTEntryObject *, PyObject *);
PyAPI_FUNC(struct symtable *) PySymtable_Build(mod_ty, const char *,
PyFutureFeatures *);
PyAPI_FUNC(PySTEntryObject *) PySymtable_Lookup(struct symtable *, void *);
PyAPI_FUNC(void) PySymtable_Free(struct symtable *);
/* Flags for def-use information */
#define DEF_GLOBAL 1 /* global stmt */
#define DEF_LOCAL 2 /* assignment in code block */
#define DEF_PARAM 2<<1 /* formal parameter */
#define USE 2<<2 /* name is used */
#define DEF_FREE 2<<3 /* name used but not defined in nested block */
#define DEF_FREE_CLASS 2<<4 /* free variable from class's method */
#define DEF_IMPORT 2<<5 /* assignment occurred via import */
#define DEF_BOUND (DEF_LOCAL | DEF_PARAM | DEF_IMPORT)
/* GLOBAL_EXPLICIT and GLOBAL_IMPLICIT are used internally by the symbol
table. GLOBAL is returned from PyST_GetScope() for either of them.
It is stored in ste_symbols at bits 12-14.
*/
#define SCOPE_OFF 11
#define SCOPE_MASK 7
#define LOCAL 1
#define GLOBAL_EXPLICIT 2
#define GLOBAL_IMPLICIT 3
#define FREE 4
#define CELL 5
/* The following three names are used for the ste_unoptimized bit field */
#define OPT_IMPORT_STAR 1
#define OPT_EXEC 2
#define OPT_BARE_EXEC 4
#define OPT_TOPLEVEL 8 /* top-level names, including eval and exec */
#define GENERATOR 1
#define GENERATOR_EXPRESSION 2
#ifdef __cplusplus
}
#endif
#endif /* !Py_SYMTABLE_H */
from_cpython/Include/token.h 0 → 100644
View file @ 3fa45d83
/* Token types */
#ifndef Py_TOKEN_H
#define Py_TOKEN_H
#ifdef __cplusplus
extern "C" {
#endif
#undef TILDE /* Prevent clash of our definition with system macro. Ex AIX, ioctl.h */
#define ENDMARKER 0
#define NAME 1
#define NUMBER 2
#define STRING 3
#define NEWLINE 4
#define INDENT 5
#define DEDENT 6
#define LPAR 7
#define RPAR 8
#define LSQB 9
#define RSQB 10
#define COLON 11
#define COMMA 12
#define SEMI 13
#define PLUS 14
#define MINUS 15
#define STAR 16
#define SLASH 17
#define VBAR 18
#define AMPER 19
#define LESS 20
#define GREATER 21
#define EQUAL 22
#define DOT 23
#define PERCENT 24
#define BACKQUOTE 25
#define LBRACE 26
#define RBRACE 27
#define EQEQUAL 28
#define NOTEQUAL 29
#define LESSEQUAL 30
#define GREATEREQUAL 31
#define TILDE 32
#define CIRCUMFLEX 33
#define LEFTSHIFT 34
#define RIGHTSHIFT 35
#define DOUBLESTAR 36
#define PLUSEQUAL 37
#define MINEQUAL 38
#define STAREQUAL 39
#define SLASHEQUAL 40
#define PERCENTEQUAL 41
#define AMPEREQUAL 42
#define VBAREQUAL 43
#define CIRCUMFLEXEQUAL 44
#define LEFTSHIFTEQUAL 45
#define RIGHTSHIFTEQUAL 46
#define DOUBLESTAREQUAL 47
#define DOUBLESLASH 48
#define DOUBLESLASHEQUAL 49
#define AT 50
/* Don't forget to update the table _PyParser_TokenNames in tokenizer.c! */
#define OP 51
#define ERRORTOKEN 52
#define N_TOKENS 53
/* Special definitions for cooperation with parser */
#define NT_OFFSET 256
#define ISTERMINAL(x) ((x) < NT_OFFSET)
#define ISNONTERMINAL(x) ((x) >= NT_OFFSET)
#define ISEOF(x) ((x) == ENDMARKER)
PyAPI_DATA(char *) _PyParser_TokenNames[]; /* Token names */
PyAPI_FUNC(int) PyToken_OneChar(int);
PyAPI_FUNC(int) PyToken_TwoChars(int, int);
PyAPI_FUNC(int) PyToken_ThreeChars(int, int, int);
#ifdef __cplusplus
}
#endif
#endif /* !Py_TOKEN_H */
from_cpython/Parser/Python.asdl 0 → 100644
View file @ 3fa45d83
-- ASDL's five builtin types are identifier, int, string, object, bool
module Python version "$Revision$"
{
mod = Module(stmt* body)
| Interactive(stmt* body)
| Expression(expr body)
-- not really an actual node but useful in Jython's typesystem.
| Suite(stmt* body)
stmt = FunctionDef(identifier name, arguments args,
stmt* body, expr* decorator_list)
| ClassDef(identifier name, expr* bases, stmt* body, expr* decorator_list)
| Return(expr? value)
| Delete(expr* targets)
| Assign(expr* targets, expr value)
| AugAssign(expr target, operator op, expr value)
-- not sure if bool is allowed, can always use int
| Print(expr? dest, expr* values, bool nl)
-- use 'orelse' because else is a keyword in target languages
| For(expr target, expr iter, stmt* body, stmt* orelse)
| While(expr test, stmt* body, stmt* orelse)
| If(expr test, stmt* body, stmt* orelse)
| With(expr context_expr, expr? optional_vars, stmt* body)
-- 'type' is a bad name
| Raise(expr? type, expr? inst, expr? tback)
| TryExcept(stmt* body, excepthandler* handlers, stmt* orelse)
| TryFinally(stmt* body, stmt* finalbody)
| Assert(expr test, expr? msg)
| Import(alias* names)
| ImportFrom(identifier? module, alias* names, int? level)
-- Doesn't capture requirement that locals must be
-- defined if globals is
-- still supports use as a function!
| Exec(expr body, expr? globals, expr? locals)
| Global(identifier* names)
| Expr(expr value)
| Pass | Break | Continue
-- XXX Jython will be different
-- col_offset is the byte offset in the utf8 string the parser uses
attributes (int lineno, int col_offset)
-- BoolOp() can use left & right?
expr = BoolOp(boolop op, expr* values)
| BinOp(expr left, operator op, expr right)
| UnaryOp(unaryop op, expr operand)
| Lambda(arguments args, expr body)
| IfExp(expr test, expr body, expr orelse)
| Dict(expr* keys, expr* values)
| Set(expr* elts)
| ListComp(expr elt, comprehension* generators)
| SetComp(expr elt, comprehension* generators)
| DictComp(expr key, expr value, comprehension* generators)
| GeneratorExp(expr elt, comprehension* generators)
-- the grammar constrains where yield expressions can occur
| Yield(expr? value)
-- need sequences for compare to distinguish between
-- x < 4 < 3 and (x < 4) < 3
| Compare(expr left, cmpop* ops, expr* comparators)
| Call(expr func, expr* args, keyword* keywords,
expr? starargs, expr? kwargs)
| Repr(expr value)
| Num(object n) -- a number as a PyObject.
| Str(string s) -- need to specify raw, unicode, etc?
-- other literals? bools?
-- the following expression can appear in assignment context
| Attribute(expr value, identifier attr, expr_context ctx)
| Subscript(expr value, slice slice, expr_context ctx)
| Name(identifier id, expr_context ctx)
| List(expr* elts, expr_context ctx)
| Tuple(expr* elts, expr_context ctx)
-- col_offset is the byte offset in the utf8 string the parser uses
attributes (int lineno, int col_offset)
expr_context = Load | Store | Del | AugLoad | AugStore | Param
slice = Ellipsis | Slice(expr? lower, expr? upper, expr? step)
| ExtSlice(slice* dims)
| Index(expr value)
boolop = And | Or
operator = Add | Sub | Mult | Div | Mod | Pow | LShift
| RShift | BitOr | BitXor | BitAnd | FloorDiv
unaryop = Invert | Not | UAdd | USub
cmpop = Eq | NotEq | Lt | LtE | Gt | GtE | Is | IsNot | In | NotIn
comprehension = (expr target, expr iter, expr* ifs)
-- not sure what to call the first argument for raise and except
excepthandler = ExceptHandler(expr? type, expr? name, stmt* body)
attributes (int lineno, int col_offset)
arguments = (expr* args, identifier? vararg,
identifier? kwarg, expr* defaults)
-- keyword arguments supplied to call
keyword = (identifier arg, expr value)
-- import name with optional 'as' alias.
alias = (identifier name, identifier? asname)
}
from_cpython/Parser/acceler.c 0 → 100644
View file @ 3fa45d83
/* Parser accelerator module */
/* The parser as originally conceived had disappointing performance.
This module does some precomputation that speeds up the selection
of a DFA based upon a token, turning a search through an array
into a simple indexing operation. The parser now cannot work
without the accelerators installed. Note that the accelerators
are installed dynamically when the parser is initialized, they
are not part of the static data structure written on graminit.[ch]
by the parser generator. */
#include "pgenheaders.h"
#include "grammar.h"
#include "node.h"
#include "token.h"
#include "parser.h"
/* Forward references */
static void fixdfa(grammar *, dfa *);
static void fixstate(grammar *, state *);
void
PyGrammar_AddAccelerators(grammar *g)
{
dfa *d;
int i;
d = g->g_dfa;
for (i = g->g_ndfas; --i >= 0; d++)
fixdfa(g, d);
g->g_accel = 1;
}
void
PyGrammar_RemoveAccelerators(grammar *g)
{
dfa *d;
int i;
g->g_accel = 0;
d = g->g_dfa;
for (i = g->g_ndfas; --i >= 0; d++) {
state *s;
int j;
s = d->d_state;
for (j = 0; j < d->d_nstates; j++, s++) {
if (s->s_accel)
PyObject_FREE(s->s_accel);
s->s_accel = NULL;
}
}
}
static void
fixdfa(grammar *g, dfa *d)
{
state *s;
int j;
s = d->d_state;
for (j = 0; j < d->d_nstates; j++, s++)
fixstate(g, s);
}
static void
fixstate(grammar *g, state *s)
{
arc *a;
int k;
int *accel;
int nl = g->g_ll.ll_nlabels;
s->s_accept = 0;
accel = (int *) PyObject_MALLOC(nl * sizeof(int));
if (accel == NULL) {
fprintf(stderr, "no mem to build parser accelerators\n");
exit(1);
}
for (k = 0; k < nl; k++)
accel[k] = -1;
a = s->s_arc;
for (k = s->s_narcs; --k >= 0; a++) {
int lbl = a->a_lbl;
label *l = &g->g_ll.ll_label[lbl];
int type = l->lb_type;
if (a->a_arrow >= (1 << 7)) {
printf("XXX too many states!\n");
continue;
}
if (ISNONTERMINAL(type)) {
dfa *d1 = PyGrammar_FindDFA(g, type);
int ibit;
if (type - NT_OFFSET >= (1 << 7)) {
printf("XXX too high nonterminal number!\n");
continue;
}
for (ibit = 0; ibit < g->g_ll.ll_nlabels; ibit++) {
if (testbit(d1->d_first, ibit)) {
if (accel[ibit] != -1)
printf("XXX ambiguity!\n");
accel[ibit] = a->a_arrow | (1 << 7) |
((type - NT_OFFSET) << 8);
}
}
}
else if (lbl == EMPTY)
s->s_accept = 1;
else if (lbl >= 0 && lbl < nl)
accel[lbl] = a->a_arrow;
}
while (nl > 0 && accel[nl-1] == -1)
nl--;
for (k = 0; k < nl && accel[k] == -1;)
k++;
if (k < nl) {
int i;
s->s_accel = (int *) PyObject_MALLOC((nl-k) * sizeof(int));
if (s->s_accel == NULL) {
fprintf(stderr, "no mem to add parser accelerators\n");
exit(1);
}
s->s_lower = k;
s->s_upper = nl;
for (i = 0; k < nl; i++, k++)
s->s_accel[i] = accel[k];
}
PyObject_FREE(accel);
}
from_cpython/Parser/asdl.py 0 → 100644
View file @ 3fa45d83
"""An implementation of the Zephyr Abstract Syntax Definition Language.
See http://asdl.sourceforge.net/ and
http://www.cs.princeton.edu/research/techreps/TR-554-97
Only supports top level module decl, not view. I'm guessing that view
is intended to support the browser and I'm not interested in the
browser.
Changes for Python: Add support for module versions
"""
import os
import traceback
import spark
class Token(object):
# spark seems to dispatch in the parser based on a token's
# type attribute
def __init__(self, type, lineno):
self.type = type
self.lineno = lineno
def __str__(self):
return self.type
def __repr__(self):
return str(self)
class Id(Token):
def __init__(self, value, lineno):
self.type = 'Id'
self.value = value
self.lineno = lineno
def __str__(self):
return self.value
class String(Token):
def __init__(self, value, lineno):
self.type = 'String'
self.value = value
self.lineno = lineno
class ASDLSyntaxError(Exception):
def __init__(self, lineno, token=None, msg=None):
self.lineno = lineno
self.token = token
self.msg = msg
def __str__(self):
if self.msg is None:
return "Error at '%s', line %d" % (self.token, self.lineno)
else:
return "%s, line %d" % (self.msg, self.lineno)
class ASDLScanner(spark.GenericScanner, object):
def tokenize(self, input):
self.rv = []
self.lineno = 1
super(ASDLScanner, self).tokenize(input)
return self.rv
def t_id(self, s):
r"[\w\.]+"
# XXX doesn't distinguish upper vs. lower, which is
# significant for ASDL.
self.rv.append(Id(s, self.lineno))
def t_string(self, s):
r'"[^"]*"'
self.rv.append(String(s, self.lineno))
def t_xxx(self, s): # not sure what this production means
r"<="
self.rv.append(Token(s, self.lineno))
def t_punctuation(self, s):
r"[\{\}\*\=\|\(\)\,\?\:]"
self.rv.append(Token(s, self.lineno))
def t_comment(self, s):
r"\-\-[^\n]*"
pass
def t_newline(self, s):
r"\n"
self.lineno += 1
def t_whitespace(self, s):
r"[ \t]+"
pass
def t_default(self, s):
r" . +"
raise ValueError, "unmatched input: %s" % `s`
class ASDLParser(spark.GenericParser, object):
def __init__(self):
super(ASDLParser, self).__init__("module")
def typestring(self, tok):
return tok.type
def error(self, tok):
raise ASDLSyntaxError(tok.lineno, tok)
def p_module_0(self, (module, name, version, _0, _1)):
" module ::= Id Id version { } "
if module.value != "module":
raise ASDLSyntaxError(module.lineno,
msg="expected 'module', found %s" % module)
return Module(name, None, version)
def p_module(self, (module, name, version, _0, definitions, _1)):
" module ::= Id Id version { definitions } "
if module.value != "module":
raise ASDLSyntaxError(module.lineno,
msg="expected 'module', found %s" % module)
return Module(name, definitions, version)
def p_version(self, (version, V)):
"version ::= Id String"
if version.value != "version":
raise ASDLSyntaxError(version.lineno,
msg="expected 'version', found %" % version)
return V
def p_definition_0(self, (definition,)):
" definitions ::= definition "
return definition
def p_definition_1(self, (definitions, definition)):
" definitions ::= definition definitions "
return definitions + definition
def p_definition(self, (id, _, type)):
" definition ::= Id = type "
return [Type(id, type)]
def p_type_0(self, (product,)):
" type ::= product "
return product
def p_type_1(self, (sum,)):
" type ::= sum "
return Sum(sum)
def p_type_2(self, (sum, id, _0, attributes, _1)):
" type ::= sum Id ( fields ) "
if id.value != "attributes":
raise ASDLSyntaxError(id.lineno,
msg="expected attributes, found %s" % id)
if attributes:
attributes.reverse()
return Sum(sum, attributes)
def p_product(self, (_0, fields, _1)):
" product ::= ( fields ) "
# XXX can't I just construct things in the right order?
fields.reverse()
return Product(fields)
def p_sum_0(self, (constructor,)):
" sum ::= constructor "
return [constructor]
def p_sum_1(self, (constructor, _, sum)):
" sum ::= constructor | sum "
return [constructor] + sum
def p_sum_2(self, (constructor, _, sum)):
" sum ::= constructor | sum "
return [constructor] + sum
def p_constructor_0(self, (id,)):
" constructor ::= Id "
return Constructor(id)
def p_constructor_1(self, (id, _0, fields, _1)):
" constructor ::= Id ( fields ) "
# XXX can't I just construct things in the right order?
fields.reverse()
return Constructor(id, fields)
def p_fields_0(self, (field,)):
" fields ::= field "
return [field]
def p_fields_1(self, (field, _, fields)):
" fields ::= field , fields "
return fields + [field]
def p_field_0(self, (type,)):
" field ::= Id "
return Field(type)
def p_field_1(self, (type, name)):
" field ::= Id Id "
return Field(type, name)
def p_field_2(self, (type, _, name)):
" field ::= Id * Id "
return Field(type, name, seq=True)
def p_field_3(self, (type, _, name)):
" field ::= Id ? Id "
return Field(type, name, opt=True)
def p_field_4(self, (type, _)):
" field ::= Id * "
return Field(type, seq=True)
def p_field_5(self, (type, _)):
" field ::= Id ? "
return Field(type, opt=True)
builtin_types = ("identifier", "string", "int", "bool", "object")
# below is a collection of classes to capture the AST of an AST :-)
# not sure if any of the methods are useful yet, but I'm adding them
# piecemeal as they seem helpful
class AST(object):
pass # a marker class
class Module(AST):
def __init__(self, name, dfns, version):
self.name = name
self.dfns = dfns
self.version = version
self.types = {} # maps type name to value (from dfns)
for type in dfns:
self.types[type.name.value] = type.value
def __repr__(self):
return "Module(%s, %s)" % (self.name, self.dfns)
class Type(AST):
def __init__(self, name, value):
self.name = name
self.value = value
def __repr__(self):
return "Type(%s, %s)" % (self.name, self.value)
class Constructor(AST):
def __init__(self, name, fields=None):
self.name = name
self.fields = fields or []
def __repr__(self):
return "Constructor(%s, %s)" % (self.name, self.fields)
class Field(AST):
def __init__(self, type, name=None, seq=False, opt=False):
self.type = type
self.name = name
self.seq = seq
self.opt = opt
def __repr__(self):
if self.seq:
extra = ", seq=True"
elif self.opt:
extra = ", opt=True"
else:
extra = ""
if self.name is None:
return "Field(%s%s)" % (self.type, extra)
else:
return "Field(%s, %s%s)" % (self.type, self.name, extra)
class Sum(AST):
def __init__(self, types, attributes=None):
self.types = types
self.attributes = attributes or []
def __repr__(self):
if self.attributes is None:
return "Sum(%s)" % self.types
else:
return "Sum(%s, %s)" % (self.types, self.attributes)
class Product(AST):
def __init__(self, fields):
self.fields = fields
def __repr__(self):
return "Product(%s)" % self.fields
class VisitorBase(object):
def __init__(self, skip=False):
self.cache = {}
self.skip = skip
def visit(self, object, *args):
meth = self._dispatch(object)
if meth is None:
return
try:
meth(object, *args)
except Exception, err:
print "Error visiting", repr(object)
print err
traceback.print_exc()
# XXX hack
if hasattr(self, 'file'):
self.file.flush()
os._exit(1)
def _dispatch(self, object):
assert isinstance(object, AST), repr(object)
klass = object.__class__
meth = self.cache.get(klass)
if meth is None:
methname = "visit" + klass.__name__
if self.skip:
meth = getattr(self, methname, None)
else:
meth = getattr(self, methname)
self.cache[klass] = meth
return meth
class Check(VisitorBase):
def __init__(self):
super(Check, self).__init__(skip=True)
self.cons = {}
self.errors = 0
self.types = {}
def visitModule(self, mod):
for dfn in mod.dfns:
self.visit(dfn)
def visitType(self, type):
self.visit(type.value, str(type.name))
def visitSum(self, sum, name):
for t in sum.types:
self.visit(t, name)
def visitConstructor(self, cons, name):
key = str(cons.name)
conflict = self.cons.get(key)
if conflict is None:
self.cons[key] = name
else:
print "Redefinition of constructor %s" % key
print "Defined in %s and %s" % (conflict, name)
self.errors += 1
for f in cons.fields:
self.visit(f, key)
def visitField(self, field, name):
key = str(field.type)
l = self.types.setdefault(key, [])
l.append(name)
def visitProduct(self, prod, name):
for f in prod.fields:
self.visit(f, name)
def check(mod):
v = Check()
v.visit(mod)
for t in v.types:
if t not in mod.types and not t in builtin_types:
v.errors += 1
uses = ", ".join(v.types[t])
print "Undefined type %s, used in %s" % (t, uses)
return not v.errors
def parse(file):
scanner = ASDLScanner()
parser = ASDLParser()
buf = open(file).read()
tokens = scanner.tokenize(buf)
try:
return parser.parse(tokens)
except ASDLSyntaxError, err:
print err
lines = buf.split("\n")
print lines[err.lineno - 1] # lines starts at 0, files at 1
if __name__ == "__main__":
import glob
import sys
if len(sys.argv) > 1:
files = sys.argv[1:]
else:
testdir = "tests"
files = glob.glob(testdir + "/*.asdl")
for file in files:
print file
mod = parse(file)
print "module", mod.name
print len(mod.dfns), "definitions"
if not check(mod):
print "Check failed"
else:
for dfn in mod.dfns:
print dfn.type
from_cpython/Parser/asdl_c.py 0 → 100755
View file @ 3fa45d83
#! /usr/bin/env python
"""Generate C code from an ASDL description."""
# TO DO
# handle fields that have a type but no name
import os, sys
import asdl
TABSIZE = 8
MAX_COL = 80
def get_c_type(name):
"""Return a string for the C name of the type.
This function special cases the default types provided by asdl:
identifier, string, int, bool.
"""
# XXX ack! need to figure out where Id is useful and where string
if isinstance(name, asdl.Id):
name = name.value
if name in asdl.builtin_types:
return name
else:
return "%s_ty" % name
def reflow_lines(s, depth):
"""Reflow the line s indented depth tabs.
Return a sequence of lines where no line extends beyond MAX_COL
when properly indented. The first line is properly indented based
exclusively on depth * TABSIZE. All following lines -- these are
the reflowed lines generated by this function -- start at the same
column as the first character beyond the opening { in the first
line.
"""
size = MAX_COL - depth * TABSIZE
if len(s) < size:
return [s]
lines = []
cur = s
padding = ""
while len(cur) > size:
i = cur.rfind(' ', 0, size)
# XXX this should be fixed for real
if i == -1 and 'GeneratorExp' in cur:
i = size + 3
assert i != -1, "Impossible line %d to reflow: %r" % (size, s)
lines.append(padding + cur[:i])
if len(lines) == 1:
# find new size based on brace
j = cur.find('{', 0, i)
if j >= 0:
j += 2 # account for the brace and the space after it
size -= j
padding = " " * j
else:
j = cur.find('(', 0, i)
if j >= 0:
j += 1 # account for the paren (no space after it)
size -= j
padding = " " * j
cur = cur[i+1:]
else:
lines.append(padding + cur)
return lines
def is_simple(sum):
"""Return True if a sum is a simple.
A sum is simple if its types have no fields, e.g.
unaryop = Invert | Not | UAdd | USub
"""
for t in sum.types:
if t.fields:
return False
return True
class EmitVisitor(asdl.VisitorBase):
"""Visit that emits lines"""
def __init__(self, file):
self.file = file
super(EmitVisitor, self).__init__()
def emit(self, s, depth, reflow=True):
# XXX reflow long lines?
if reflow:
lines = reflow_lines(s, depth)
else:
lines = [s]
for line in lines:
line = (" " * TABSIZE * depth) + line + "\n"
self.file.write(line)
class TypeDefVisitor(EmitVisitor):
def visitModule(self, mod):
for dfn in mod.dfns:
self.visit(dfn)
def visitType(self, type, depth=0):
self.visit(type.value, type.name, depth)
def visitSum(self, sum, name, depth):
if is_simple(sum):
self.simple_sum(sum, name, depth)
else:
self.sum_with_constructors(sum, name, depth)
def simple_sum(self, sum, name, depth):
enum = []
for i in range(len(sum.types)):
type = sum.types[i]
enum.append("%s=%d" % (type.name, i + 1))
enums = ", ".join(enum)
ctype = get_c_type(name)
s = "typedef enum _%s { %s } %s;" % (name, enums, ctype)
self.emit(s, depth)
self.emit("", depth)
def sum_with_constructors(self, sum, name, depth):
ctype = get_c_type(name)
s = "typedef struct _%(name)s *%(ctype)s;" % locals()
self.emit(s, depth)
self.emit("", depth)
def visitProduct(self, product, name, depth):
ctype = get_c_type(name)
s = "typedef struct _%(name)s *%(ctype)s;" % locals()
self.emit(s, depth)
self.emit("", depth)
class StructVisitor(EmitVisitor):
"""Visitor to generate typdefs for AST."""
def visitModule(self, mod):
for dfn in mod.dfns:
self.visit(dfn)
def visitType(self, type, depth=0):
self.visit(type.value, type.name, depth)
def visitSum(self, sum, name, depth):
if not is_simple(sum):
self.sum_with_constructors(sum, name, depth)
def sum_with_constructors(self, sum, name, depth):
def emit(s, depth=depth):
self.emit(s % sys._getframe(1).f_locals, depth)
enum = []
for i in range(len(sum.types)):
type = sum.types[i]
enum.append("%s_kind=%d" % (type.name, i + 1))
emit("enum _%(name)s_kind {" + ", ".join(enum) + "};")
emit("struct _%(name)s {")
emit("enum _%(name)s_kind kind;", depth + 1)
emit("union {", depth + 1)
for t in sum.types:
self.visit(t, depth + 2)
emit("} v;", depth + 1)
for field in sum.attributes:
# rudimentary attribute handling
type = str(field.type)
assert type in asdl.builtin_types, type
emit("%s %s;" % (type, field.name), depth + 1);
emit("};")
emit("")
def visitConstructor(self, cons, depth):
if cons.fields:
self.emit("struct {", depth)
for f in cons.fields:
self.visit(f, depth + 1)
self.emit("} %s;" % cons.name, depth)
self.emit("", depth)
else:
# XXX not sure what I want here, nothing is probably fine
pass
def visitField(self, field, depth):
# XXX need to lookup field.type, because it might be something
# like a builtin...
ctype = get_c_type(field.type)
name = field.name
if field.seq:
if field.type.value in ('cmpop',):
self.emit("asdl_int_seq *%(name)s;" % locals(), depth)
else:
self.emit("asdl_seq *%(name)s;" % locals(), depth)
else:
self.emit("%(ctype)s %(name)s;" % locals(), depth)
def visitProduct(self, product, name, depth):
self.emit("struct _%(name)s {" % locals(), depth)
for f in product.fields:
self.visit(f, depth + 1)
self.emit("};", depth)
self.emit("", depth)
class PrototypeVisitor(EmitVisitor):
"""Generate function prototypes for the .h file"""
def visitModule(self, mod):
for dfn in mod.dfns:
self.visit(dfn)
def visitType(self, type):
self.visit(type.value, type.name)
def visitSum(self, sum, name):
if is_simple(sum):
pass # XXX
else:
for t in sum.types:
self.visit(t, name, sum.attributes)
def get_args(self, fields):
"""Return list of C argument into, one for each field.
Argument info is 3-tuple of a C type, variable name, and flag
that is true if type can be NULL.
"""
args = []
unnamed = {}
for f in fields:
if f.name is None:
name = f.type
c = unnamed[name] = unnamed.get(name, 0) + 1
if c > 1:
name = "name%d" % (c - 1)
else:
name = f.name
# XXX should extend get_c_type() to handle this
if f.seq:
if f.type.value in ('cmpop',):
ctype = "asdl_int_seq *"
else:
ctype = "asdl_seq *"
else:
ctype = get_c_type(f.type)
args.append((ctype, name, f.opt or f.seq))
return args
def visitConstructor(self, cons, type, attrs):
args = self.get_args(cons.fields)
attrs = self.get_args(attrs)
ctype = get_c_type(type)
self.emit_function(cons.name, ctype, args, attrs)
def emit_function(self, name, ctype, args, attrs, union=True):
args = args + attrs
if args:
argstr = ", ".join(["%s %s" % (atype, aname)
for atype, aname, opt in args])
argstr += ", PyArena *arena"
else:
argstr = "PyArena *arena"
margs = "a0"
for i in range(1, len(args)+1):
margs += ", a%d" % i
self.emit("#define %s(%s) _Py_%s(%s)" % (name, margs, name, margs), 0,
reflow=False)
self.emit("%s _Py_%s(%s);" % (ctype, name, argstr), False)
def visitProduct(self, prod, name):
self.emit_function(name, get_c_type(name),
self.get_args(prod.fields), [], union=False)
class FunctionVisitor(PrototypeVisitor):
"""Visitor to generate constructor functions for AST."""
def emit_function(self, name, ctype, args, attrs, union=True):
def emit(s, depth=0, reflow=True):
self.emit(s, depth, reflow)
argstr = ", ".join(["%s %s" % (atype, aname)
for atype, aname, opt in args + attrs])
if argstr:
argstr += ", PyArena *arena"
else:
argstr = "PyArena *arena"
self.emit("%s" % ctype, 0)
emit("%s(%s)" % (name, argstr))
emit("{")
emit("%s p;" % ctype, 1)
for argtype, argname, opt in args:
# XXX hack alert: false is allowed for a bool
if not opt and not (argtype == "bool" or argtype == "int"):
emit("if (!%s) {" % argname, 1)
emit("PyErr_SetString(PyExc_ValueError,", 2)
msg = "field %s is required for %s" % (argname, name)
emit(' "%s");' % msg,
2, reflow=False)
emit('return NULL;', 2)
emit('}', 1)
emit("p = (%s)PyArena_Malloc(arena, sizeof(*p));" % ctype, 1);
emit("if (!p)", 1)
emit("return NULL;", 2)
if union:
self.emit_body_union(name, args, attrs)
else:
self.emit_body_struct(name, args, attrs)
emit("return p;", 1)
emit("}")
emit("")
def emit_body_union(self, name, args, attrs):
def emit(s, depth=0, reflow=True):
self.emit(s, depth, reflow)
emit("p->kind = %s_kind;" % name, 1)
for argtype, argname, opt in args:
emit("p->v.%s.%s = %s;" % (name, argname, argname), 1)
for argtype, argname, opt in attrs:
emit("p->%s = %s;" % (argname, argname), 1)
def emit_body_struct(self, name, args, attrs):
def emit(s, depth=0, reflow=True):
self.emit(s, depth, reflow)
for argtype, argname, opt in args:
emit("p->%s = %s;" % (argname, argname), 1)
assert not attrs
class PickleVisitor(EmitVisitor):
def visitModule(self, mod):
for dfn in mod.dfns:
self.visit(dfn)
def visitType(self, type):
self.visit(type.value, type.name)
def visitSum(self, sum, name):
pass
def visitProduct(self, sum, name):
pass
def visitConstructor(self, cons, name):
pass
def visitField(self, sum):
pass
class Obj2ModPrototypeVisitor(PickleVisitor):
def visitProduct(self, prod, name):
code = "static int obj2ast_%s(PyObject* obj, %s* out, PyArena* arena);"
self.emit(code % (name, get_c_type(name)), 0)
visitSum = visitProduct
class Obj2ModVisitor(PickleVisitor):
def funcHeader(self, name):
ctype = get_c_type(name)
self.emit("int", 0)
self.emit("obj2ast_%s(PyObject* obj, %s* out, PyArena* arena)" % (name, ctype), 0)
self.emit("{", 0)
self.emit("PyObject* tmp = NULL;", 1)
self.emit("int isinstance;", 1)
self.emit("", 0)
def sumTrailer(self, name):
self.emit("", 0)
self.emit("tmp = PyObject_Repr(obj);", 1)
# there's really nothing more we can do if this fails ...
self.emit("if (tmp == NULL) goto failed;", 1)
error = "expected some sort of %s, but got %%.400s" % name
format = "PyErr_Format(PyExc_TypeError, \"%s\", PyString_AS_STRING(tmp));"
self.emit(format % error, 1, reflow=False)
self.emit("failed:", 0)
self.emit("Py_XDECREF(tmp);", 1)
self.emit("return 1;", 1)
self.emit("}", 0)
self.emit("", 0)
def simpleSum(self, sum, name):
self.funcHeader(name)
for t in sum.types:
line = ("isinstance = PyObject_IsInstance(obj, "
"(PyObject *)%s_type);")
self.emit(line % (t.name,), 1)
self.emit("if (isinstance == -1) {", 1)
self.emit("return 1;", 2)
self.emit("}", 1)
self.emit("if (isinstance) {", 1)
self.emit("*out = %s;" % t.name, 2)
self.emit("return 0;", 2)
self.emit("}", 1)
self.sumTrailer(name)
def buildArgs(self, fields):
return ", ".join(fields + ["arena"])
def complexSum(self, sum, name):
self.funcHeader(name)
for a in sum.attributes:
self.visitAttributeDeclaration(a, name, sum=sum)
self.emit("", 0)
# XXX: should we only do this for 'expr'?
self.emit("if (obj == Py_None) {", 1)
self.emit("*out = NULL;", 2)
self.emit("return 0;", 2)
self.emit("}", 1)
for a in sum.attributes:
self.visitField(a, name, sum=sum, depth=1)
for t in sum.types:
line = "isinstance = PyObject_IsInstance(obj, (PyObject*)%s_type);"
self.emit(line % (t.name,), 1)
self.emit("if (isinstance == -1) {", 1)
self.emit("return 1;", 2)
self.emit("}", 1)
self.emit("if (isinstance) {", 1)
for f in t.fields:
self.visitFieldDeclaration(f, t.name, sum=sum, depth=2)
self.emit("", 0)
for f in t.fields:
self.visitField(f, t.name, sum=sum, depth=2)
args = [f.name.value for f in t.fields] + [a.name.value for a in sum.attributes]
self.emit("*out = %s(%s);" % (t.name, self.buildArgs(args)), 2)
self.emit("if (*out == NULL) goto failed;", 2)
self.emit("return 0;", 2)
self.emit("}", 1)
self.sumTrailer(name)
def visitAttributeDeclaration(self, a, name, sum=sum):
ctype = get_c_type(a.type)
self.emit("%s %s;" % (ctype, a.name), 1)
def visitSum(self, sum, name):
if is_simple(sum):
self.simpleSum(sum, name)
else:
self.complexSum(sum, name)
def visitProduct(self, prod, name):
ctype = get_c_type(name)
self.emit("int", 0)
self.emit("obj2ast_%s(PyObject* obj, %s* out, PyArena* arena)" % (name, ctype), 0)
self.emit("{", 0)
self.emit("PyObject* tmp = NULL;", 1)
for f in prod.fields:
self.visitFieldDeclaration(f, name, prod=prod, depth=1)
self.emit("", 0)
for f in prod.fields:
self.visitField(f, name, prod=prod, depth=1)
args = [f.name.value for f in prod.fields]
self.emit("*out = %s(%s);" % (name, self.buildArgs(args)), 1)
self.emit("return 0;", 1)
self.emit("failed:", 0)
self.emit("Py_XDECREF(tmp);", 1)
self.emit("return 1;", 1)
self.emit("}", 0)
self.emit("", 0)
def visitFieldDeclaration(self, field, name, sum=None, prod=None, depth=0):
ctype = get_c_type(field.type)
if field.seq:
if self.isSimpleType(field):
self.emit("asdl_int_seq* %s;" % field.name, depth)
else:
self.emit("asdl_seq* %s;" % field.name, depth)
else:
ctype = get_c_type(field.type)
self.emit("%s %s;" % (ctype, field.name), depth)
def isSimpleSum(self, field):
# XXX can the members of this list be determined automatically?
return field.type.value in ('expr_context', 'boolop', 'operator',
'unaryop', 'cmpop')
def isNumeric(self, field):
return get_c_type(field.type) in ("int", "bool")
def isSimpleType(self, field):
return self.isSimpleSum(field) or self.isNumeric(field)
def visitField(self, field, name, sum=None, prod=None, depth=0):
ctype = get_c_type(field.type)
self.emit("if (PyObject_HasAttrString(obj, \"%s\")) {" % field.name, depth)
self.emit("int res;", depth+1)
if field.seq:
self.emit("Py_ssize_t len;", depth+1)
self.emit("Py_ssize_t i;", depth+1)
self.emit("tmp = PyObject_GetAttrString(obj, \"%s\");" % field.name, depth+1)
self.emit("if (tmp == NULL) goto failed;", depth+1)
if field.seq:
self.emit("if (!PyList_Check(tmp)) {", depth+1)
self.emit("PyErr_Format(PyExc_TypeError, \"%s field \\\"%s\\\" must "
"be a list, not a %%.200s\", tmp->ob_type->tp_name);" %
(name, field.name),
depth+2, reflow=False)
self.emit("goto failed;", depth+2)
self.emit("}", depth+1)
self.emit("len = PyList_GET_SIZE(tmp);", depth+1)
if self.isSimpleType(field):
self.emit("%s = asdl_int_seq_new(len, arena);" % field.name, depth+1)
else:
self.emit("%s = asdl_seq_new(len, arena);" % field.name, depth+1)
self.emit("if (%s == NULL) goto failed;" % field.name, depth+1)
self.emit("for (i = 0; i < len; i++) {", depth+1)
self.emit("%s value;" % ctype, depth+2)
self.emit("res = obj2ast_%s(PyList_GET_ITEM(tmp, i), &value, arena);" %
field.type, depth+2, reflow=False)
self.emit("if (res != 0) goto failed;", depth+2)
self.emit("asdl_seq_SET(%s, i, value);" % field.name, depth+2)
self.emit("}", depth+1)
else:
self.emit("res = obj2ast_%s(tmp, &%s, arena);" %
(field.type, field.name), depth+1)
self.emit("if (res != 0) goto failed;", depth+1)
self.emit("Py_XDECREF(tmp);", depth+1)
self.emit("tmp = NULL;", depth+1)
self.emit("} else {", depth)
if not field.opt:
message = "required field \\\"%s\\\" missing from %s" % (field.name, name)
format = "PyErr_SetString(PyExc_TypeError, \"%s\");"
self.emit(format % message, depth+1, reflow=False)
self.emit("return 1;", depth+1)
else:
if self.isNumeric(field):
self.emit("%s = 0;" % field.name, depth+1)
elif not self.isSimpleType(field):
self.emit("%s = NULL;" % field.name, depth+1)
else:
raise TypeError("could not determine the default value for %s" % field.name)
self.emit("}", depth)
class MarshalPrototypeVisitor(PickleVisitor):
def prototype(self, sum, name):
ctype = get_c_type(name)
self.emit("static int marshal_write_%s(PyObject **, int *, %s);"
% (name, ctype), 0)
visitProduct = visitSum = prototype
class PyTypesDeclareVisitor(PickleVisitor):
def visitProduct(self, prod, name):
self.emit("static PyTypeObject *%s_type;" % name, 0)
self.emit("static PyObject* ast2obj_%s(void*);" % name, 0)
if prod.fields:
self.emit("static char *%s_fields[]={" % name,0)
for f in prod.fields:
self.emit('"%s",' % f.name, 1)
self.emit("};", 0)
def visitSum(self, sum, name):
self.emit("static PyTypeObject *%s_type;" % name, 0)
if sum.attributes:
self.emit("static char *%s_attributes[] = {" % name, 0)
for a in sum.attributes:
self.emit('"%s",' % a.name, 1)
self.emit("};", 0)
ptype = "void*"
if is_simple(sum):
ptype = get_c_type(name)
tnames = []
for t in sum.types:
tnames.append(str(t.name)+"_singleton")
tnames = ", *".join(tnames)
self.emit("static PyObject *%s;" % tnames, 0)
self.emit("static PyObject* ast2obj_%s(%s);" % (name, ptype), 0)
for t in sum.types:
self.visitConstructor(t, name)
def visitConstructor(self, cons, name):
self.emit("static PyTypeObject *%s_type;" % cons.name, 0)
if cons.fields:
self.emit("static char *%s_fields[]={" % cons.name, 0)
for t in cons.fields:
self.emit('"%s",' % t.name, 1)
self.emit("};",0)
class PyTypesVisitor(PickleVisitor):
def visitModule(self, mod):
self.emit("""
static int
ast_type_init(PyObject *self, PyObject *args, PyObject *kw)
{
Py_ssize_t i, numfields = 0;
int res = -1;
PyObject *key, *value, *fields;
fields = PyObject_GetAttrString((PyObject*)Py_TYPE(self), "_fields");
if (!fields)
PyErr_Clear();
if (fields) {
numfields = PySequence_Size(fields);
if (numfields == -1)
goto cleanup;
}
res = 0; /* if no error occurs, this stays 0 to the end */
if (PyTuple_GET_SIZE(args) > 0) {
if (numfields != PyTuple_GET_SIZE(args)) {
PyErr_Format(PyExc_TypeError, "%.400s constructor takes %s"
"%zd positional argument%s",
Py_TYPE(self)->tp_name,
numfields == 0 ? "" : "either 0 or ",
numfields, numfields == 1 ? "" : "s");
res = -1;
goto cleanup;
}
for (i = 0; i < PyTuple_GET_SIZE(args); i++) {
/* cannot be reached when fields is NULL */
PyObject *name = PySequence_GetItem(fields, i);
if (!name) {
res = -1;
goto cleanup;
}
res = PyObject_SetAttr(self, name, PyTuple_GET_ITEM(args, i));
Py_DECREF(name);
if (res < 0)
goto cleanup;
}
}
if (kw) {
i = 0; /* needed by PyDict_Next */
while (PyDict_Next(kw, &i, &key, &value)) {
res = PyObject_SetAttr(self, key, value);
if (res < 0)
goto cleanup;
}
}
cleanup:
Py_XDECREF(fields);
return res;
}
/* Pickling support */
static PyObject *
ast_type_reduce(PyObject *self, PyObject *unused)
{
PyObject *res;
PyObject *dict = PyObject_GetAttrString(self, "__dict__");
if (dict == NULL) {
if (PyErr_ExceptionMatches(PyExc_AttributeError))
PyErr_Clear();
else
return NULL;
}
if (dict) {
res = Py_BuildValue("O()O", Py_TYPE(self), dict);
Py_DECREF(dict);
return res;
}
return Py_BuildValue("O()", Py_TYPE(self));
}
static PyMethodDef ast_type_methods[] = {
{"__reduce__", ast_type_reduce, METH_NOARGS, NULL},
{NULL}
};
static PyTypeObject AST_type = {
PyVarObject_HEAD_INIT(&PyType_Type, 0)
"_ast.AST",
sizeof(PyObject),
0,
0, /* tp_dealloc */
0, /* tp_print */
0, /* tp_getattr */
0, /* tp_setattr */
0, /* tp_compare */
0, /* tp_repr */
0, /* tp_as_number */
0, /* tp_as_sequence */
0, /* tp_as_mapping */
0, /* tp_hash */
0, /* tp_call */
0, /* tp_str */
PyObject_GenericGetAttr, /* tp_getattro */
PyObject_GenericSetAttr, /* tp_setattro */
0, /* tp_as_buffer */
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */
0, /* tp_doc */
0, /* tp_traverse */
0, /* tp_clear */
0, /* tp_richcompare */
0, /* tp_weaklistoffset */
0, /* tp_iter */
0, /* tp_iternext */
ast_type_methods, /* tp_methods */
0, /* tp_members */
0, /* tp_getset */
0, /* tp_base */
0, /* tp_dict */
0, /* tp_descr_get */
0, /* tp_descr_set */
0, /* tp_dictoffset */
(initproc)ast_type_init, /* tp_init */
PyType_GenericAlloc, /* tp_alloc */
PyType_GenericNew, /* tp_new */
PyObject_Del, /* tp_free */
};
static PyTypeObject* make_type(char *type, PyTypeObject* base, char**fields, int num_fields)
{
PyObject *fnames, *result;
int i;
fnames = PyTuple_New(num_fields);
if (!fnames) return NULL;
for (i = 0; i < num_fields; i++) {
PyObject *field = PyString_FromString(fields[i]);
if (!field) {
Py_DECREF(fnames);
return NULL;
}
PyTuple_SET_ITEM(fnames, i, field);
}
result = PyObject_CallFunction((PyObject*)&PyType_Type, "s(O){sOss}",
type, base, "_fields", fnames, "__module__", "_ast");
Py_DECREF(fnames);
return (PyTypeObject*)result;
}
static int add_attributes(PyTypeObject* type, char**attrs, int num_fields)
{
int i, result;
PyObject *s, *l = PyTuple_New(num_fields);
if (!l)
return 0;
for (i = 0; i < num_fields; i++) {
s = PyString_FromString(attrs[i]);
if (!s) {
Py_DECREF(l);
return 0;
}
PyTuple_SET_ITEM(l, i, s);
}
result = PyObject_SetAttrString((PyObject*)type, "_attributes", l) >= 0;
Py_DECREF(l);
return result;
}
/* Conversion AST -> Python */
static PyObject* ast2obj_list(asdl_seq *seq, PyObject* (*func)(void*))
{
int i, n = asdl_seq_LEN(seq);
PyObject *result = PyList_New(n);
PyObject *value;
if (!result)
return NULL;
for (i = 0; i < n; i++) {
value = func(asdl_seq_GET(seq, i));
if (!value) {
Py_DECREF(result);
return NULL;
}
PyList_SET_ITEM(result, i, value);
}
return result;
}
static PyObject* ast2obj_object(void *o)
{
if (!o)
o = Py_None;
Py_INCREF((PyObject*)o);
return (PyObject*)o;
}
#define ast2obj_identifier ast2obj_object
#define ast2obj_string ast2obj_object
static PyObject* ast2obj_bool(bool b)
{
return PyBool_FromLong(b);
}
static PyObject* ast2obj_int(long b)
{
return PyInt_FromLong(b);
}
/* Conversion Python -> AST */
static int obj2ast_object(PyObject* obj, PyObject** out, PyArena* arena)
{
if (obj == Py_None)
obj = NULL;
if (obj)
PyArena_AddPyObject(arena, obj);
Py_XINCREF(obj);
*out = obj;
return 0;
}
static int obj2ast_identifier(PyObject* obj, PyObject** out, PyArena* arena)
{
if (!PyString_CheckExact(obj) && obj != Py_None) {
PyErr_Format(PyExc_TypeError,
"AST identifier must be of type str");
return 1;
}
return obj2ast_object(obj, out, arena);
}
static int obj2ast_string(PyObject* obj, PyObject** out, PyArena* arena)
{
if (!PyString_CheckExact(obj) && !PyUnicode_CheckExact(obj)) {
PyErr_SetString(PyExc_TypeError,
"AST string must be of type str or unicode");
return 1;
}
return obj2ast_object(obj, out, arena);
}
static int obj2ast_int(PyObject* obj, int* out, PyArena* arena)
{
int i;
if (!PyInt_Check(obj) && !PyLong_Check(obj)) {
PyObject *s = PyObject_Repr(obj);
if (s == NULL) return 1;
PyErr_Format(PyExc_ValueError, "invalid integer value: %.400s",
PyString_AS_STRING(s));
Py_DECREF(s);
return 1;
}
i = (int)PyLong_AsLong(obj);
if (i == -1 && PyErr_Occurred())
return 1;
*out = i;
return 0;
}
static int obj2ast_bool(PyObject* obj, bool* out, PyArena* arena)
{
if (!PyBool_Check(obj)) {
PyObject *s = PyObject_Repr(obj);
if (s == NULL) return 1;
PyErr_Format(PyExc_ValueError, "invalid boolean value: %.400s",
PyString_AS_STRING(s));
Py_DECREF(s);
return 1;
}
*out = (obj == Py_True);
return 0;
}
static int add_ast_fields(void)
{
PyObject *empty_tuple, *d;
if (PyType_Ready(&AST_type) < 0)
return -1;
d = AST_type.tp_dict;
empty_tuple = PyTuple_New(0);
if (!empty_tuple ||
PyDict_SetItemString(d, "_fields", empty_tuple) < 0 ||
PyDict_SetItemString(d, "_attributes", empty_tuple) < 0) {
Py_XDECREF(empty_tuple);
return -1;
}
Py_DECREF(empty_tuple);
return 0;
}
""", 0, reflow=False)
self.emit("static int init_types(void)",0)
self.emit("{", 0)
self.emit("static int initialized;", 1)
self.emit("if (initialized) return 1;", 1)
self.emit("if (add_ast_fields() < 0) return 0;", 1)
for dfn in mod.dfns:
self.visit(dfn)
self.emit("initialized = 1;", 1)
self.emit("return 1;", 1);
self.emit("}", 0)
def visitProduct(self, prod, name):
if prod.fields:
fields = name.value+"_fields"
else:
fields = "NULL"
self.emit('%s_type = make_type("%s", &AST_type, %s, %d);' %
(name, name, fields, len(prod.fields)), 1)
self.emit("if (!%s_type) return 0;" % name, 1)
def visitSum(self, sum, name):
self.emit('%s_type = make_type("%s", &AST_type, NULL, 0);' %
(name, name), 1)
self.emit("if (!%s_type) return 0;" % name, 1)
if sum.attributes:
self.emit("if (!add_attributes(%s_type, %s_attributes, %d)) return 0;" %
(name, name, len(sum.attributes)), 1)
else:
self.emit("if (!add_attributes(%s_type, NULL, 0)) return 0;" % name, 1)
simple = is_simple(sum)
for t in sum.types:
self.visitConstructor(t, name, simple)
def visitConstructor(self, cons, name, simple):
if cons.fields:
fields = cons.name.value+"_fields"
else:
fields = "NULL"
self.emit('%s_type = make_type("%s", %s_type, %s, %d);' %
(cons.name, cons.name, name, fields, len(cons.fields)), 1)
self.emit("if (!%s_type) return 0;" % cons.name, 1)
if simple:
self.emit("%s_singleton = PyType_GenericNew(%s_type, NULL, NULL);" %
(cons.name, cons.name), 1)
self.emit("if (!%s_singleton) return 0;" % cons.name, 1)
class ASTModuleVisitor(PickleVisitor):
def visitModule(self, mod):
self.emit("PyMODINIT_FUNC", 0)
self.emit("init_ast(void)", 0)
self.emit("{", 0)
self.emit("PyObject *m, *d;", 1)
self.emit("if (!init_types()) return;", 1)
self.emit('m = Py_InitModule3("_ast", NULL, NULL);', 1)
self.emit("if (!m) return;", 1)
self.emit("d = PyModule_GetDict(m);", 1)
self.emit('if (PyDict_SetItemString(d, "AST", (PyObject*)&AST_type) < 0) return;', 1)
self.emit('if (PyModule_AddIntConstant(m, "PyCF_ONLY_AST", PyCF_ONLY_AST) < 0)', 1)
self.emit("return;", 2)
# Value of version: "$Revision$"
self.emit('if (PyModule_AddStringConstant(m, "__version__", "%s") < 0)'
% mod.version, 1)
self.emit("return;", 2)
for dfn in mod.dfns:
self.visit(dfn)
self.emit("}", 0)
def visitProduct(self, prod, name):
self.addObj(name)
def visitSum(self, sum, name):
self.addObj(name)
for t in sum.types:
self.visitConstructor(t, name)
def visitConstructor(self, cons, name):
self.addObj(cons.name)
def addObj(self, name):
self.emit('if (PyDict_SetItemString(d, "%s", (PyObject*)%s_type) < 0) return;' % (name, name), 1)
_SPECIALIZED_SEQUENCES = ('stmt', 'expr')
def find_sequence(fields, doing_specialization):
"""Return True if any field uses a sequence."""
for f in fields:
if f.seq:
if not doing_specialization:
return True
if str(f.type) not in _SPECIALIZED_SEQUENCES:
return True
return False
def has_sequence(types, doing_specialization):
for t in types:
if find_sequence(t.fields, doing_specialization):
return True
return False
class StaticVisitor(PickleVisitor):
CODE = '''Very simple, always emit this static code. Override CODE'''
def visit(self, object):
self.emit(self.CODE, 0, reflow=False)
class ObjVisitor(PickleVisitor):
def func_begin(self, name):
ctype = get_c_type(name)
self.emit("PyObject*", 0)
self.emit("ast2obj_%s(void* _o)" % (name), 0)
self.emit("{", 0)
self.emit("%s o = (%s)_o;" % (ctype, ctype), 1)
self.emit("PyObject *result = NULL, *value = NULL;", 1)
self.emit('if (!o) {', 1)
self.emit("Py_INCREF(Py_None);", 2)
self.emit('return Py_None;', 2)
self.emit("}", 1)
self.emit('', 0)
def func_end(self):
self.emit("return result;", 1)
self.emit("failed:", 0)
self.emit("Py_XDECREF(value);", 1)
self.emit("Py_XDECREF(result);", 1)
self.emit("return NULL;", 1)
self.emit("}", 0)
self.emit("", 0)
def visitSum(self, sum, name):
if is_simple(sum):
self.simpleSum(sum, name)
return
self.func_begin(name)
self.emit("switch (o->kind) {", 1)
for i in range(len(sum.types)):
t = sum.types[i]
self.visitConstructor(t, i + 1, name)
self.emit("}", 1)
for a in sum.attributes:
self.emit("value = ast2obj_%s(o->%s);" % (a.type, a.name), 1)
self.emit("if (!value) goto failed;", 1)
self.emit('if (PyObject_SetAttrString(result, "%s", value) < 0)' % a.name, 1)
self.emit('goto failed;', 2)
self.emit('Py_DECREF(value);', 1)
self.func_end()
def simpleSum(self, sum, name):
self.emit("PyObject* ast2obj_%s(%s_ty o)" % (name, name), 0)
self.emit("{", 0)
self.emit("switch(o) {", 1)
for t in sum.types:
self.emit("case %s:" % t.name, 2)
self.emit("Py_INCREF(%s_singleton);" % t.name, 3)
self.emit("return %s_singleton;" % t.name, 3)
self.emit("default:", 2)
self.emit('/* should never happen, but just in case ... */', 3)
code = "PyErr_Format(PyExc_SystemError, \"unknown %s found\");" % name
self.emit(code, 3, reflow=False)
self.emit("return NULL;", 3)
self.emit("}", 1)
self.emit("}", 0)
def visitProduct(self, prod, name):
self.func_begin(name)
self.emit("result = PyType_GenericNew(%s_type, NULL, NULL);" % name, 1);
self.emit("if (!result) return NULL;", 1)
for field in prod.fields:
self.visitField(field, name, 1, True)
self.func_end()
def visitConstructor(self, cons, enum, name):
self.emit("case %s_kind:" % cons.name, 1)
self.emit("result = PyType_GenericNew(%s_type, NULL, NULL);" % cons.name, 2);
self.emit("if (!result) goto failed;", 2)
for f in cons.fields:
self.visitField(f, cons.name, 2, False)
self.emit("break;", 2)
def visitField(self, field, name, depth, product):
def emit(s, d):
self.emit(s, depth + d)
if product:
value = "o->%s" % field.name
else:
value = "o->v.%s.%s" % (name, field.name)
self.set(field, value, depth)
emit("if (!value) goto failed;", 0)
emit('if (PyObject_SetAttrString(result, "%s", value) == -1)' % field.name, 0)
emit("goto failed;", 1)
emit("Py_DECREF(value);", 0)
def emitSeq(self, field, value, depth, emit):
emit("seq = %s;" % value, 0)
emit("n = asdl_seq_LEN(seq);", 0)
emit("value = PyList_New(n);", 0)
emit("if (!value) goto failed;", 0)
emit("for (i = 0; i < n; i++) {", 0)
self.set("value", field, "asdl_seq_GET(seq, i)", depth + 1)
emit("if (!value1) goto failed;", 1)
emit("PyList_SET_ITEM(value, i, value1);", 1)
emit("value1 = NULL;", 1)
emit("}", 0)
def set(self, field, value, depth):
if field.seq:
# XXX should really check for is_simple, but that requires a symbol table
if field.type.value == "cmpop":
# While the sequence elements are stored as void*,
# ast2obj_cmpop expects an enum
self.emit("{", depth)
self.emit("int i, n = asdl_seq_LEN(%s);" % value, depth+1)
self.emit("value = PyList_New(n);", depth+1)
self.emit("if (!value) goto failed;", depth+1)
self.emit("for(i = 0; i < n; i++)", depth+1)
# This cannot fail, so no need for error handling
self.emit("PyList_SET_ITEM(value, i, ast2obj_cmpop((cmpop_ty)asdl_seq_GET(%s, i)));" % value,
depth+2, reflow=False)
self.emit("}", depth)
else:
self.emit("value = ast2obj_list(%s, ast2obj_%s);" % (value, field.type), depth)
else:
ctype = get_c_type(field.type)
self.emit("value = ast2obj_%s(%s);" % (field.type, value), depth, reflow=False)
class PartingShots(StaticVisitor):
CODE = """
PyObject* PyAST_mod2obj(mod_ty t)
{
init_types();
return ast2obj_mod(t);
}
/* mode is 0 for "exec", 1 for "eval" and 2 for "single" input */
mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode)
{
mod_ty res;
PyObject *req_type[3];
char *req_name[3];
int isinstance;
req_type[0] = (PyObject*)Module_type;
req_type[1] = (PyObject*)Expression_type;
req_type[2] = (PyObject*)Interactive_type;
req_name[0] = "Module";
req_name[1] = "Expression";
req_name[2] = "Interactive";
assert(0 <= mode && mode <= 2);
init_types();
isinstance = PyObject_IsInstance(ast, req_type[mode]);
if (isinstance == -1)
return NULL;
if (!isinstance) {
PyErr_Format(PyExc_TypeError, "expected %s node, got %.400s",
req_name[mode], Py_TYPE(ast)->tp_name);
return NULL;
}
if (obj2ast_mod(ast, &res, arena) != 0)
return NULL;
else
return res;
}
int PyAST_Check(PyObject* obj)
{
init_types();
return PyObject_IsInstance(obj, (PyObject*)&AST_type);
}
"""
class ChainOfVisitors:
def __init__(self, *visitors):
self.visitors = visitors
def visit(self, object):
for v in self.visitors:
v.visit(object)
v.emit("", 0)
common_msg = "/* File automatically generated by %s. */\n\n"
c_file_msg = """
/*
__version__ %s.
This module must be committed separately after each AST grammar change;
The __version__ number is set to the revision number of the commit
containing the grammar change.
*/
"""
def main(srcfile):
argv0 = sys.argv[0]
components = argv0.split(os.sep)
argv0 = os.sep.join(components[-2:])
auto_gen_msg = common_msg % argv0
mod = asdl.parse(srcfile)
mod.version = "82160"
if not asdl.check(mod):
sys.exit(1)
if INC_DIR:
p = "%s/%s-ast.h" % (INC_DIR, mod.name)
f = open(p, "wb")
f.write(auto_gen_msg)
f.write('#include "asdl.h"\n\n')
c = ChainOfVisitors(TypeDefVisitor(f),
StructVisitor(f),
PrototypeVisitor(f),
)
c.visit(mod)
f.write("PyObject* PyAST_mod2obj(mod_ty t);\n")
f.write("mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode);\n")
f.write("int PyAST_Check(PyObject* obj);\n")
f.close()
if SRC_DIR:
p = os.path.join(SRC_DIR, str(mod.name) + "-ast.c")
f = open(p, "wb")
f.write(auto_gen_msg)
f.write(c_file_msg % mod.version)
f.write('#include "Python.h"\n')
f.write('#include "%s-ast.h"\n' % mod.name)
f.write('\n')
f.write("static PyTypeObject AST_type;\n")
v = ChainOfVisitors(
PyTypesDeclareVisitor(f),
PyTypesVisitor(f),
Obj2ModPrototypeVisitor(f),
FunctionVisitor(f),
ObjVisitor(f),
Obj2ModVisitor(f),
ASTModuleVisitor(f),
PartingShots(f),
)
v.visit(mod)
f.close()
if __name__ == "__main__":
import sys
import getopt
INC_DIR = ''
SRC_DIR = ''
opts, args = getopt.getopt(sys.argv[1:], "h:c:")
if len(opts) != 1:
print "Must specify exactly one output file"
sys.exit(1)
for o, v in opts:
if o == '-h':
INC_DIR = v
if o == '-c':
SRC_DIR = v
if len(args) != 1:
print "Must specify single input file"
sys.exit(1)
main(args[0])
from_cpython/Parser/bitset.c 0 → 100644
View file @ 3fa45d83
/* Bitset primitives used by the parser generator */
#include "pgenheaders.h"
#include "bitset.h"
bitset
newbitset(int nbits)
{
int nbytes = NBYTES(nbits);
bitset ss = (char *)PyObject_MALLOC(sizeof(BYTE) * nbytes);
if (ss == NULL)
Py_FatalError("no mem for bitset");
ss += nbytes;
while (--nbytes >= 0)
*--ss = 0;
return ss;
}
void
delbitset(bitset ss)
{
PyObject_FREE(ss);
}
int
addbit(bitset ss, int ibit)
{
int ibyte = BIT2BYTE(ibit);
BYTE mask = BIT2MASK(ibit);
if (ss[ibyte] & mask)
return 0; /* Bit already set */
ss[ibyte] |= mask;
return 1;
}
#if 0 /* Now a macro */
int
testbit(bitset ss, int ibit)
{
return (ss[BIT2BYTE(ibit)] & BIT2MASK(ibit)) != 0;
}
#endif
int
samebitset(bitset ss1, bitset ss2, int nbits)
{
int i;
for (i = NBYTES(nbits); --i >= 0; )
if (*ss1++ != *ss2++)
return 0;
return 1;
}
void
mergebitset(bitset ss1, bitset ss2, int nbits)
{
int i;
for (i = NBYTES(nbits); --i >= 0; )
*ss1++ |= *ss2++;
}
from_cpython/Parser/firstsets.c 0 → 100644
View file @ 3fa45d83
/* Computation of FIRST stets */
#include "pgenheaders.h"
#include "grammar.h"
#include "token.h"
extern int Py_DebugFlag;
/* Forward */
static void calcfirstset(grammar *, dfa *);
void
addfirstsets(grammar *g)
{
int i;
dfa *d;
if (Py_DebugFlag)
printf("Adding FIRST sets ...\n");
for (i = 0; i < g->g_ndfas; i++) {
d = &g->g_dfa[i];
if (d->d_first == NULL)
calcfirstset(g, d);
}
}
static void
calcfirstset(grammar *g, dfa *d)
{
int i, j;
state *s;
arc *a;
int nsyms;
int *sym;
int nbits;
static bitset dummy;
bitset result;
int type;
dfa *d1;
label *l0;
if (Py_DebugFlag)
printf("Calculate FIRST set for '%s'\n", d->d_name);
if (dummy == NULL)
dummy = newbitset(1);
if (d->d_first == dummy) {
fprintf(stderr, "Left-recursion for '%s'\n", d->d_name);
return;
}
if (d->d_first != NULL) {
fprintf(stderr, "Re-calculating FIRST set for '%s' ???\n",
d->d_name);
}
d->d_first = dummy;
l0 = g->g_ll.ll_label;
nbits = g->g_ll.ll_nlabels;
result = newbitset(nbits);
sym = (int *)PyObject_MALLOC(sizeof(int));
if (sym == NULL)
Py_FatalError("no mem for new sym in calcfirstset");
nsyms = 1;
sym[0] = findlabel(&g->g_ll, d->d_type, (char *)NULL);
s = &d->d_state[d->d_initial];
for (i = 0; i < s->s_narcs; i++) {
a = &s->s_arc[i];
for (j = 0; j < nsyms; j++) {
if (sym[j] == a->a_lbl)
break;
}
if (j >= nsyms) { /* New label */
sym = (int *)PyObject_REALLOC(sym,
sizeof(int) * (nsyms + 1));
if (sym == NULL)
Py_FatalError(
"no mem to resize sym in calcfirstset");
sym[nsyms++] = a->a_lbl;
type = l0[a->a_lbl].lb_type;
if (ISNONTERMINAL(type)) {
d1 = PyGrammar_FindDFA(g, type);
if (d1->d_first == dummy) {
fprintf(stderr,
"Left-recursion below '%s'\n",
d->d_name);
}
else {
if (d1->d_first == NULL)
calcfirstset(g, d1);
mergebitset(result,
d1->d_first, nbits);
}
}
else if (ISTERMINAL(type)) {
addbit(result, a->a_lbl);
}
}
}
d->d_first = result;
if (Py_DebugFlag) {
printf("FIRST set for '%s': {", d->d_name);
for (i = 0; i < nbits; i++) {
if (testbit(result, i))
printf(" %s", PyGrammar_LabelRepr(&l0[i]));
}
printf(" }\n");
}
PyObject_FREE(sym);
}
from_cpython/Parser/grammar.c 0 → 100644
View file @ 3fa45d83
/* Grammar implementation */
#include "Python.h"
#include "pgenheaders.h"
#include <ctype.h>
#include "token.h"
#include "grammar.h"
#ifdef RISCOS
#include <unixlib.h>
#endif
extern int Py_DebugFlag;
grammar *
newgrammar(int start)
{
grammar *g;
g = (grammar *)PyObject_MALLOC(sizeof(grammar));
if (g == NULL)
Py_FatalError("no mem for new grammar");
g->g_ndfas = 0;
g->g_dfa = NULL;
g->g_start = start;
g->g_ll.ll_nlabels = 0;
g->g_ll.ll_label = NULL;
g->g_accel = 0;
return g;
}
dfa *
adddfa(grammar *g, int type, char *name)
{
dfa *d;
g->g_dfa = (dfa *)PyObject_REALLOC(g->g_dfa,
sizeof(dfa) * (g->g_ndfas + 1));
if (g->g_dfa == NULL)
Py_FatalError("no mem to resize dfa in adddfa");
d = &g->g_dfa[g->g_ndfas++];
d->d_type = type;
d->d_name = strdup(name);
d->d_nstates = 0;
d->d_state = NULL;
d->d_initial = -1;
d->d_first = NULL;
return d; /* Only use while fresh! */
}
int
addstate(dfa *d)
{
state *s;
d->d_state = (state *)PyObject_REALLOC(d->d_state,
sizeof(state) * (d->d_nstates + 1));
if (d->d_state == NULL)
Py_FatalError("no mem to resize state in addstate");
s = &d->d_state[d->d_nstates++];
s->s_narcs = 0;
s->s_arc = NULL;
s->s_lower = 0;
s->s_upper = 0;
s->s_accel = NULL;
s->s_accept = 0;
return s - d->d_state;
}
void
addarc(dfa *d, int from, int to, int lbl)
{
state *s;
arc *a;
assert(0 <= from && from < d->d_nstates);
assert(0 <= to && to < d->d_nstates);
s = &d->d_state[from];
s->s_arc = (arc *)PyObject_REALLOC(s->s_arc, sizeof(arc) * (s->s_narcs + 1));
if (s->s_arc == NULL)
Py_FatalError("no mem to resize arc list in addarc");
a = &s->s_arc[s->s_narcs++];
a->a_lbl = lbl;
a->a_arrow = to;
}
int
addlabel(labellist *ll, int type, char *str)
{
int i;
label *lb;
for (i = 0; i < ll->ll_nlabels; i++) {
if (ll->ll_label[i].lb_type == type &&
strcmp(ll->ll_label[i].lb_str, str) == 0)
return i;
}
ll->ll_label = (label *)PyObject_REALLOC(ll->ll_label,
sizeof(label) * (ll->ll_nlabels + 1));
if (ll->ll_label == NULL)
Py_FatalError("no mem to resize labellist in addlabel");
lb = &ll->ll_label[ll->ll_nlabels++];
lb->lb_type = type;
lb->lb_str = strdup(str);
if (Py_DebugFlag)
printf("Label @ %8p, %d: %s\n", ll, ll->ll_nlabels,
PyGrammar_LabelRepr(lb));
return lb - ll->ll_label;
}
/* Same, but rather dies than adds */
int
findlabel(labellist *ll, int type, char *str)
{
int i;
for (i = 0; i < ll->ll_nlabels; i++) {
if (ll->ll_label[i].lb_type == type /*&&
strcmp(ll->ll_label[i].lb_str, str) == 0*/)
return i;
}
fprintf(stderr, "Label %d/'%s' not found\n", type, str);
Py_FatalError("grammar.c:findlabel()");
return 0; /* Make gcc -Wall happy */
}
/* Forward */
static void translabel(grammar *, label *);
void
translatelabels(grammar *g)
{
int i;
#ifdef Py_DEBUG
printf("Translating labels ...\n");
#endif
/* Don't translate EMPTY */
for (i = EMPTY+1; i < g->g_ll.ll_nlabels; i++)
translabel(g, &g->g_ll.ll_label[i]);
}
static void
translabel(grammar *g, label *lb)
{
int i;
if (Py_DebugFlag)
printf("Translating label %s ...\n", PyGrammar_LabelRepr(lb));
if (lb->lb_type == NAME) {
for (i = 0; i < g->g_ndfas; i++) {
if (strcmp(lb->lb_str, g->g_dfa[i].d_name) == 0) {
if (Py_DebugFlag)
printf(
"Label %s is non-terminal %d.\n",
lb->lb_str,
g->g_dfa[i].d_type);
lb->lb_type = g->g_dfa[i].d_type;
free(lb->lb_str);
lb->lb_str = NULL;
return;
}
}
for (i = 0; i < (int)N_TOKENS; i++) {
if (strcmp(lb->lb_str, _PyParser_TokenNames[i]) == 0) {
if (Py_DebugFlag)
printf("Label %s is terminal %d.\n",
lb->lb_str, i);
lb->lb_type = i;
free(lb->lb_str);
lb->lb_str = NULL;
return;
}
}
printf("Can't translate NAME label '%s'\n", lb->lb_str);
return;
}
if (lb->lb_type == STRING) {
if (isalpha(Py_CHARMASK(lb->lb_str[1])) ||
lb->lb_str[1] == '_') {
char *p;
char *src;
char *dest;
size_t name_len;
if (Py_DebugFlag)
printf("Label %s is a keyword\n", lb->lb_str);
lb->lb_type = NAME;
src = lb->lb_str + 1;
p = strchr(src, '\'');
if (p)
name_len = p - src;
else
name_len = strlen(src);
dest = (char *)malloc(name_len + 1);
if (!dest) {
printf("Can't alloc dest '%s'\n", src);
return;
}
strncpy(dest, src, name_len);
dest[name_len] = '\0';
free(lb->lb_str);
lb->lb_str = dest;
}
else if (lb->lb_str[2] == lb->lb_str[0]) {
int type = (int) PyToken_OneChar(lb->lb_str[1]);
if (type != OP) {
lb->lb_type = type;
free(lb->lb_str);
lb->lb_str = NULL;
}
else
printf("Unknown OP label %s\n",
lb->lb_str);
}
else if (lb->lb_str[2] && lb->lb_str[3] == lb->lb_str[0]) {
int type = (int) PyToken_TwoChars(lb->lb_str[1],
lb->lb_str[2]);
if (type != OP) {
lb->lb_type = type;
free(lb->lb_str);
lb->lb_str = NULL;
}
else
printf("Unknown OP label %s\n",
lb->lb_str);
}
else if (lb->lb_str[2] && lb->lb_str[3] && lb->lb_str[4] == lb->lb_str[0]) {
int type = (int) PyToken_ThreeChars(lb->lb_str[1],
lb->lb_str[2],
lb->lb_str[3]);
if (type != OP) {
lb->lb_type = type;
free(lb->lb_str);
lb->lb_str = NULL;
}
else
printf("Unknown OP label %s\n",
lb->lb_str);
}
else
printf("Can't translate STRING label %s\n",
lb->lb_str);
}
else
printf("Can't translate label '%s'\n",
PyGrammar_LabelRepr(lb));
}
from_cpython/Parser/grammar1.c 0 → 100644
View file @ 3fa45d83
/* Grammar subroutines needed by parser */
#include "Python.h"
#include "pgenheaders.h"
#include "grammar.h"
#include "token.h"
/* Return the DFA for the given type */
dfa *
PyGrammar_FindDFA(grammar *g, register int type)
{
register dfa *d;
#if 1
/* Massive speed-up */
d = &g->g_dfa[type - NT_OFFSET];
assert(d->d_type == type);
return d;
#else
/* Old, slow version */
register int i;
for (i = g->g_ndfas, d = g->g_dfa; --i >= 0; d++) {
if (d->d_type == type)
return d;
}
assert(0);
/* NOTREACHED */
#endif
}
char *
PyGrammar_LabelRepr(label *lb)
{
static char buf[100];
if (lb->lb_type == ENDMARKER)
return "EMPTY";
else if (ISNONTERMINAL(lb->lb_type)) {
if (lb->lb_str == NULL) {
PyOS_snprintf(buf, sizeof(buf), "NT%d", lb->lb_type);
return buf;
}
else
return lb->lb_str;
}
else {
if (lb->lb_str == NULL)
return _PyParser_TokenNames[lb->lb_type];
else {
PyOS_snprintf(buf, sizeof(buf), "%.32s(%.32s)",
_PyParser_TokenNames[lb->lb_type], lb->lb_str);
return buf;
}
}
}
from_cpython/Parser/intrcheck.c 0 → 100644
View file @ 3fa45d83
/* Check for interrupts */
#include "Python.h"
#include "pythread.h"
#ifdef QUICKWIN
#include <io.h>
void
PyOS_InitInterrupts(void)
{
}
void
PyOS_FiniInterrupts(void)
{
}
int
PyOS_InterruptOccurred(void)
{
_wyield();
}
#define OK
#endif /* QUICKWIN */
#if defined(_M_IX86) && !defined(__QNX__)
#include <io.h>
#endif
#if defined(MSDOS) && !defined(QUICKWIN)
#ifdef __GNUC__
/* This is for DJGPP's GO32 extender. I don't know how to trap
* control-C (There's no API for ctrl-C, and I don't want to mess with
* the interrupt vectors.) However, this DOES catch control-break.
* --Amrit
*/
#include <go32.h>
void
PyOS_InitInterrupts(void)
{
_go32_want_ctrl_break(1 /* TRUE */);
}
void
PyOS_FiniInterrupts(void)
{
}
int
PyOS_InterruptOccurred(void)
{
return _go32_was_ctrl_break_hit();
}
#else /* !__GNUC__ */
/* This might work for MS-DOS (untested though): */
void
PyOS_InitInterrupts(void)
{
}
void
PyOS_FiniInterrupts(void)
{
}
int
PyOS_InterruptOccurred(void)
{
int interrupted = 0;
while (kbhit()) {
if (getch() == '\003')
interrupted = 1;
}
return interrupted;
}
#endif /* __GNUC__ */
#define OK
#endif /* MSDOS && !QUICKWIN */
#ifndef OK
/* Default version -- for real operating systems and for Standard C */
#include <stdio.h>
#include <string.h>
#include <signal.h>
static int interrupted;
void
PyErr_SetInterrupt(void)
{
interrupted = 1;
}
extern int PyErr_CheckSignals(void);
static int
checksignals_witharg(void * arg)
{
return PyErr_CheckSignals();
}
static void
intcatcher(int sig)
{
extern void Py_Exit(int);
static char message[] =
"python: to interrupt a truly hanging Python program, interrupt once more.\n";
switch (interrupted++) {
case 0:
break;
case 1:
#ifdef RISCOS
fprintf(stderr, message);
#else
write(2, message, strlen(message));
#endif
break;
case 2:
interrupted = 0;
Py_Exit(1);
break;
}
PyOS_setsig(SIGINT, intcatcher);
Py_AddPendingCall(checksignals_witharg, NULL);
}
static void (*old_siginthandler)(int) = SIG_DFL;
void
PyOS_InitInterrupts(void)
{
if ((old_siginthandler = PyOS_setsig(SIGINT, SIG_IGN)) != SIG_IGN)
PyOS_setsig(SIGINT, intcatcher);
}
void
PyOS_FiniInterrupts(void)
{
PyOS_setsig(SIGINT, old_siginthandler);
}
int
PyOS_InterruptOccurred(void)
{
if (!interrupted)
return 0;
interrupted = 0;
return 1;
}
#endif /* !OK */
void
PyOS_AfterFork(void)
{
#ifdef WITH_THREAD
PyEval_ReInitThreads();
PyThread_ReInitTLS();
#endif
}
from_cpython/Parser/listnode.c 0 → 100644
View file @ 3fa45d83
/* List a node on a file */
#include "pgenheaders.h"
#include "token.h"
#include "node.h"
/* Forward */
static void list1node(FILE *, node *);
static void listnode(FILE *, node *);
void
PyNode_ListTree(node *n)
{
listnode(stdout, n);
}
static int level, atbol;
static void
listnode(FILE *fp, node *n)
{
level = 0;
atbol = 1;
list1node(fp, n);
}
static void
list1node(FILE *fp, node *n)
{
if (n == 0)
return;
if (ISNONTERMINAL(TYPE(n))) {
int i;
for (i = 0; i < NCH(n); i++)
list1node(fp, CHILD(n, i));
}
else if (ISTERMINAL(TYPE(n))) {
switch (TYPE(n)) {
case INDENT:
++level;
break;
case DEDENT:
--level;
break;
default:
if (atbol) {
int i;
for (i = 0; i < level; ++i)
fprintf(fp, "\t");
atbol = 0;
}
if (TYPE(n) == NEWLINE) {
if (STR(n) != NULL)
fprintf(fp, "%s", STR(n));
fprintf(fp, "\n");
atbol = 1;
}
else
fprintf(fp, "%s ", STR(n));
break;
}
}
else
fprintf(fp, "? ");
}
from_cpython/Parser/metagrammar.c 0 → 100644
View file @ 3fa45d83
#include "pgenheaders.h"
#include "metagrammar.h"
#include "grammar.h"
#include "pgen.h"
static arc arcs_0_0[3] = {
{2, 0},
{3, 0},
{4, 1},
};
static arc arcs_0_1[1] = {
{0, 1},
};
static state states_0[2] = {
{3, arcs_0_0},
{1, arcs_0_1},
};
static arc arcs_1_0[1] = {
{5, 1},
};
static arc arcs_1_1[1] = {
{6, 2},
};
static arc arcs_1_2[1] = {
{7, 3},
};
static arc arcs_1_3[1] = {
{3, 4},
};
static arc arcs_1_4[1] = {
{0, 4},
};
static state states_1[5] = {
{1, arcs_1_0},
{1, arcs_1_1},
{1, arcs_1_2},
{1, arcs_1_3},
{1, arcs_1_4},
};
static arc arcs_2_0[1] = {
{8, 1},
};
static arc arcs_2_1[2] = {
{9, 0},
{0, 1},
};
static state states_2[2] = {
{1, arcs_2_0},
{2, arcs_2_1},
};
static arc arcs_3_0[1] = {
{10, 1},
};
static arc arcs_3_1[2] = {
{10, 1},
{0, 1},
};
static state states_3[2] = {
{1, arcs_3_0},
{2, arcs_3_1},
};
static arc arcs_4_0[2] = {
{11, 1},
{13, 2},
};
static arc arcs_4_1[1] = {
{7, 3},
};
static arc arcs_4_2[3] = {
{14, 4},
{15, 4},
{0, 2},
};
static arc arcs_4_3[1] = {
{12, 4},
};
static arc arcs_4_4[1] = {
{0, 4},
};
static state states_4[5] = {
{2, arcs_4_0},
{1, arcs_4_1},
{3, arcs_4_2},
{1, arcs_4_3},
{1, arcs_4_4},
};
static arc arcs_5_0[3] = {
{5, 1},
{16, 1},
{17, 2},
};
static arc arcs_5_1[1] = {
{0, 1},
};
static arc arcs_5_2[1] = {
{7, 3},
};
static arc arcs_5_3[1] = {
{18, 1},
};
static state states_5[4] = {
{3, arcs_5_0},
{1, arcs_5_1},
{1, arcs_5_2},
{1, arcs_5_3},
};
static dfa dfas[6] = {
{256, "MSTART", 0, 2, states_0,
"\070\000\000"},
{257, "RULE", 0, 5, states_1,
"\040\000\000"},
{258, "RHS", 0, 2, states_2,
"\040\010\003"},
{259, "ALT", 0, 2, states_3,
"\040\010\003"},
{260, "ITEM", 0, 5, states_4,
"\040\010\003"},
{261, "ATOM", 0, 4, states_5,
"\040\000\003"},
};
static label labels[19] = {
{0, "EMPTY"},
{256, 0},
{257, 0},
{4, 0},
{0, 0},
{1, 0},
{11, 0},
{258, 0},
{259, 0},
{18, 0},
{260, 0},
{9, 0},
{10, 0},
{261, 0},
{16, 0},
{14, 0},
{3, 0},
{7, 0},
{8, 0},
};
static grammar _PyParser_Grammar = {
6,
dfas,
{19, labels},
256
};
grammar *
meta_grammar(void)
{
return &_PyParser_Grammar;
}
grammar *
Py_meta_grammar(void)
{
return meta_grammar();
}
from_cpython/Parser/node.c 0 → 100644
View file @ 3fa45d83
/* Parse tree node implementation */
#include "Python.h"
#include "node.h"
#include "errcode.h"
node *
PyNode_New(int type)
{
node *n = (node *) PyObject_MALLOC(1 * sizeof(node));
if (n == NULL)
return NULL;
n->n_type = type;
n->n_str = NULL;
n->n_lineno = 0;
n->n_nchildren = 0;
n->n_child = NULL;
return n;
}
/* See comments at XXXROUNDUP below. Returns -1 on overflow. */
static int
fancy_roundup(int n)
{
/* Round up to the closest power of 2 >= n. */
int result = 256;
assert(n > 128);
while (result < n) {
result <<= 1;
if (result <= 0)
return -1;
}
return result;
}
/* A gimmick to make massive numbers of reallocs quicker. The result is
* a number >= the input. In PyNode_AddChild, it's used like so, when
* we're about to add child number current_size + 1:
*
* if XXXROUNDUP(current_size) < XXXROUNDUP(current_size + 1):
* allocate space for XXXROUNDUP(current_size + 1) total children
* else:
* we already have enough space
*
* Since a node starts out empty, we must have
*
* XXXROUNDUP(0) < XXXROUNDUP(1)
*
* so that we allocate space for the first child. One-child nodes are very
* common (presumably that would change if we used a more abstract form
* of syntax tree), so to avoid wasting memory it's desirable that
* XXXROUNDUP(1) == 1. That in turn forces XXXROUNDUP(0) == 0.
*
* Else for 2 <= n <= 128, we round up to the closest multiple of 4. Why 4?
* Rounding up to a multiple of an exact power of 2 is very efficient, and
* most nodes with more than one child have <= 4 kids.
*
* Else we call fancy_roundup() to grow proportionately to n. We've got an
* extreme case then (like test_longexp.py), and on many platforms doing
* anything less than proportional growth leads to exorbitant runtime
* (e.g., MacPython), or extreme fragmentation of user address space (e.g.,
* Win98).
*
* In a run of compileall across the 2.3a0 Lib directory, Andrew MacIntyre
* reported that, with this scheme, 89% of PyObject_REALLOC calls in
* PyNode_AddChild passed 1 for the size, and 9% passed 4. So this usually
* wastes very little memory, but is very effective at sidestepping
* platform-realloc disasters on vulnerable platforms.
*
* Note that this would be straightforward if a node stored its current
* capacity. The code is tricky to avoid that.
*/
#define XXXROUNDUP(n) ((n) <= 1 ? (n) : \
(n) <= 128 ? (((n) + 3) & ~3) : \
fancy_roundup(n))
int
PyNode_AddChild(register node *n1, int type, char *str, int lineno, int col_offset)
{
const int nch = n1->n_nchildren;
int current_capacity;
int required_capacity;
node *n;
if (nch == INT_MAX || nch < 0)
return E_OVERFLOW;
current_capacity = XXXROUNDUP(nch);
required_capacity = XXXROUNDUP(nch + 1);
if (current_capacity < 0 || required_capacity < 0)
return E_OVERFLOW;
if (current_capacity < required_capacity) {
if (required_capacity > PY_SIZE_MAX / sizeof(node)) {
return E_NOMEM;
}
n = n1->n_child;
n = (node *) PyObject_REALLOC(n,
required_capacity * sizeof(node));
if (n == NULL)
return E_NOMEM;
n1->n_child = n;
}
n = &n1->n_child[n1->n_nchildren++];
n->n_type = type;
n->n_str = str;
n->n_lineno = lineno;
n->n_col_offset = col_offset;
n->n_nchildren = 0;
n->n_child = NULL;
return 0;
}
/* Forward */
static void freechildren(node *);
static Py_ssize_t sizeofchildren(node *n);
void
PyNode_Free(node *n)
{
if (n != NULL) {
freechildren(n);
PyObject_FREE(n);
}
}
Py_ssize_t
_PyNode_SizeOf(node *n)
{
Py_ssize_t res = 0;
if (n != NULL)
res = sizeof(node) + sizeofchildren(n);
return res;
}
static void
freechildren(node *n)
{
int i;
for (i = NCH(n); --i >= 0; )
freechildren(CHILD(n, i));
if (n->n_child != NULL)
PyObject_FREE(n->n_child);
if (STR(n) != NULL)
PyObject_FREE(STR(n));
}
static Py_ssize_t
sizeofchildren(node *n)
{
Py_ssize_t res = 0;
int i;
for (i = NCH(n); --i >= 0; )
res += sizeofchildren(CHILD(n, i));
if (n->n_child != NULL)
/* allocated size of n->n_child array */
res += XXXROUNDUP(NCH(n)) * sizeof(node);
if (STR(n) != NULL)
res += strlen(STR(n)) + 1;
return res;
}
from_cpython/Parser/parser.c 0 → 100644
View file @ 3fa45d83
/* Parser implementation */
/* For a description, see the comments at end of this file */
/* XXX To do: error recovery */
#include "Python.h"
#include "pgenheaders.h"
#include "token.h"
#include "grammar.h"
#include "node.h"
#include "parser.h"
#include "errcode.h"
#ifdef Py_DEBUG
extern int Py_DebugFlag;
#define D(x) if (!Py_DebugFlag); else x
#else
#define D(x)
#endif
/* STACK DATA TYPE */
static void s_reset(stack *);
static void
s_reset(stack *s)
{
s->s_top = &s->s_base[MAXSTACK];
}
#define s_empty(s) ((s)->s_top == &(s)->s_base[MAXSTACK])
static int
s_push(register stack *s, dfa *d, node *parent)
{
register stackentry *top;
if (s->s_top == s->s_base) {
fprintf(stderr, "s_push: parser stack overflow\n");
return E_NOMEM;
}
top = --s->s_top;
top->s_dfa = d;
top->s_parent = parent;
top->s_state = 0;
return 0;
}
#ifdef Py_DEBUG
static void
s_pop(register stack *s)
{
if (s_empty(s))
Py_FatalError("s_pop: parser stack underflow -- FATAL");
s->s_top++;
}
#else /* !Py_DEBUG */
#define s_pop(s) (s)->s_top++
#endif
/* PARSER CREATION */
parser_state *
PyParser_New(grammar *g, int start)
{
parser_state *ps;
if (!g->g_accel)
PyGrammar_AddAccelerators(g);
ps = (parser_state *)PyMem_MALLOC(sizeof(parser_state));
if (ps == NULL)
return NULL;
ps->p_grammar = g;
#ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
ps->p_flags = 0;
#endif
ps->p_tree = PyNode_New(start);
if (ps->p_tree == NULL) {
PyMem_FREE(ps);
return NULL;
}
s_reset(&ps->p_stack);
(void) s_push(&ps->p_stack, PyGrammar_FindDFA(g, start), ps->p_tree);
return ps;
}
void
PyParser_Delete(parser_state *ps)
{
/* NB If you want to save the parse tree,
you must set p_tree to NULL before calling delparser! */
PyNode_Free(ps->p_tree);
PyMem_FREE(ps);
}
/* PARSER STACK OPERATIONS */
static int
shift(register stack *s, int type, char *str, int newstate, int lineno, int col_offset)
{
int err;
assert(!s_empty(s));
err = PyNode_AddChild(s->s_top->s_parent, type, str, lineno, col_offset);
if (err)
return err;
s->s_top->s_state = newstate;
return 0;
}
static int
push(register stack *s, int type, dfa *d, int newstate, int lineno, int col_offset)
{
int err;
register node *n;
n = s->s_top->s_parent;
assert(!s_empty(s));
err = PyNode_AddChild(n, type, (char *)NULL, lineno, col_offset);
if (err)
return err;
s->s_top->s_state = newstate;
return s_push(s, d, CHILD(n, NCH(n)-1));
}
/* PARSER PROPER */
static int
classify(parser_state *ps, int type, char *str)
{
grammar *g = ps->p_grammar;
register int n = g->g_ll.ll_nlabels;
if (type == NAME) {
register char *s = str;
register label *l = g->g_ll.ll_label;
register int i;
for (i = n; i > 0; i--, l++) {
if (l->lb_type != NAME || l->lb_str == NULL ||
l->lb_str[0] != s[0] ||
strcmp(l->lb_str, s) != 0)
continue;
#ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
if (ps->p_flags & CO_FUTURE_PRINT_FUNCTION &&
s[0] == 'p' && strcmp(s, "print") == 0) {
break; /* no longer a keyword */
}
#endif
D(printf("It's a keyword\n"));
return n - i;
}
}
{
register label *l = g->g_ll.ll_label;
register int i;
for (i = n; i > 0; i--, l++) {
if (l->lb_type == type && l->lb_str == NULL) {
D(printf("It's a token we know\n"));
return n - i;
}
}
}
D(printf("Illegal token\n"));
return -1;
}
#ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
static void
future_hack(parser_state *ps)
{
node *n = ps->p_stack.s_top->s_parent;
node *ch, *cch;
int i;
/* from __future__ import ..., must have at least 4 children */
n = CHILD(n, 0);
if (NCH(n) < 4)
return;
ch = CHILD(n, 0);
if (STR(ch) == NULL || strcmp(STR(ch), "from") != 0)
return;
ch = CHILD(n, 1);
if (NCH(ch) == 1 && STR(CHILD(ch, 0)) &&
strcmp(STR(CHILD(ch, 0)), "__future__") != 0)
return;
ch = CHILD(n, 3);
/* ch can be a star, a parenthesis or import_as_names */
if (TYPE(ch) == STAR)
return;
if (TYPE(ch) == LPAR)
ch = CHILD(n, 4);
for (i = 0; i < NCH(ch); i += 2) {
cch = CHILD(ch, i);
if (NCH(cch) >= 1 && TYPE(CHILD(cch, 0)) == NAME) {
char *str_ch = STR(CHILD(cch, 0));
if (strcmp(str_ch, FUTURE_WITH_STATEMENT) == 0) {
ps->p_flags |= CO_FUTURE_WITH_STATEMENT;
} else if (strcmp(str_ch, FUTURE_PRINT_FUNCTION) == 0) {
ps->p_flags |= CO_FUTURE_PRINT_FUNCTION;
} else if (strcmp(str_ch, FUTURE_UNICODE_LITERALS) == 0) {
ps->p_flags |= CO_FUTURE_UNICODE_LITERALS;
}
}
}
}
#endif /* future keyword */
int
PyParser_AddToken(register parser_state *ps, register int type, char *str,
int lineno, int col_offset, int *expected_ret)
{
register int ilabel;
int err;
D(printf("Token %s/'%s' ... ", _PyParser_TokenNames[type], str));
/* Find out which label this token is */
ilabel = classify(ps, type, str);
if (ilabel < 0)
return E_SYNTAX;
/* Loop until the token is shifted or an error occurred */
for (;;) {
/* Fetch the current dfa and state */
register dfa *d = ps->p_stack.s_top->s_dfa;
register state *s = &d->d_state[ps->p_stack.s_top->s_state];
D(printf(" DFA '%s', state %d:",
d->d_name, ps->p_stack.s_top->s_state));
/* Check accelerator */
if (s->s_lower <= ilabel && ilabel < s->s_upper) {
register int x = s->s_accel[ilabel - s->s_lower];
if (x != -1) {
if (x & (1<<7)) {
/* Push non-terminal */
int nt = (x >> 8) + NT_OFFSET;
int arrow = x & ((1<<7)-1);
dfa *d1 = PyGrammar_FindDFA(
ps->p_grammar, nt);
if ((err = push(&ps->p_stack, nt, d1,
arrow, lineno, col_offset)) > 0) {
D(printf(" MemError: push\n"));
return err;
}
D(printf(" Push ...\n"));
continue;
}
/* Shift the token */
if ((err = shift(&ps->p_stack, type, str,
x, lineno, col_offset)) > 0) {
D(printf(" MemError: shift.\n"));
return err;
}
D(printf(" Shift.\n"));
/* Pop while we are in an accept-only state */
while (s = &d->d_state
[ps->p_stack.s_top->s_state],
s->s_accept && s->s_narcs == 1) {
D(printf(" DFA '%s', state %d: "
"Direct pop.\n",
d->d_name,
ps->p_stack.s_top->s_state));
#ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
if (d->d_name[0] == 'i' &&
strcmp(d->d_name,
"import_stmt") == 0)
future_hack(ps);
#endif
s_pop(&ps->p_stack);
if (s_empty(&ps->p_stack)) {
D(printf(" ACCEPT.\n"));
return E_DONE;
}
d = ps->p_stack.s_top->s_dfa;
}
return E_OK;
}
}
if (s->s_accept) {
#ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
if (d->d_name[0] == 'i' &&
strcmp(d->d_name, "import_stmt") == 0)
future_hack(ps);
#endif
/* Pop this dfa and try again */
s_pop(&ps->p_stack);
D(printf(" Pop ...\n"));
if (s_empty(&ps->p_stack)) {
D(printf(" Error: bottom of stack.\n"));
return E_SYNTAX;
}
continue;
}
/* Stuck, report syntax error */
D(printf(" Error.\n"));
if (expected_ret) {
if (s->s_lower == s->s_upper - 1) {
/* Only one possible expected token */
*expected_ret = ps->p_grammar->
g_ll.ll_label[s->s_lower].lb_type;
}
else
*expected_ret = -1;
}
return E_SYNTAX;
}
}
#ifdef Py_DEBUG
/* DEBUG OUTPUT */
void
dumptree(grammar *g, node *n)
{
int i;
if (n == NULL)
printf("NIL");
else {
label l;
l.lb_type = TYPE(n);
l.lb_str = STR(n);
printf("%s", PyGrammar_LabelRepr(&l));
if (ISNONTERMINAL(TYPE(n))) {
printf("(");
for (i = 0; i < NCH(n); i++) {
if (i > 0)
printf(",");
dumptree(g, CHILD(n, i));
}
printf(")");
}
}
}
void
showtree(grammar *g, node *n)
{
int i;
if (n == NULL)
return;
if (ISNONTERMINAL(TYPE(n))) {
for (i = 0; i < NCH(n); i++)
showtree(g, CHILD(n, i));
}
else if (ISTERMINAL(TYPE(n))) {
printf("%s", _PyParser_TokenNames[TYPE(n)]);
if (TYPE(n) == NUMBER || TYPE(n) == NAME)
printf("(%s)", STR(n));
printf(" ");
}
else
printf("? ");
}
void
printtree(parser_state *ps)
{
if (Py_DebugFlag) {
printf("Parse tree:\n");
dumptree(ps->p_grammar, ps->p_tree);
printf("\n");
printf("Tokens:\n");
showtree(ps->p_grammar, ps->p_tree);
printf("\n");
}
printf("Listing:\n");
PyNode_ListTree(ps->p_tree);
printf("\n");
}
#endif /* Py_DEBUG */
/*
Description
-----------
The parser's interface is different than usual: the function addtoken()
must be called for each token in the input. This makes it possible to
turn it into an incremental parsing system later. The parsing system
constructs a parse tree as it goes.
A parsing rule is represented as a Deterministic Finite-state Automaton
(DFA). A node in a DFA represents a state of the parser; an arc represents
a transition. Transitions are either labeled with terminal symbols or
with non-terminals. When the parser decides to follow an arc labeled
with a non-terminal, it is invoked recursively with the DFA representing
the parsing rule for that as its initial state; when that DFA accepts,
the parser that invoked it continues. The parse tree constructed by the
recursively called parser is inserted as a child in the current parse tree.
The DFA's can be constructed automatically from a more conventional
language description. An extended LL(1) grammar (ELL(1)) is suitable.
Certain restrictions make the parser's life easier: rules that can produce
the empty string should be outlawed (there are other ways to put loops
or optional parts in the language). To avoid the need to construct
FIRST sets, we can require that all but the last alternative of a rule
(really: arc going out of a DFA's state) must begin with a terminal
symbol.
As an example, consider this grammar:
expr: term (OP term)*
term: CONSTANT | '(' expr ')'
The DFA corresponding to the rule for expr is:
------->.---term-->.------->
^ |
| |
\----OP----/
The parse tree generated for the input a+b is:
(expr: (term: (NAME: a)), (OP: +), (term: (NAME: b)))
*/
from_cpython/Parser/parser.h 0 → 100644
View file @ 3fa45d83
#ifndef Py_PARSER_H
#define Py_PARSER_H
#ifdef __cplusplus
extern "C" {
#endif
/* Parser interface */
#define MAXSTACK 1500
typedef struct {
int s_state; /* State in current DFA */
dfa *s_dfa; /* Current DFA */
struct _node *s_parent; /* Where to add next node */
} stackentry;
typedef struct {
stackentry *s_top; /* Top entry */
stackentry s_base[MAXSTACK];/* Array of stack entries */
/* NB The stack grows down */
} stack;
typedef struct {
stack p_stack; /* Stack of parser states */
grammar *p_grammar; /* Grammar to use */
node *p_tree; /* Top of parse tree */
#ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
unsigned long p_flags; /* see co_flags in Include/code.h */
#endif
} parser_state;
parser_state *PyParser_New(grammar *g, int start);
void PyParser_Delete(parser_state *ps);
int PyParser_AddToken(parser_state *ps, int type, char *str, int lineno, int col_offset,
int *expected_ret);
void PyGrammar_AddAccelerators(grammar *g);
#ifdef __cplusplus
}
#endif
#endif /* !Py_PARSER_H */
from_cpython/Parser/parsetok.c 0 → 100644
View file @ 3fa45d83
/* Parser-tokenizer link implementation */
#include "pgenheaders.h"
#include "tokenizer.h"
#include "node.h"
#include "grammar.h"
#include "parser.h"
#include "parsetok.h"
#include "errcode.h"
#include "graminit.h"
int Py_TabcheckFlag;
/* Forward */
static node *parsetok(struct tok_state *, grammar *, int, perrdetail *, int *);
static void initerr(perrdetail *err_ret, const char* filename);
/* Parse input coming from a string. Return error code, print some errors. */
node *
PyParser_ParseString(const char *s, grammar *g, int start, perrdetail *err_ret)
{
return PyParser_ParseStringFlagsFilename(s, NULL, g, start, err_ret, 0);
}
node *
PyParser_ParseStringFlags(const char *s, grammar *g, int start,
perrdetail *err_ret, int flags)
{
return PyParser_ParseStringFlagsFilename(s, NULL,
g, start, err_ret, flags);
}
node *
PyParser_ParseStringFlagsFilename(const char *s, const char *filename,
grammar *g, int start,
perrdetail *err_ret, int flags)
{
int iflags = flags;
return PyParser_ParseStringFlagsFilenameEx(s, filename, g, start,
err_ret, &iflags);
}
node *
PyParser_ParseStringFlagsFilenameEx(const char *s, const char *filename,
grammar *g, int start,
perrdetail *err_ret, int *flags)
{
struct tok_state *tok;
initerr(err_ret, filename);
if ((tok = PyTokenizer_FromString(s, start == file_input)) == NULL) {
err_ret->error = PyErr_Occurred() ? E_DECODE : E_NOMEM;
return NULL;
}
tok->filename = filename ? filename : "<string>";
if (Py_TabcheckFlag || Py_VerboseFlag) {
tok->altwarning = (tok->filename != NULL);
if (Py_TabcheckFlag >= 2)
tok->alterror++;
}
return parsetok(tok, g, start, err_ret, flags);
}
/* Parse input coming from a file. Return error code, print some errors. */
node *
PyParser_ParseFile(FILE *fp, const char *filename, grammar *g, int start,
char *ps1, char *ps2, perrdetail *err_ret)
{
return PyParser_ParseFileFlags(fp, filename, g, start, ps1, ps2,
err_ret, 0);
}
node *
PyParser_ParseFileFlags(FILE *fp, const char *filename, grammar *g, int start,
char *ps1, char *ps2, perrdetail *err_ret, int flags)
{
int iflags = flags;
return PyParser_ParseFileFlagsEx(fp, filename, g, start, ps1, ps2, err_ret, &iflags);
}
node *
PyParser_ParseFileFlagsEx(FILE *fp, const char *filename, grammar *g, int start,
char *ps1, char *ps2, perrdetail *err_ret, int *flags)
{
struct tok_state *tok;
initerr(err_ret, filename);
if ((tok = PyTokenizer_FromFile(fp, ps1, ps2)) == NULL) {
err_ret->error = E_NOMEM;
return NULL;
}
tok->filename = filename;
if (Py_TabcheckFlag || Py_VerboseFlag) {
tok->altwarning = (filename != NULL);
if (Py_TabcheckFlag >= 2)
tok->alterror++;
}
return parsetok(tok, g, start, err_ret, flags);
}
#if 0
static char with_msg[] =
"%s:%d: Warning: 'with' will become a reserved keyword in Python 2.6\n";
static char as_msg[] =
"%s:%d: Warning: 'as' will become a reserved keyword in Python 2.6\n";
static void
warn(const char *msg, const char *filename, int lineno)
{
if (filename == NULL)
filename = "<string>";
PySys_WriteStderr(msg, filename, lineno);
}
#endif
/* Parse input coming from the given tokenizer structure.
Return error code. */
static node *
parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
int *flags)
{
parser_state *ps;
node *n;
int started = 0;
if ((ps = PyParser_New(g, start)) == NULL) {
fprintf(stderr, "no mem for new parser\n");
err_ret->error = E_NOMEM;
PyTokenizer_Free(tok);
return NULL;
}
#ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
if (*flags & PyPARSE_PRINT_IS_FUNCTION) {
ps->p_flags |= CO_FUTURE_PRINT_FUNCTION;
}
if (*flags & PyPARSE_UNICODE_LITERALS) {
ps->p_flags |= CO_FUTURE_UNICODE_LITERALS;
}
#endif
for (;;) {
char *a, *b;
int type;
size_t len;
char *str;
int col_offset;
type = PyTokenizer_Get(tok, &a, &b);
if (type == ERRORTOKEN) {
err_ret->error = tok->done;
break;
}
if (type == ENDMARKER && started) {
type = NEWLINE; /* Add an extra newline */
started = 0;
/* Add the right number of dedent tokens,
except if a certain flag is given --
codeop.py uses this. */
if (tok->indent &&
!(*flags & PyPARSE_DONT_IMPLY_DEDENT))
{
tok->pendin = -tok->indent;
tok->indent = 0;
}
}
else
started = 1;
len = b - a; /* XXX this may compute NULL - NULL */
str = (char *) PyObject_MALLOC(len + 1);
if (str == NULL) {
fprintf(stderr, "no mem for next token\n");
err_ret->error = E_NOMEM;
break;
}
if (len > 0)
strncpy(str, a, len);
str[len] = '\0';
#ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
#endif
if (a >= tok->line_start)
col_offset = a - tok->line_start;
else
col_offset = -1;
if ((err_ret->error =
PyParser_AddToken(ps, (int)type, str, tok->lineno, col_offset,
&(err_ret->expected))) != E_OK) {
if (err_ret->error != E_DONE) {
PyObject_FREE(str);
err_ret->token = type;
}
break;
}
}
if (err_ret->error == E_DONE) {
n = ps->p_tree;
ps->p_tree = NULL;
}
else
n = NULL;
#ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
*flags = ps->p_flags;
#endif
PyParser_Delete(ps);
if (n == NULL) {
if (tok->lineno <= 1 && tok->done == E_EOF)
err_ret->error = E_EOF;
err_ret->lineno = tok->lineno;
if (tok->buf != NULL) {
char *text = NULL;
size_t len;
assert(tok->cur - tok->buf < INT_MAX);
err_ret->offset = (int)(tok->cur - tok->buf);
len = tok->inp - tok->buf;
#ifdef Py_USING_UNICODE
text = PyTokenizer_RestoreEncoding(tok, len, &err_ret->offset);
#endif
if (text == NULL) {
text = (char *) PyObject_MALLOC(len + 1);
if (text != NULL) {
if (len > 0)
strncpy(text, tok->buf, len);
text[len] = '\0';
}
}
err_ret->text = text;
}
} else if (tok->encoding != NULL) {
/* 'nodes->n_str' uses PyObject_*, while 'tok->encoding' was
* allocated using PyMem_
*/
node* r = PyNode_New(encoding_decl);
if (r)
r->n_str = PyObject_MALLOC(strlen(tok->encoding)+1);
if (!r || !r->n_str) {
err_ret->error = E_NOMEM;
if (r)
PyObject_FREE(r);
n = NULL;
goto done;
}
strcpy(r->n_str, tok->encoding);
PyMem_FREE(tok->encoding);
tok->encoding = NULL;
r->n_nchildren = 1;
r->n_child = n;
n = r;
}
done:
PyTokenizer_Free(tok);
return n;
}
static void
initerr(perrdetail *err_ret, const char *filename)
{
err_ret->error = E_OK;
err_ret->filename = filename;
err_ret->lineno = 0;
err_ret->offset = 0;
err_ret->text = NULL;
err_ret->token = -1;
err_ret->expected = -1;
}
from_cpython/Parser/pgen.c 0 → 100644
View file @ 3fa45d83
/* Parser generator */
/* For a description, see the comments at end of this file */
#include "Python.h"
#include "pgenheaders.h"
#include "token.h"
#include "node.h"
#include "grammar.h"
#include "metagrammar.h"
#include "pgen.h"
extern int Py_DebugFlag;
extern int Py_IgnoreEnvironmentFlag; /* needed by Py_GETENV */
/* PART ONE -- CONSTRUCT NFA -- Cf. Algorithm 3.2 from [Aho&Ullman 77] */
typedef struct _nfaarc {
int ar_label;
int ar_arrow;
} nfaarc;
typedef struct _nfastate {
int st_narcs;
nfaarc *st_arc;
} nfastate;
typedef struct _nfa {
int nf_type;
char *nf_name;
int nf_nstates;
nfastate *nf_state;
int nf_start, nf_finish;
} nfa;
/* Forward */
static void compile_rhs(labellist *ll,
nfa *nf, node *n, int *pa, int *pb);
static void compile_alt(labellist *ll,
nfa *nf, node *n, int *pa, int *pb);
static void compile_item(labellist *ll,
nfa *nf, node *n, int *pa, int *pb);
static void compile_atom(labellist *ll,
nfa *nf, node *n, int *pa, int *pb);
static int
addnfastate(nfa *nf)
{
nfastate *st;
nf->nf_state = (nfastate *)PyObject_REALLOC(nf->nf_state,
sizeof(nfastate) * (nf->nf_nstates + 1));
if (nf->nf_state == NULL)
Py_FatalError("out of mem");
st = &nf->nf_state[nf->nf_nstates++];
st->st_narcs = 0;
st->st_arc = NULL;
return st - nf->nf_state;
}
static void
addnfaarc(nfa *nf, int from, int to, int lbl)
{
nfastate *st;
nfaarc *ar;
st = &nf->nf_state[from];
st->st_arc = (nfaarc *)PyObject_REALLOC(st->st_arc,
sizeof(nfaarc) * (st->st_narcs + 1));
if (st->st_arc == NULL)
Py_FatalError("out of mem");
ar = &st->st_arc[st->st_narcs++];
ar->ar_label = lbl;
ar->ar_arrow = to;
}
static nfa *
newnfa(char *name)
{
nfa *nf;
static int type = NT_OFFSET; /* All types will be disjunct */
nf = (nfa *)PyObject_MALLOC(sizeof(nfa));
if (nf == NULL)
Py_FatalError("no mem for new nfa");
nf->nf_type = type++;
nf->nf_name = name; /* XXX strdup(name) ??? */
nf->nf_nstates = 0;
nf->nf_state = NULL;
nf->nf_start = nf->nf_finish = -1;
return nf;
}
typedef struct _nfagrammar {
int gr_nnfas;
nfa **gr_nfa;
labellist gr_ll;
} nfagrammar;
/* Forward */
static void compile_rule(nfagrammar *gr, node *n);
static nfagrammar *
newnfagrammar(void)
{
nfagrammar *gr;
gr = (nfagrammar *)PyObject_MALLOC(sizeof(nfagrammar));
if (gr == NULL)
Py_FatalError("no mem for new nfa grammar");
gr->gr_nnfas = 0;
gr->gr_nfa = NULL;
gr->gr_ll.ll_nlabels = 0;
gr->gr_ll.ll_label = NULL;
addlabel(&gr->gr_ll, ENDMARKER, "EMPTY");
return gr;
}
static nfa *
addnfa(nfagrammar *gr, char *name)
{
nfa *nf;
nf = newnfa(name);
gr->gr_nfa = (nfa **)PyObject_REALLOC(gr->gr_nfa,
sizeof(nfa*) * (gr->gr_nnfas + 1));
if (gr->gr_nfa == NULL)
Py_FatalError("out of mem");
gr->gr_nfa[gr->gr_nnfas++] = nf;
addlabel(&gr->gr_ll, NAME, nf->nf_name);
return nf;
}
#ifdef Py_DEBUG
static char REQNFMT[] = "metacompile: less than %d children\n";
#define REQN(i, count) \
if (i < count) { \
fprintf(stderr, REQNFMT, count); \
Py_FatalError("REQN"); \
} else
#else
#define REQN(i, count) /* empty */
#endif
static nfagrammar *
metacompile(node *n)
{
nfagrammar *gr;
int i;
if (Py_DebugFlag)
printf("Compiling (meta-) parse tree into NFA grammar\n");
gr = newnfagrammar();
REQ(n, MSTART);
i = n->n_nchildren - 1; /* Last child is ENDMARKER */
n = n->n_child;
for (; --i >= 0; n++) {
if (n->n_type != NEWLINE)
compile_rule(gr, n);
}
return gr;
}
static void
compile_rule(nfagrammar *gr, node *n)
{
nfa *nf;
REQ(n, RULE);
REQN(n->n_nchildren, 4);
n = n->n_child;
REQ(n, NAME);
nf = addnfa(gr, n->n_str);
n++;
REQ(n, COLON);
n++;
REQ(n, RHS);
compile_rhs(&gr->gr_ll, nf, n, &nf->nf_start, &nf->nf_finish);
n++;
REQ(n, NEWLINE);
}
static void
compile_rhs(labellist *ll, nfa *nf, node *n, int *pa, int *pb)
{
int i;
int a, b;
REQ(n, RHS);
i = n->n_nchildren;
REQN(i, 1);
n = n->n_child;
REQ(n, ALT);
compile_alt(ll, nf, n, pa, pb);
if (--i <= 0)
return;
n++;
a = *pa;
b = *pb;
*pa = addnfastate(nf);
*pb = addnfastate(nf);
addnfaarc(nf, *pa, a, EMPTY);
addnfaarc(nf, b, *pb, EMPTY);
for (; --i >= 0; n++) {
REQ(n, VBAR);
REQN(i, 1);
--i;
n++;
REQ(n, ALT);
compile_alt(ll, nf, n, &a, &b);
addnfaarc(nf, *pa, a, EMPTY);
addnfaarc(nf, b, *pb, EMPTY);
}
}
static void
compile_alt(labellist *ll, nfa *nf, node *n, int *pa, int *pb)
{
int i;
int a, b;
REQ(n, ALT);
i = n->n_nchildren;
REQN(i, 1);
n = n->n_child;
REQ(n, ITEM);
compile_item(ll, nf, n, pa, pb);
--i;
n++;
for (; --i >= 0; n++) {
REQ(n, ITEM);
compile_item(ll, nf, n, &a, &b);
addnfaarc(nf, *pb, a, EMPTY);
*pb = b;
}
}
static void
compile_item(labellist *ll, nfa *nf, node *n, int *pa, int *pb)
{
int i;
int a, b;
REQ(n, ITEM);
i = n->n_nchildren;
REQN(i, 1);
n = n->n_child;
if (n->n_type == LSQB) {
REQN(i, 3);
n++;
REQ(n, RHS);
*pa = addnfastate(nf);
*pb = addnfastate(nf);
addnfaarc(nf, *pa, *pb, EMPTY);
compile_rhs(ll, nf, n, &a, &b);
addnfaarc(nf, *pa, a, EMPTY);
addnfaarc(nf, b, *pb, EMPTY);
REQN(i, 1);
n++;
REQ(n, RSQB);
}
else {
compile_atom(ll, nf, n, pa, pb);
if (--i <= 0)
return;
n++;
addnfaarc(nf, *pb, *pa, EMPTY);
if (n->n_type == STAR)
*pb = *pa;
else
REQ(n, PLUS);
}
}
static void
compile_atom(labellist *ll, nfa *nf, node *n, int *pa, int *pb)
{
int i;
REQ(n, ATOM);
i = n->n_nchildren;
REQN(i, 1);
n = n->n_child;
if (n->n_type == LPAR) {
REQN(i, 3);
n++;
REQ(n, RHS);
compile_rhs(ll, nf, n, pa, pb);
n++;
REQ(n, RPAR);
}
else if (n->n_type == NAME || n->n_type == STRING) {
*pa = addnfastate(nf);
*pb = addnfastate(nf);
addnfaarc(nf, *pa, *pb, addlabel(ll, n->n_type, n->n_str));
}
else
REQ(n, NAME);
}
static void
dumpstate(labellist *ll, nfa *nf, int istate)
{
nfastate *st;
int i;
nfaarc *ar;
printf("%c%2d%c",
istate == nf->nf_start ? '*' : ' ',
istate,
istate == nf->nf_finish ? '.' : ' ');
st = &nf->nf_state[istate];
ar = st->st_arc;
for (i = 0; i < st->st_narcs; i++) {
if (i > 0)
printf("\n ");
printf("-> %2d %s", ar->ar_arrow,
PyGrammar_LabelRepr(&ll->ll_label[ar->ar_label]));
ar++;
}
printf("\n");
}
static void
dumpnfa(labellist *ll, nfa *nf)
{
int i;
printf("NFA '%s' has %d states; start %d, finish %d\n",
nf->nf_name, nf->nf_nstates, nf->nf_start, nf->nf_finish);
for (i = 0; i < nf->nf_nstates; i++)
dumpstate(ll, nf, i);
}
/* PART TWO -- CONSTRUCT DFA -- Algorithm 3.1 from [Aho&Ullman 77] */
static void
addclosure(bitset ss, nfa *nf, int istate)
{
if (addbit(ss, istate)) {
nfastate *st = &nf->nf_state[istate];
nfaarc *ar = st->st_arc;
int i;
for (i = st->st_narcs; --i >= 0; ) {
if (ar->ar_label == EMPTY)
addclosure(ss, nf, ar->ar_arrow);
ar++;
}
}
}
typedef struct _ss_arc {
bitset sa_bitset;
int sa_arrow;
int sa_label;
} ss_arc;
typedef struct _ss_state {
bitset ss_ss;
int ss_narcs;
struct _ss_arc *ss_arc;
int ss_deleted;
int ss_finish;
int ss_rename;
} ss_state;
typedef struct _ss_dfa {
int sd_nstates;
ss_state *sd_state;
} ss_dfa;
/* Forward */
static void printssdfa(int xx_nstates, ss_state *xx_state, int nbits,
labellist *ll, char *msg);
static void simplify(int xx_nstates, ss_state *xx_state);
static void convert(dfa *d, int xx_nstates, ss_state *xx_state);
static void
makedfa(nfagrammar *gr, nfa *nf, dfa *d)
{
int nbits = nf->nf_nstates;
bitset ss;
int xx_nstates;
ss_state *xx_state, *yy;
ss_arc *zz;
int istate, jstate, iarc, jarc, ibit;
nfastate *st;
nfaarc *ar;
ss = newbitset(nbits);
addclosure(ss, nf, nf->nf_start);
xx_state = (ss_state *)PyObject_MALLOC(sizeof(ss_state));
if (xx_state == NULL)
Py_FatalError("no mem for xx_state in makedfa");
xx_nstates = 1;
yy = &xx_state[0];
yy->ss_ss = ss;
yy->ss_narcs = 0;
yy->ss_arc = NULL;
yy->ss_deleted = 0;
yy->ss_finish = testbit(ss, nf->nf_finish);
if (yy->ss_finish)
printf("Error: nonterminal '%s' may produce empty.\n",
nf->nf_name);
/* This algorithm is from a book written before
the invention of structured programming... */
/* For each unmarked state... */
for (istate = 0; istate < xx_nstates; ++istate) {
size_t size;
yy = &xx_state[istate];
ss = yy->ss_ss;
/* For all its states... */
for (ibit = 0; ibit < nf->nf_nstates; ++ibit) {
if (!testbit(ss, ibit))
continue;
st = &nf->nf_state[ibit];
/* For all non-empty arcs from this state... */
for (iarc = 0; iarc < st->st_narcs; iarc++) {
ar = &st->st_arc[iarc];
if (ar->ar_label == EMPTY)
continue;
/* Look up in list of arcs from this state */
for (jarc = 0; jarc < yy->ss_narcs; ++jarc) {
zz = &yy->ss_arc[jarc];
if (ar->ar_label == zz->sa_label)
goto found;
}
/* Add new arc for this state */
size = sizeof(ss_arc) * (yy->ss_narcs + 1);
yy->ss_arc = (ss_arc *)PyObject_REALLOC(
yy->ss_arc, size);
if (yy->ss_arc == NULL)
Py_FatalError("out of mem");
zz = &yy->ss_arc[yy->ss_narcs++];
zz->sa_label = ar->ar_label;
zz->sa_bitset = newbitset(nbits);
zz->sa_arrow = -1;
found: ;
/* Add destination */
addclosure(zz->sa_bitset, nf, ar->ar_arrow);
}
}
/* Now look up all the arrow states */
for (jarc = 0; jarc < xx_state[istate].ss_narcs; jarc++) {
zz = &xx_state[istate].ss_arc[jarc];
for (jstate = 0; jstate < xx_nstates; jstate++) {
if (samebitset(zz->sa_bitset,
xx_state[jstate].ss_ss, nbits)) {
zz->sa_arrow = jstate;
goto done;
}
}
size = sizeof(ss_state) * (xx_nstates + 1);
xx_state = (ss_state *)PyObject_REALLOC(xx_state,
size);
if (xx_state == NULL)
Py_FatalError("out of mem");
zz->sa_arrow = xx_nstates;
yy = &xx_state[xx_nstates++];
yy->ss_ss = zz->sa_bitset;
yy->ss_narcs = 0;
yy->ss_arc = NULL;
yy->ss_deleted = 0;
yy->ss_finish = testbit(yy->ss_ss, nf->nf_finish);
done: ;
}
}
if (Py_DebugFlag)
printssdfa(xx_nstates, xx_state, nbits, &gr->gr_ll,
"before minimizing");
simplify(xx_nstates, xx_state);
if (Py_DebugFlag)
printssdfa(xx_nstates, xx_state, nbits, &gr->gr_ll,
"after minimizing");
convert(d, xx_nstates, xx_state);
/* XXX cleanup */
PyObject_FREE(xx_state);
}
static void
printssdfa(int xx_nstates, ss_state *xx_state, int nbits,
labellist *ll, char *msg)
{
int i, ibit, iarc;
ss_state *yy;
ss_arc *zz;
printf("Subset DFA %s\n", msg);
for (i = 0; i < xx_nstates; i++) {
yy = &xx_state[i];
if (yy->ss_deleted)
continue;
printf(" Subset %d", i);
if (yy->ss_finish)
printf(" (finish)");
printf(" { ");
for (ibit = 0; ibit < nbits; ibit++) {
if (testbit(yy->ss_ss, ibit))
printf("%d ", ibit);
}
printf("}\n");
for (iarc = 0; iarc < yy->ss_narcs; iarc++) {
zz = &yy->ss_arc[iarc];
printf(" Arc to state %d, label %s\n",
zz->sa_arrow,
PyGrammar_LabelRepr(
&ll->ll_label[zz->sa_label]));
}
}
}
/* PART THREE -- SIMPLIFY DFA */
/* Simplify the DFA by repeatedly eliminating states that are
equivalent to another oner. This is NOT Algorithm 3.3 from
[Aho&Ullman 77]. It does not always finds the minimal DFA,
but it does usually make a much smaller one... (For an example
of sub-optimal behavior, try S: x a b+ | y a b+.)
*/
static int
samestate(ss_state *s1, ss_state *s2)
{
int i;
if (s1->ss_narcs != s2->ss_narcs || s1->ss_finish != s2->ss_finish)
return 0;
for (i = 0; i < s1->ss_narcs; i++) {
if (s1->ss_arc[i].sa_arrow != s2->ss_arc[i].sa_arrow ||
s1->ss_arc[i].sa_label != s2->ss_arc[i].sa_label)
return 0;
}
return 1;
}
static void
renamestates(int xx_nstates, ss_state *xx_state, int from, int to)
{
int i, j;
if (Py_DebugFlag)
printf("Rename state %d to %d.\n", from, to);
for (i = 0; i < xx_nstates; i++) {
if (xx_state[i].ss_deleted)
continue;
for (j = 0; j < xx_state[i].ss_narcs; j++) {
if (xx_state[i].ss_arc[j].sa_arrow == from)
xx_state[i].ss_arc[j].sa_arrow = to;
}
}
}
static void
simplify(int xx_nstates, ss_state *xx_state)
{
int changes;
int i, j;
do {
changes = 0;
for (i = 1; i < xx_nstates; i++) {
if (xx_state[i].ss_deleted)
continue;
for (j = 0; j < i; j++) {
if (xx_state[j].ss_deleted)
continue;
if (samestate(&xx_state[i], &xx_state[j])) {
xx_state[i].ss_deleted++;
renamestates(xx_nstates, xx_state,
i, j);
changes++;
break;
}
}
}
} while (changes);
}
/* PART FOUR -- GENERATE PARSING TABLES */
/* Convert the DFA into a grammar that can be used by our parser */
static void
convert(dfa *d, int xx_nstates, ss_state *xx_state)
{
int i, j;
ss_state *yy;
ss_arc *zz;
for (i = 0; i < xx_nstates; i++) {
yy = &xx_state[i];
if (yy->ss_deleted)
continue;
yy->ss_rename = addstate(d);
}
for (i = 0; i < xx_nstates; i++) {
yy = &xx_state[i];
if (yy->ss_deleted)
continue;
for (j = 0; j < yy->ss_narcs; j++) {
zz = &yy->ss_arc[j];
addarc(d, yy->ss_rename,
xx_state[zz->sa_arrow].ss_rename,
zz->sa_label);
}
if (yy->ss_finish)
addarc(d, yy->ss_rename, yy->ss_rename, 0);
}
d->d_initial = 0;
}
/* PART FIVE -- GLUE IT ALL TOGETHER */
static grammar *
maketables(nfagrammar *gr)
{
int i;
nfa *nf;
dfa *d;
grammar *g;
if (gr->gr_nnfas == 0)
return NULL;
g = newgrammar(gr->gr_nfa[0]->nf_type);
/* XXX first rule must be start rule */
g->g_ll = gr->gr_ll;
for (i = 0; i < gr->gr_nnfas; i++) {
nf = gr->gr_nfa[i];
if (Py_DebugFlag) {
printf("Dump of NFA for '%s' ...\n", nf->nf_name);
dumpnfa(&gr->gr_ll, nf);
printf("Making DFA for '%s' ...\n", nf->nf_name);
}
d = adddfa(g, nf->nf_type, nf->nf_name);
makedfa(gr, gr->gr_nfa[i], d);
}
return g;
}
grammar *
pgen(node *n)
{
nfagrammar *gr;
grammar *g;
gr = metacompile(n);
g = maketables(gr);
translatelabels(g);
addfirstsets(g);
PyObject_FREE(gr);
return g;
}
grammar *
Py_pgen(node *n)
{
return pgen(n);
}
/*
Description
-----------
Input is a grammar in extended BNF (using * for repetition, + for
at-least-once repetition, [] for optional parts, | for alternatives and
() for grouping). This has already been parsed and turned into a parse
tree.
Each rule is considered as a regular expression in its own right.
It is turned into a Non-deterministic Finite Automaton (NFA), which
is then turned into a Deterministic Finite Automaton (DFA), which is then
optimized to reduce the number of states. See [Aho&Ullman 77] chapter 3,
or similar compiler books (this technique is more often used for lexical
analyzers).
The DFA's are used by the parser as parsing tables in a special way
that's probably unique. Before they are usable, the FIRST sets of all
non-terminals are computed.
Reference
---------
[Aho&Ullman 77]
Aho&Ullman, Principles of Compiler Design, Addison-Wesley 1977
(first edition)
*/
from_cpython/Parser/pgenmain.c 0 → 100644
View file @ 3fa45d83
/* Parser generator main program */
/* This expects a filename containing the grammar as argv[1] (UNIX)
or asks the console for such a file name (THINK C).
It writes its output on two files in the current directory:
- "graminit.c" gets the grammar as a bunch of initialized data
- "graminit.h" gets the grammar's non-terminals as #defines.
Error messages and status info during the generation process are
written to stdout, or sometimes to stderr. */
/* XXX TO DO:
- check for duplicate definitions of names (instead of fatal err)
*/
#include "Python.h"
#include "pgenheaders.h"
#include "grammar.h"
#include "node.h"
#include "parsetok.h"
#include "pgen.h"
int Py_DebugFlag;
int Py_VerboseFlag;
int Py_IgnoreEnvironmentFlag;
/* Forward */
grammar *getgrammar(char *filename);
void
Py_Exit(int sts)
{
exit(sts);
}
int
main(int argc, char **argv)
{
grammar *g;
FILE *fp;
char *filename, *graminit_h, *graminit_c;
if (argc != 4) {
fprintf(stderr,
"usage: %s grammar graminit.h graminit.c\n", argv[0]);
Py_Exit(2);
}
filename = argv[1];
graminit_h = argv[2];
graminit_c = argv[3];
g = getgrammar(filename);
fp = fopen(graminit_c, "w");
if (fp == NULL) {
perror(graminit_c);
Py_Exit(1);
}
if (Py_DebugFlag)
printf("Writing %s ...\n", graminit_c);
printgrammar(g, fp);
fclose(fp);
fp = fopen(graminit_h, "w");
if (fp == NULL) {
perror(graminit_h);
Py_Exit(1);
}
if (Py_DebugFlag)
printf("Writing %s ...\n", graminit_h);
printnonterminals(g, fp);
fclose(fp);
Py_Exit(0);
return 0; /* Make gcc -Wall happy */
}
grammar *
getgrammar(char *filename)
{
FILE *fp;
node *n;
grammar *g0, *g;
perrdetail err;
fp = fopen(filename, "r");
if (fp == NULL) {
perror(filename);
Py_Exit(1);
}
g0 = meta_grammar();
n = PyParser_ParseFile(fp, filename, g0, g0->g_start,
(char *)NULL, (char *)NULL, &err);
fclose(fp);
if (n == NULL) {
fprintf(stderr, "Parsing error %d, line %d.\n",
err.error, err.lineno);
if (err.text != NULL) {
size_t i;
fprintf(stderr, "%s", err.text);
i = strlen(err.text);
if (i == 0 || err.text[i-1] != '\n')
fprintf(stderr, "\n");
for (i = 0; i < err.offset; i++) {
if (err.text[i] == '\t')
putc('\t', stderr);
else
putc(' ', stderr);
}
fprintf(stderr, "^\n");
PyObject_FREE(err.text);
}
Py_Exit(1);
}
g = pgen(n);
if (g == NULL) {
printf("Bad grammar.\n");
Py_Exit(1);
}
return g;
}
/* Can't happen in pgen */
PyObject*
PyErr_Occurred()
{
return 0;
}
void
Py_FatalError(const char *msg)
{
fprintf(stderr, "pgen: FATAL ERROR: %s\n", msg);
Py_Exit(1);
}
/* No-nonsense my_readline() for tokenizer.c */
char *
PyOS_Readline(FILE *sys_stdin, FILE *sys_stdout, char *prompt)
{
size_t n = 1000;
char *p = (char *)PyMem_MALLOC(n);
char *q;
if (p == NULL)
return NULL;
fprintf(stderr, "%s", prompt);
q = fgets(p, n, sys_stdin);
if (q == NULL) {
*p = '\0';
return p;
}
n = strlen(p);
if (n > 0 && p[n-1] != '\n')
p[n-1] = '\n';
return (char *)PyMem_REALLOC(p, n+1);
}
/* No-nonsense fgets */
char *
Py_UniversalNewlineFgets(char *buf, int n, FILE *stream, PyObject *fobj)
{
return fgets(buf, n, stream);
}
#include <stdarg.h>
void
PySys_WriteStderr(const char *format, ...)
{
va_list va;
va_start(va, format);
vfprintf(stderr, format, va);
va_end(va);
}
from_cpython/Parser/printgrammar.c 0 → 100644
View file @ 3fa45d83
/* Print a bunch of C initializers that represent a grammar */
#include "pgenheaders.h"
#include "grammar.h"
/* Forward */
static void printarcs(int, dfa *, FILE *);
static void printstates(grammar *, FILE *);
static void printdfas(grammar *, FILE *);
static void printlabels(grammar *, FILE *);
void
printgrammar(grammar *g, FILE *fp)
{
fprintf(fp, "/* Generated by Parser/pgen */\n\n");
fprintf(fp, "#include \"pgenheaders.h\"\n");
fprintf(fp, "#include \"grammar.h\"\n");
fprintf(fp, "PyAPI_DATA(grammar) _PyParser_Grammar;\n");
printdfas(g, fp);
printlabels(g, fp);
fprintf(fp, "grammar _PyParser_Grammar = {\n");
fprintf(fp, " %d,\n", g->g_ndfas);
fprintf(fp, " dfas,\n");
fprintf(fp, " {%d, labels},\n", g->g_ll.ll_nlabels);
fprintf(fp, " %d\n", g->g_start);
fprintf(fp, "};\n");
}
void
printnonterminals(grammar *g, FILE *fp)
{
dfa *d;
int i;
fprintf(fp, "/* Generated by Parser/pgen */\n\n");
d = g->g_dfa;
for (i = g->g_ndfas; --i >= 0; d++)
fprintf(fp, "#define %s %d\n", d->d_name, d->d_type);
}
static void
printarcs(int i, dfa *d, FILE *fp)
{
arc *a;
state *s;
int j, k;
s = d->d_state;
for (j = 0; j < d->d_nstates; j++, s++) {
fprintf(fp, "static arc arcs_%d_%d[%d] = {\n",
i, j, s->s_narcs);
a = s->s_arc;
for (k = 0; k < s->s_narcs; k++, a++)
fprintf(fp, " {%d, %d},\n", a->a_lbl, a->a_arrow);
fprintf(fp, "};\n");
}
}
static void
printstates(grammar *g, FILE *fp)
{
state *s;
dfa *d;
int i, j;
d = g->g_dfa;
for (i = 0; i < g->g_ndfas; i++, d++) {
printarcs(i, d, fp);
fprintf(fp, "static state states_%d[%d] = {\n",
i, d->d_nstates);
s = d->d_state;
for (j = 0; j < d->d_nstates; j++, s++)
fprintf(fp, " {%d, arcs_%d_%d},\n",
s->s_narcs, i, j);
fprintf(fp, "};\n");
}
}
static void
printdfas(grammar *g, FILE *fp)
{
dfa *d;
int i, j;
printstates(g, fp);
fprintf(fp, "static dfa dfas[%d] = {\n", g->g_ndfas);
d = g->g_dfa;
for (i = 0; i < g->g_ndfas; i++, d++) {
fprintf(fp, " {%d, \"%s\", %d, %d, states_%d,\n",
d->d_type, d->d_name, d->d_initial, d->d_nstates, i);
fprintf(fp, " \"");
for (j = 0; j < NBYTES(g->g_ll.ll_nlabels); j++)
fprintf(fp, "\\%03o", d->d_first[j] & 0xff);
fprintf(fp, "\"},\n");
}
fprintf(fp, "};\n");
}
static void
printlabels(grammar *g, FILE *fp)
{
label *l;
int i;
fprintf(fp, "static label labels[%d] = {\n", g->g_ll.ll_nlabels);
l = g->g_ll.ll_label;
for (i = g->g_ll.ll_nlabels; --i >= 0; l++) {
if (l->lb_str == NULL)
fprintf(fp, " {%d, 0},\n", l->lb_type);
else
fprintf(fp, " {%d, \"%s\"},\n",
l->lb_type, l->lb_str);
}
fprintf(fp, "};\n");
}
from_cpython/Parser/spark.py 0 → 100644
View file @ 3fa45d83
# Copyright (c) 1998-2002 John Aycock
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
__version__ = 'SPARK-0.7 (pre-alpha-5)'
import re
import string
def _namelist(instance):
namelist, namedict, classlist = [], {}, [instance.__class__]
for c in classlist:
for b in c.__bases__:
classlist.append(b)
for name in c.__dict__.keys():
if not namedict.has_key(name):
namelist.append(name)
namedict[name] = 1
return namelist
class GenericScanner:
def __init__(self, flags=0):
pattern = self.reflect()
self.re = re.compile(pattern, re.VERBOSE|flags)
self.index2func = {}
for name, number in self.re.groupindex.items():
self.index2func[number-1] = getattr(self, 't_' + name)
def makeRE(self, name):
doc = getattr(self, name).__doc__
rv = '(?P<%s>%s)' % (name[2:], doc)
return rv
def reflect(self):
rv = []
for name in _namelist(self):
if name[:2] == 't_' and name != 't_default':
rv.append(self.makeRE(name))
rv.append(self.makeRE('t_default'))
return string.join(rv, '|')
def error(self, s, pos):
print "Lexical error at position %s" % pos
raise SystemExit
def tokenize(self, s):
pos = 0
n = len(s)
while pos < n:
m = self.re.match(s, pos)
if m is None:
self.error(s, pos)
groups = m.groups()
for i in range(len(groups)):
if groups[i] and self.index2func.has_key(i):
self.index2func[i](groups[i])
pos = m.end()
def t_default(self, s):
r'( . | \n )+'
print "Specification error: unmatched input"
raise SystemExit
#
# Extracted from GenericParser and made global so that [un]picking works.
#
class _State:
def __init__(self, stateno, items):
self.T, self.complete, self.items = [], [], items
self.stateno = stateno
class GenericParser:
#
# An Earley parser, as per J. Earley, "An Efficient Context-Free
# Parsing Algorithm", CACM 13(2), pp. 94-102. Also J. C. Earley,
# "An Efficient Context-Free Parsing Algorithm", Ph.D. thesis,
# Carnegie-Mellon University, August 1968. New formulation of
# the parser according to J. Aycock, "Practical Earley Parsing
# and the SPARK Toolkit", Ph.D. thesis, University of Victoria,
# 2001, and J. Aycock and R. N. Horspool, "Practical Earley
# Parsing", unpublished paper, 2001.
#
def __init__(self, start):
self.rules = {}
self.rule2func = {}
self.rule2name = {}
self.collectRules()
self.augment(start)
self.ruleschanged = 1
_NULLABLE = '\e_'
_START = 'START'
_BOF = '|-'
#
# When pickling, take the time to generate the full state machine;
# some information is then extraneous, too. Unfortunately we
# can't save the rule2func map.
#
def __getstate__(self):
if self.ruleschanged:
#
# XXX - duplicated from parse()
#
self.computeNull()
self.newrules = {}
self.new2old = {}
self.makeNewRules()
self.ruleschanged = 0
self.edges, self.cores = {}, {}
self.states = { 0: self.makeState0() }
self.makeState(0, self._BOF)
#
# XXX - should find a better way to do this..
#
changes = 1
while changes:
changes = 0
for k, v in self.edges.items():
if v is None:
state, sym = k
if self.states.has_key(state):
self.goto(state, sym)
changes = 1
rv = self.__dict__.copy()
for s in self.states.values():
del s.items
del rv['rule2func']
del rv['nullable']
del rv['cores']
return rv
def __setstate__(self, D):
self.rules = {}
self.rule2func = {}
self.rule2name = {}
self.collectRules()
start = D['rules'][self._START][0][1][1] # Blech.
self.augment(start)
D['rule2func'] = self.rule2func
D['makeSet'] = self.makeSet_fast
self.__dict__ = D
#
# A hook for GenericASTBuilder and GenericASTMatcher. Mess
# thee not with this; nor shall thee toucheth the _preprocess
# argument to addRule.
#
def preprocess(self, rule, func): return rule, func
def addRule(self, doc, func, _preprocess=1):
fn = func
rules = string.split(doc)
index = []
for i in range(len(rules)):
if rules[i] == '::=':
index.append(i-1)
index.append(len(rules))
for i in range(len(index)-1):
lhs = rules[index[i]]
rhs = rules[index[i]+2:index[i+1]]
rule = (lhs, tuple(rhs))
if _preprocess:
rule, fn = self.preprocess(rule, func)
if self.rules.has_key(lhs):
self.rules[lhs].append(rule)
else:
self.rules[lhs] = [ rule ]
self.rule2func[rule] = fn
self.rule2name[rule] = func.__name__[2:]
self.ruleschanged = 1
def collectRules(self):
for name in _namelist(self):
if name[:2] == 'p_':
func = getattr(self, name)
doc = func.__doc__
self.addRule(doc, func)
def augment(self, start):
rule = '%s ::= %s %s' % (self._START, self._BOF, start)
self.addRule(rule, lambda args: args[1], 0)
def computeNull(self):
self.nullable = {}
tbd = []
for rulelist in self.rules.values():
lhs = rulelist[0][0]
self.nullable[lhs] = 0
for rule in rulelist:
rhs = rule[1]
if len(rhs) == 0:
self.nullable[lhs] = 1
continue
#
# We only need to consider rules which
# consist entirely of nonterminal symbols.
# This should be a savings on typical
# grammars.
#
for sym in rhs:
if not self.rules.has_key(sym):
break
else:
tbd.append(rule)
changes = 1
while changes:
changes = 0
for lhs, rhs in tbd:
if self.nullable[lhs]:
continue
for sym in rhs:
if not self.nullable[sym]:
break
else:
self.nullable[lhs] = 1
changes = 1
def makeState0(self):
s0 = _State(0, [])
for rule in self.newrules[self._START]:
s0.items.append((rule, 0))
return s0
def finalState(self, tokens):
#
# Yuck.
#
if len(self.newrules[self._START]) == 2 and len(tokens) == 0:
return 1
start = self.rules[self._START][0][1][1]
return self.goto(1, start)
def makeNewRules(self):
worklist = []
for rulelist in self.rules.values():
for rule in rulelist:
worklist.append((rule, 0, 1, rule))
for rule, i, candidate, oldrule in worklist:
lhs, rhs = rule
n = len(rhs)
while i < n:
sym = rhs[i]
if not self.rules.has_key(sym) or \
not self.nullable[sym]:
candidate = 0
i = i + 1
continue
newrhs = list(rhs)
newrhs[i] = self._NULLABLE+sym
newrule = (lhs, tuple(newrhs))
worklist.append((newrule, i+1,
candidate, oldrule))
candidate = 0
i = i + 1
else:
if candidate:
lhs = self._NULLABLE+lhs
rule = (lhs, rhs)
if self.newrules.has_key(lhs):
self.newrules[lhs].append(rule)
else:
self.newrules[lhs] = [ rule ]
self.new2old[rule] = oldrule
def typestring(self, token):
return None
def error(self, token):
print "Syntax error at or near `%s' token" % token
raise SystemExit
def parse(self, tokens):
sets = [ [(1,0), (2,0)] ]
self.links = {}
if self.ruleschanged:
self.computeNull()
self.newrules = {}
self.new2old = {}
self.makeNewRules()
self.ruleschanged = 0
self.edges, self.cores = {}, {}
self.states = { 0: self.makeState0() }
self.makeState(0, self._BOF)
for i in xrange(len(tokens)):
sets.append([])
if sets[i] == []:
break
self.makeSet(tokens[i], sets, i)
else:
sets.append([])
self.makeSet(None, sets, len(tokens))
#_dump(tokens, sets, self.states)
finalitem = (self.finalState(tokens), 0)
if finalitem not in sets[-2]:
if len(tokens) > 0:
self.error(tokens[i-1])
else:
self.error(None)
return self.buildTree(self._START, finalitem,
tokens, len(sets)-2)
def isnullable(self, sym):
#
# For symbols in G_e only. If we weren't supporting 1.5,
# could just use sym.startswith().
#
return self._NULLABLE == sym[0:len(self._NULLABLE)]
def skip(self, (lhs, rhs), pos=0):
n = len(rhs)
while pos < n:
if not self.isnullable(rhs[pos]):
break
pos = pos + 1
return pos
def makeState(self, state, sym):
assert sym is not None
#
# Compute \epsilon-kernel state's core and see if
# it exists already.
#
kitems = []
for rule, pos in self.states[state].items:
lhs, rhs = rule
if rhs[pos:pos+1] == (sym,):
kitems.append((rule, self.skip(rule, pos+1)))
core = kitems
core.sort()
tcore = tuple(core)
if self.cores.has_key(tcore):
return self.cores[tcore]
#
# Nope, doesn't exist. Compute it and the associated
# \epsilon-nonkernel state together; we'll need it right away.
#
k = self.cores[tcore] = len(self.states)
K, NK = _State(k, kitems), _State(k+1, [])
self.states[k] = K
predicted = {}
edges = self.edges
rules = self.newrules
for X in K, NK:
worklist = X.items
for item in worklist:
rule, pos = item
lhs, rhs = rule
if pos == len(rhs):
X.complete.append(rule)
continue
nextSym = rhs[pos]
key = (X.stateno, nextSym)
if not rules.has_key(nextSym):
if not edges.has_key(key):
edges[key] = None
X.T.append(nextSym)
else:
edges[key] = None
if not predicted.has_key(nextSym):
predicted[nextSym] = 1
for prule in rules[nextSym]:
ppos = self.skip(prule)
new = (prule, ppos)
NK.items.append(new)
#
# Problem: we know K needs generating, but we
# don't yet know about NK. Can't commit anything
# regarding NK to self.edges until we're sure. Should
# we delay committing on both K and NK to avoid this
# hacky code? This creates other problems..
#
if X is K:
edges = {}
if NK.items == []:
return k
#
# Check for \epsilon-nonkernel's core. Unfortunately we
# need to know the entire set of predicted nonterminals
# to do this without accidentally duplicating states.
#
core = predicted.keys()
core.sort()
tcore = tuple(core)
if self.cores.has_key(tcore):
self.edges[(k, None)] = self.cores[tcore]
return k
nk = self.cores[tcore] = self.edges[(k, None)] = NK.stateno
self.edges.update(edges)
self.states[nk] = NK
return k
def goto(self, state, sym):
key = (state, sym)
if not self.edges.has_key(key):
#
# No transitions from state on sym.
#
return None
rv = self.edges[key]
if rv is None:
#
# Target state isn't generated yet. Remedy this.
#
rv = self.makeState(state, sym)
self.edges[key] = rv
return rv
def gotoT(self, state, t):
return [self.goto(state, t)]
def gotoST(self, state, st):
rv = []
for t in self.states[state].T:
if st == t:
rv.append(self.goto(state, t))
return rv
def add(self, set, item, i=None, predecessor=None, causal=None):
if predecessor is None:
if item not in set:
set.append(item)
else:
key = (item, i)
if item not in set:
self.links[key] = []
set.append(item)
self.links[key].append((predecessor, causal))
def makeSet(self, token, sets, i):
cur, next = sets[i], sets[i+1]
ttype = token is not None and self.typestring(token) or None
if ttype is not None:
fn, arg = self.gotoT, ttype
else:
fn, arg = self.gotoST, token
for item in cur:
ptr = (item, i)
state, parent = item
add = fn(state, arg)
for k in add:
if k is not None:
self.add(next, (k, parent), i+1, ptr)
nk = self.goto(k, None)
if nk is not None:
self.add(next, (nk, i+1))
if parent == i:
continue
for rule in self.states[state].complete:
lhs, rhs = rule
for pitem in sets[parent]:
pstate, pparent = pitem
k = self.goto(pstate, lhs)
if k is not None:
why = (item, i, rule)
pptr = (pitem, parent)
self.add(cur, (k, pparent),
i, pptr, why)
nk = self.goto(k, None)
if nk is not None:
self.add(cur, (nk, i))
def makeSet_fast(self, token, sets, i):
#
# Call *only* when the entire state machine has been built!
# It relies on self.edges being filled in completely, and
# then duplicates and inlines code to boost speed at the
# cost of extreme ugliness.
#
cur, next = sets[i], sets[i+1]
ttype = token is not None and self.typestring(token) or None
for item in cur:
ptr = (item, i)
state, parent = item
if ttype is not None:
k = self.edges.get((state, ttype), None)
if k is not None:
#self.add(next, (k, parent), i+1, ptr)
#INLINED --v
new = (k, parent)
key = (new, i+1)
if new not in next:
self.links[key] = []
next.append(new)
self.links[key].append((ptr, None))
#INLINED --^
#nk = self.goto(k, None)
nk = self.edges.get((k, None), None)
if nk is not None:
#self.add(next, (nk, i+1))
#INLINED --v
new = (nk, i+1)
if new not in next:
next.append(new)
#INLINED --^
else:
add = self.gotoST(state, token)
for k in add:
if k is not None:
self.add(next, (k, parent), i+1, ptr)
#nk = self.goto(k, None)
nk = self.edges.get((k, None), None)
if nk is not None:
self.add(next, (nk, i+1))
if parent == i:
continue
for rule in self.states[state].complete:
lhs, rhs = rule
for pitem in sets[parent]:
pstate, pparent = pitem
#k = self.goto(pstate, lhs)
k = self.edges.get((pstate, lhs), None)
if k is not None:
why = (item, i, rule)
pptr = (pitem, parent)
#self.add(cur, (k, pparent),
# i, pptr, why)
#INLINED --v
new = (k, pparent)
key = (new, i)
if new not in cur:
self.links[key] = []
cur.append(new)
self.links[key].append((pptr, why))
#INLINED --^
#nk = self.goto(k, None)
nk = self.edges.get((k, None), None)
if nk is not None:
#self.add(cur, (nk, i))
#INLINED --v
new = (nk, i)
if new not in cur:
cur.append(new)
#INLINED --^
def predecessor(self, key, causal):
for p, c in self.links[key]:
if c == causal:
return p
assert 0
def causal(self, key):
links = self.links[key]
if len(links) == 1:
return links[0][1]
choices = []
rule2cause = {}
for p, c in links:
rule = c[2]
choices.append(rule)
rule2cause[rule] = c
return rule2cause[self.ambiguity(choices)]
def deriveEpsilon(self, nt):
if len(self.newrules[nt]) > 1:
rule = self.ambiguity(self.newrules[nt])
else:
rule = self.newrules[nt][0]
#print rule
rhs = rule[1]
attr = [None] * len(rhs)
for i in range(len(rhs)-1, -1, -1):
attr[i] = self.deriveEpsilon(rhs[i])
return self.rule2func[self.new2old[rule]](attr)
def buildTree(self, nt, item, tokens, k):
state, parent = item
choices = []
for rule in self.states[state].complete:
if rule[0] == nt:
choices.append(rule)
rule = choices[0]
if len(choices) > 1:
rule = self.ambiguity(choices)
#print rule
rhs = rule[1]
attr = [None] * len(rhs)
for i in range(len(rhs)-1, -1, -1):
sym = rhs[i]
if not self.newrules.has_key(sym):
if sym != self._BOF:
attr[i] = tokens[k-1]
key = (item, k)
item, k = self.predecessor(key, None)
#elif self.isnullable(sym):
elif self._NULLABLE == sym[0:len(self._NULLABLE)]:
attr[i] = self.deriveEpsilon(sym)
else:
key = (item, k)
why = self.causal(key)
attr[i] = self.buildTree(sym, why[0],
tokens, why[1])
item, k = self.predecessor(key, why)
return self.rule2func[self.new2old[rule]](attr)
def ambiguity(self, rules):
#
# XXX - problem here and in collectRules() if the same rule
# appears in >1 method. Also undefined results if rules
# causing the ambiguity appear in the same method.
#
sortlist = []
name2index = {}
for i in range(len(rules)):
lhs, rhs = rule = rules[i]
name = self.rule2name[self.new2old[rule]]
sortlist.append((len(rhs), name))
name2index[name] = i
sortlist.sort()
list = map(lambda (a,b): b, sortlist)
return rules[name2index[self.resolve(list)]]
def resolve(self, list):
#
# Resolve ambiguity in favor of the shortest RHS.
# Since we walk the tree from the top down, this
# should effectively resolve in favor of a "shift".
#
return list[0]
#
# GenericASTBuilder automagically constructs a concrete/abstract syntax tree
# for a given input. The extra argument is a class (not an instance!)
# which supports the "__setslice__" and "__len__" methods.
#
# XXX - silently overrides any user code in methods.
#
class GenericASTBuilder(GenericParser):
def __init__(self, AST, start):
GenericParser.__init__(self, start)
self.AST = AST
def preprocess(self, rule, func):
rebind = lambda lhs, self=self: \
lambda args, lhs=lhs, self=self: \
self.buildASTNode(args, lhs)
lhs, rhs = rule
return rule, rebind(lhs)
def buildASTNode(self, args, lhs):
children = []
for arg in args:
if isinstance(arg, self.AST):
children.append(arg)
else:
children.append(self.terminal(arg))
return self.nonterminal(lhs, children)
def terminal(self, token): return token
def nonterminal(self, type, args):
rv = self.AST(type)
rv[:len(args)] = args
return rv
#
# GenericASTTraversal is a Visitor pattern according to Design Patterns. For
# each node it attempts to invoke the method n_<node type>, falling
# back onto the default() method if the n_* can't be found. The preorder
# traversal also looks for an exit hook named n_<node type>_exit (no default
# routine is called if it's not found). To prematurely halt traversal
# of a subtree, call the prune() method -- this only makes sense for a
# preorder traversal. Node type is determined via the typestring() method.
#
class GenericASTTraversalPruningException:
pass
class GenericASTTraversal:
def __init__(self, ast):
self.ast = ast
def typestring(self, node):
return node.type
def prune(self):
raise GenericASTTraversalPruningException
def preorder(self, node=None):
if node is None:
node = self.ast
try:
name = 'n_' + self.typestring(node)
if hasattr(self, name):
func = getattr(self, name)
func(node)
else:
self.default(node)
except GenericASTTraversalPruningException:
return
for kid in node:
self.preorder(kid)
name = name + '_exit'
if hasattr(self, name):
func = getattr(self, name)
func(node)
def postorder(self, node=None):
if node is None:
node = self.ast
for kid in node:
self.postorder(kid)
name = 'n_' + self.typestring(node)
if hasattr(self, name):
func = getattr(self, name)
func(node)
else:
self.default(node)
def default(self, node):
pass
#
# GenericASTMatcher. AST nodes must have "__getitem__" and "__cmp__"
# implemented.
#
# XXX - makes assumptions about how GenericParser walks the parse tree.
#
class GenericASTMatcher(GenericParser):
def __init__(self, start, ast):
GenericParser.__init__(self, start)
self.ast = ast
def preprocess(self, rule, func):
rebind = lambda func, self=self: \
lambda args, func=func, self=self: \
self.foundMatch(args, func)
lhs, rhs = rule
rhslist = list(rhs)
rhslist.reverse()
return (lhs, tuple(rhslist)), rebind(func)
def foundMatch(self, args, func):
func(args[-1])
return args[-1]
def match_r(self, node):
self.input.insert(0, node)
children = 0
for child in node:
if children == 0:
self.input.insert(0, '(')
children = children + 1
self.match_r(child)
if children > 0:
self.input.insert(0, ')')
def match(self, ast=None):
if ast is None:
ast = self.ast
self.input = []
self.match_r(ast)
self.parse(self.input)
def resolve(self, list):
#
# Resolve ambiguity in favor of the longest RHS.
#
return list[-1]
def _dump(tokens, sets, states):
for i in range(len(sets)):
print 'set', i
for item in sets[i]:
print '\t', item
for (lhs, rhs), pos in states[item[0]].items:
print '\t\t', lhs, '::=',
print string.join(rhs[:pos]),
print '.',
print string.join(rhs[pos:])
if i < len(tokens):
print
print 'token', str(tokens[i])
print
from_cpython/Parser/tokenizer.c 0 → 100644
View file @ 3fa45d83
/* Tokenizer implementation */
#include "Python.h"
#include "pgenheaders.h"
#include <ctype.h>
#include <assert.h>
#include "tokenizer.h"
#include "errcode.h"
#ifndef PGEN
#include "unicodeobject.h"
#include "stringobject.h"
#include "fileobject.h"
#include "codecs.h"
#include "abstract.h"
#include "pydebug.h"
#endif /* PGEN */
extern char *PyOS_Readline(FILE *, FILE *, char *);
/* Return malloc'ed string including trailing \n;
empty malloc'ed string for EOF;
NULL if interrupted */
/* Don't ever change this -- it would break the portability of Python code */
#define TABSIZE 8
/* Forward */
static struct tok_state *tok_new(void);
static int tok_nextc(struct tok_state *tok);
static void tok_backup(struct tok_state *tok, int c);
/* Token names */
char *_PyParser_TokenNames[] = {
"ENDMARKER",
"NAME",
"NUMBER",
"STRING",
"NEWLINE",
"INDENT",
"DEDENT",
"LPAR",
"RPAR",
"LSQB",
"RSQB",
"COLON",
"COMMA",
"SEMI",
"PLUS",
"MINUS",
"STAR",
"SLASH",
"VBAR",
"AMPER",
"LESS",
"GREATER",
"EQUAL",
"DOT",
"PERCENT",
"BACKQUOTE",
"LBRACE",
"RBRACE",
"EQEQUAL",
"NOTEQUAL",
"LESSEQUAL",
"GREATEREQUAL",
"TILDE",
"CIRCUMFLEX",
"LEFTSHIFT",
"RIGHTSHIFT",
"DOUBLESTAR",
"PLUSEQUAL",
"MINEQUAL",
"STAREQUAL",
"SLASHEQUAL",
"PERCENTEQUAL",
"AMPEREQUAL",
"VBAREQUAL",
"CIRCUMFLEXEQUAL",
"LEFTSHIFTEQUAL",
"RIGHTSHIFTEQUAL",
"DOUBLESTAREQUAL",
"DOUBLESLASH",
"DOUBLESLASHEQUAL",
"AT",
/* This table must match the #defines in token.h! */
"OP",
"<ERRORTOKEN>",
"<N_TOKENS>"
};
/* Create and initialize a new tok_state structure */
static struct tok_state *
tok_new(void)
{
struct tok_state *tok = (struct tok_state *)PyMem_MALLOC(
sizeof(struct tok_state));
if (tok == NULL)
return NULL;
tok->buf = tok->cur = tok->end = tok->inp = tok->start = NULL;
tok->done = E_OK;
tok->fp = NULL;
tok->input = NULL;
tok->tabsize = TABSIZE;
tok->indent = 0;
tok->indstack[0] = 0;
tok->atbol = 1;
tok->pendin = 0;
tok->prompt = tok->nextprompt = NULL;
tok->lineno = 0;
tok->level = 0;
tok->filename = NULL;
tok->altwarning = 0;
tok->alterror = 0;
tok->alttabsize = 1;
tok->altindstack[0] = 0;
tok->decoding_state = 0;
tok->decoding_erred = 0;
tok->read_coding_spec = 0;
tok->encoding = NULL;
tok->cont_line = 0;
#ifndef PGEN
tok->decoding_readline = NULL;
tok->decoding_buffer = NULL;
#endif
return tok;
}
static char *
new_string(const char *s, Py_ssize_t len)
{
char* result = (char *)PyMem_MALLOC(len + 1);
if (result != NULL) {
memcpy(result, s, len);
result[len] = '\0';
}
return result;
}
#ifdef PGEN
static char *
decoding_fgets(char *s, int size, struct tok_state *tok)
{
return fgets(s, size, tok->fp);
}
static int
decoding_feof(struct tok_state *tok)
{
return feof(tok->fp);
}
static char *
decode_str(const char *str, int exec_input, struct tok_state *tok)
{
return new_string(str, strlen(str));
}
#else /* PGEN */
static char *
error_ret(struct tok_state *tok) /* XXX */
{
tok->decoding_erred = 1;
if (tok->fp != NULL && tok->buf != NULL) /* see PyTokenizer_Free */
PyMem_FREE(tok->buf);
tok->buf = NULL;
return NULL; /* as if it were EOF */
}
static char *
get_normal_name(char *s) /* for utf-8 and latin-1 */
{
char buf[13];
int i;
for (i = 0; i < 12; i++) {
int c = s[i];
if (c == '\0')
break;
else if (c == '_')
buf[i] = '-';
else
buf[i] = tolower(c);
}
buf[i] = '\0';
if (strcmp(buf, "utf-8") == 0 ||
strncmp(buf, "utf-8-", 6) == 0)
return "utf-8";
else if (strcmp(buf, "latin-1") == 0 ||
strcmp(buf, "iso-8859-1") == 0 ||
strcmp(buf, "iso-latin-1") == 0 ||
strncmp(buf, "latin-1-", 8) == 0 ||
strncmp(buf, "iso-8859-1-", 11) == 0 ||
strncmp(buf, "iso-latin-1-", 12) == 0)
return "iso-8859-1";
else
return s;
}
/* Return the coding spec in S, or NULL if none is found. */
static char *
get_coding_spec(const char *s, Py_ssize_t size)
{
Py_ssize_t i;
/* Coding spec must be in a comment, and that comment must be
* the only statement on the source code line. */
for (i = 0; i < size - 6; i++) {
if (s[i] == '#')
break;
if (s[i] != ' ' && s[i] != '\t' && s[i] != '\014')
return NULL;
}
for (; i < size - 6; i++) { /* XXX inefficient search */
const char* t = s + i;
if (strncmp(t, "coding", 6) == 0) {
const char* begin = NULL;
t += 6;
if (t[0] != ':' && t[0] != '=')
continue;
do {
t++;
} while (t[0] == '\x20' || t[0] == '\t');
begin = t;
while (Py_ISALNUM(t[0]) ||
t[0] == '-' || t[0] == '_' || t[0] == '.')
t++;
if (begin < t) {
char* r = new_string(begin, t - begin);
char* q = get_normal_name(r);
if (r != q) {
PyMem_FREE(r);
r = new_string(q, strlen(q));
}
return r;
}
}
}
return NULL;
}
/* Check whether the line contains a coding spec. If it does,
invoke the set_readline function for the new encoding.
This function receives the tok_state and the new encoding.
Return 1 on success, 0 on failure. */
static int
check_coding_spec(const char* line, Py_ssize_t size, struct tok_state *tok,
int set_readline(struct tok_state *, const char *))
{
char * cs;
int r = 1;
if (tok->cont_line)
/* It's a continuation line, so it can't be a coding spec. */
return 1;
cs = get_coding_spec(line, size);
if (cs != NULL) {
tok->read_coding_spec = 1;
if (tok->encoding == NULL) {
assert(tok->decoding_state == 1); /* raw */
if (strcmp(cs, "utf-8") == 0 ||
strcmp(cs, "iso-8859-1") == 0) {
tok->encoding = cs;
} else {
#ifdef Py_USING_UNICODE
r = set_readline(tok, cs);
if (r) {
tok->encoding = cs;
tok->decoding_state = -1;
}
else {
PyErr_Format(PyExc_SyntaxError,
"encoding problem: %s", cs);
PyMem_FREE(cs);
}
#else
/* Without Unicode support, we cannot
process the coding spec. Since there
won't be any Unicode literals, that
won't matter. */
PyMem_FREE(cs);
#endif
}
} else { /* then, compare cs with BOM */
r = (strcmp(tok->encoding, cs) == 0);
if (!r)
PyErr_Format(PyExc_SyntaxError,
"encoding problem: %s with BOM", cs);
PyMem_FREE(cs);
}
}
return r;
}
/* See whether the file starts with a BOM. If it does,
invoke the set_readline function with the new encoding.
Return 1 on success, 0 on failure. */
static int
check_bom(int get_char(struct tok_state *),
void unget_char(int, struct tok_state *),
int set_readline(struct tok_state *, const char *),
struct tok_state *tok)
{
int ch1, ch2, ch3;
ch1 = get_char(tok);
tok->decoding_state = 1;
if (ch1 == EOF) {
return 1;
} else if (ch1 == 0xEF) {
ch2 = get_char(tok);
if (ch2 != 0xBB) {
unget_char(ch2, tok);
unget_char(ch1, tok);
return 1;
}
ch3 = get_char(tok);
if (ch3 != 0xBF) {
unget_char(ch3, tok);
unget_char(ch2, tok);
unget_char(ch1, tok);
return 1;
}
#if 0
/* Disable support for UTF-16 BOMs until a decision
is made whether this needs to be supported. */
} else if (ch1 == 0xFE) {
ch2 = get_char(tok);
if (ch2 != 0xFF) {
unget_char(ch2, tok);
unget_char(ch1, tok);
return 1;
}
if (!set_readline(tok, "utf-16-be"))
return 0;
tok->decoding_state = -1;
} else if (ch1 == 0xFF) {
ch2 = get_char(tok);
if (ch2 != 0xFE) {
unget_char(ch2, tok);
unget_char(ch1, tok);
return 1;
}
if (!set_readline(tok, "utf-16-le"))
return 0;
tok->decoding_state = -1;
#endif
} else {
unget_char(ch1, tok);
return 1;
}
if (tok->encoding != NULL)
PyMem_FREE(tok->encoding);
tok->encoding = new_string("utf-8", 5); /* resulting is in utf-8 */
return 1;
}
/* Read a line of text from TOK into S, using the stream in TOK.
Return NULL on failure, else S.
On entry, tok->decoding_buffer will be one of:
1) NULL: need to call tok->decoding_readline to get a new line
2) PyUnicodeObject *: decoding_feof has called tok->decoding_readline and
stored the result in tok->decoding_buffer
3) PyStringObject *: previous call to fp_readl did not have enough room
(in the s buffer) to copy entire contents of the line read
by tok->decoding_readline. tok->decoding_buffer has the overflow.
In this case, fp_readl is called in a loop (with an expanded buffer)
until the buffer ends with a '\n' (or until the end of the file is
reached): see tok_nextc and its calls to decoding_fgets.
*/
static char *
fp_readl(char *s, int size, struct tok_state *tok)
{
#ifndef Py_USING_UNICODE
/* In a non-Unicode built, this should never be called. */
Py_FatalError("fp_readl should not be called in this build.");
return NULL; /* Keep compiler happy (not reachable) */
#else
PyObject* utf8 = NULL;
PyObject* buf = tok->decoding_buffer;
char *str;
Py_ssize_t utf8len;
/* Ask for one less byte so we can terminate it */
assert(size > 0);
size--;
if (buf == NULL) {
buf = PyObject_CallObject(tok->decoding_readline, NULL);
if (buf == NULL)
return error_ret(tok);
if (!PyUnicode_Check(buf)) {
Py_DECREF(buf);
PyErr_SetString(PyExc_SyntaxError,
"codec did not return a unicode object");
return error_ret(tok);
}
} else {
tok->decoding_buffer = NULL;
if (PyString_CheckExact(buf))
utf8 = buf;
}
if (utf8 == NULL) {
utf8 = PyUnicode_AsUTF8String(buf);
Py_DECREF(buf);
if (utf8 == NULL)
return error_ret(tok);
}
str = PyString_AsString(utf8);
utf8len = PyString_GET_SIZE(utf8);
if (utf8len > size) {
tok->decoding_buffer = PyString_FromStringAndSize(str+size, utf8len-size);
if (tok->decoding_buffer == NULL) {
Py_DECREF(utf8);
return error_ret(tok);
}
utf8len = size;
}
memcpy(s, str, utf8len);
s[utf8len] = '\0';
Py_DECREF(utf8);
if (utf8len == 0)
return NULL; /* EOF */
return s;
#endif
}
/* Set the readline function for TOK to a StreamReader's
readline function. The StreamReader is named ENC.
This function is called from check_bom and check_coding_spec.
ENC is usually identical to the future value of tok->encoding,
except for the (currently unsupported) case of UTF-16.
Return 1 on success, 0 on failure. */
static int
fp_setreadl(struct tok_state *tok, const char* enc)
{
PyObject *reader, *stream, *readline;
/* XXX: constify filename argument. */
stream = PyFile_FromFile(tok->fp, (char*)tok->filename, "rb", NULL);
if (stream == NULL)
return 0;
reader = PyCodec_StreamReader(enc, stream, NULL);
Py_DECREF(stream);
if (reader == NULL)
return 0;
readline = PyObject_GetAttrString(reader, "readline");
Py_DECREF(reader);
if (readline == NULL)
return 0;
tok->decoding_readline = readline;
return 1;
}
/* Fetch the next byte from TOK. */
static int fp_getc(struct tok_state *tok) {
return getc(tok->fp);
}
/* Unfetch the last byte back into TOK. */
static void fp_ungetc(int c, struct tok_state *tok) {
ungetc(c, tok->fp);
}
/* Read a line of input from TOK. Determine encoding
if necessary. */
static char *
decoding_fgets(char *s, int size, struct tok_state *tok)
{
char *line = NULL;
int badchar = 0;
for (;;) {
if (tok->decoding_state < 0) {
/* We already have a codec associated with
this input. */
line = fp_readl(s, size, tok);
break;
} else if (tok->decoding_state > 0) {
/* We want a 'raw' read. */
line = Py_UniversalNewlineFgets(s, size,
tok->fp, NULL);
break;
} else {
/* We have not yet determined the encoding.
If an encoding is found, use the file-pointer
reader functions from now on. */
if (!check_bom(fp_getc, fp_ungetc, fp_setreadl, tok))
return error_ret(tok);
assert(tok->decoding_state != 0);
}
}
if (line != NULL && tok->lineno < 2 && !tok->read_coding_spec) {
if (!check_coding_spec(line, strlen(line), tok, fp_setreadl)) {
return error_ret(tok);
}
}
#ifndef PGEN
/* The default encoding is ASCII, so make sure we don't have any
non-ASCII bytes in it. */
if (line && !tok->encoding) {
unsigned char *c;
for (c = (unsigned char *)line; *c; c++)
if (*c > 127) {
badchar = *c;
break;
}
}
if (badchar) {
char buf[500];
/* Need to add 1 to the line number, since this line
has not been counted, yet. */
sprintf(buf,
"Non-ASCII character '\\x%.2x' "
"in file %.200s on line %i, "
"but no encoding declared; "
"see http://www.python.org/peps/pep-0263.html for details",
badchar, tok->filename, tok->lineno + 1);
PyErr_SetString(PyExc_SyntaxError, buf);
return error_ret(tok);
}
#endif
return line;
}
static int
decoding_feof(struct tok_state *tok)
{
if (tok->decoding_state >= 0) {
return feof(tok->fp);
} else {
PyObject* buf = tok->decoding_buffer;
if (buf == NULL) {
buf = PyObject_CallObject(tok->decoding_readline, NULL);
if (buf == NULL) {
error_ret(tok);
return 1;
} else {
tok->decoding_buffer = buf;
}
}
return PyObject_Length(buf) == 0;
}
}
/* Fetch a byte from TOK, using the string buffer. */
static int
buf_getc(struct tok_state *tok) {
return Py_CHARMASK(*tok->str++);
}
/* Unfetch a byte from TOK, using the string buffer. */
static void
buf_ungetc(int c, struct tok_state *tok) {
tok->str--;
assert(Py_CHARMASK(*tok->str) == c); /* tok->cur may point to read-only segment */
}
/* Set the readline function for TOK to ENC. For the string-based
tokenizer, this means to just record the encoding. */
static int
buf_setreadl(struct tok_state *tok, const char* enc) {
tok->enc = enc;
return 1;
}
/* Return a UTF-8 encoding Python string object from the
C byte string STR, which is encoded with ENC. */
#ifdef Py_USING_UNICODE
static PyObject *
translate_into_utf8(const char* str, const char* enc) {
PyObject *utf8;
PyObject* buf = PyUnicode_Decode(str, strlen(str), enc, NULL);
if (buf == NULL)
return NULL;
utf8 = PyUnicode_AsUTF8String(buf);
Py_DECREF(buf);
return utf8;
}
#endif
static char *
translate_newlines(const char *s, int exec_input, struct tok_state *tok) {
int skip_next_lf = 0, needed_length = strlen(s) + 2, final_length;
char *buf, *current;
char c = '\0';
buf = PyMem_MALLOC(needed_length);
if (buf == NULL) {
tok->done = E_NOMEM;
return NULL;
}
for (current = buf; *s; s++, current++) {
c = *s;
if (skip_next_lf) {
skip_next_lf = 0;
if (c == '\n') {
c = *++s;
if (!c)
break;
}
}
if (c == '\r') {
skip_next_lf = 1;
c = '\n';
}
*current = c;
}
/* If this is exec input, add a newline to the end of the string if
there isn't one already. */
if (exec_input && c != '\n') {
*current = '\n';
current++;
}
*current = '\0';
final_length = current - buf + 1;
if (final_length < needed_length && final_length)
/* should never fail */
buf = PyMem_REALLOC(buf, final_length);
return buf;
}
/* Decode a byte string STR for use as the buffer of TOK.
Look for encoding declarations inside STR, and record them
inside TOK. */
static const char *
decode_str(const char *input, int single, struct tok_state *tok)
{
PyObject* utf8 = NULL;
const char *str;
const char *s;
const char *newl[2] = {NULL, NULL};
int lineno = 0;
tok->input = str = translate_newlines(input, single, tok);
if (str == NULL)
return NULL;
tok->enc = NULL;
tok->str = str;
if (!check_bom(buf_getc, buf_ungetc, buf_setreadl, tok))
return error_ret(tok);
str = tok->str; /* string after BOM if any */
assert(str);
#ifdef Py_USING_UNICODE
if (tok->enc != NULL) {
utf8 = translate_into_utf8(str, tok->enc);
if (utf8 == NULL)
return error_ret(tok);
str = PyString_AsString(utf8);
}
#endif
for (s = str;; s++) {
if (*s == '\0') break;
else if (*s == '\n') {
assert(lineno < 2);
newl[lineno] = s;
lineno++;
if (lineno == 2) break;
}
}
tok->enc = NULL;
/* need to check line 1 and 2 separately since check_coding_spec
assumes a single line as input */
if (newl[0]) {
if (!check_coding_spec(str, newl[0] - str, tok, buf_setreadl))
return error_ret(tok);
if (tok->enc == NULL && newl[1]) {
if (!check_coding_spec(newl[0]+1, newl[1] - newl[0],
tok, buf_setreadl))
return error_ret(tok);
}
}
#ifdef Py_USING_UNICODE
if (tok->enc != NULL) {
assert(utf8 == NULL);
utf8 = translate_into_utf8(str, tok->enc);
if (utf8 == NULL)
return error_ret(tok);
str = PyString_AsString(utf8);
}
#endif
assert(tok->decoding_buffer == NULL);
tok->decoding_buffer = utf8; /* CAUTION */
return str;
}
#endif /* PGEN */
/* Set up tokenizer for string */
struct tok_state *
PyTokenizer_FromString(const char *str, int exec_input)
{
struct tok_state *tok = tok_new();
if (tok == NULL)
return NULL;
str = (char *)decode_str(str, exec_input, tok);
if (str == NULL) {
PyTokenizer_Free(tok);
return NULL;
}
/* XXX: constify members. */
tok->buf = tok->cur = tok->end = tok->inp = (char*)str;
return tok;
}
/* Set up tokenizer for file */
struct tok_state *
PyTokenizer_FromFile(FILE *fp, char *ps1, char *ps2)
{
struct tok_state *tok = tok_new();
if (tok == NULL)
return NULL;
if ((tok->buf = (char *)PyMem_MALLOC(BUFSIZ)) == NULL) {
PyTokenizer_Free(tok);
return NULL;
}
tok->cur = tok->inp = tok->buf;
tok->end = tok->buf + BUFSIZ;
tok->fp = fp;
tok->prompt = ps1;
tok->nextprompt = ps2;
return tok;
}
/* Free a tok_state structure */
void
PyTokenizer_Free(struct tok_state *tok)
{
if (tok->encoding != NULL)
PyMem_FREE(tok->encoding);
#ifndef PGEN
Py_XDECREF(tok->decoding_readline);
Py_XDECREF(tok->decoding_buffer);
#endif
if (tok->fp != NULL && tok->buf != NULL)
PyMem_FREE(tok->buf);
if (tok->input)
PyMem_FREE((char *)tok->input);
PyMem_FREE(tok);
}
#if !defined(PGEN) && defined(Py_USING_UNICODE)
static int
tok_stdin_decode(struct tok_state *tok, char **inp)
{
PyObject *enc, *sysstdin, *decoded, *utf8;
const char *encoding;
char *converted;
if (PySys_GetFile((char *)"stdin", NULL) != stdin)
return 0;
sysstdin = PySys_GetObject("stdin");
if (sysstdin == NULL || !PyFile_Check(sysstdin))
return 0;
enc = ((PyFileObject *)sysstdin)->f_encoding;
if (enc == NULL || !PyString_Check(enc))
return 0;
Py_INCREF(enc);
encoding = PyString_AsString(enc);
decoded = PyUnicode_Decode(*inp, strlen(*inp), encoding, NULL);
if (decoded == NULL)
goto error_clear;
utf8 = PyUnicode_AsEncodedString(decoded, "utf-8", NULL);
Py_DECREF(decoded);
if (utf8 == NULL)
goto error_clear;
assert(PyString_Check(utf8));
converted = new_string(PyString_AS_STRING(utf8),
PyString_GET_SIZE(utf8));
Py_DECREF(utf8);
if (converted == NULL)
goto error_nomem;
PyMem_FREE(*inp);
*inp = converted;
if (tok->encoding != NULL)
PyMem_FREE(tok->encoding);
tok->encoding = new_string(encoding, strlen(encoding));
if (tok->encoding == NULL)
goto error_nomem;
Py_DECREF(enc);
return 0;
error_nomem:
Py_DECREF(enc);
tok->done = E_NOMEM;
return -1;
error_clear:
Py_DECREF(enc);
if (!PyErr_ExceptionMatches(PyExc_UnicodeDecodeError)) {
tok->done = E_ERROR;
return -1;
}
/* Fallback to iso-8859-1: for backward compatibility */
PyErr_Clear();
return 0;
}
#endif
/* Get next char, updating state; error code goes into tok->done */
static int
tok_nextc(register struct tok_state *tok)
{
for (;;) {
if (tok->cur != tok->inp) {
return Py_CHARMASK(*tok->cur++); /* Fast path */
}
if (tok->done != E_OK)
return EOF;
if (tok->fp == NULL) {
char *end = strchr(tok->inp, '\n');
if (end != NULL)
end++;
else {
end = strchr(tok->inp, '\0');
if (end == tok->inp) {
tok->done = E_EOF;
return EOF;
}
}
if (tok->start == NULL)
tok->buf = tok->cur;
tok->line_start = tok->cur;
tok->lineno++;
tok->inp = end;
return Py_CHARMASK(*tok->cur++);
}
if (tok->prompt != NULL) {
char *newtok = PyOS_Readline(stdin, stdout, tok->prompt);
if (tok->nextprompt != NULL)
tok->prompt = tok->nextprompt;
if (newtok == NULL)
tok->done = E_INTR;
else if (*newtok == '\0') {
PyMem_FREE(newtok);
tok->done = E_EOF;
}
#if !defined(PGEN) && defined(Py_USING_UNICODE)
else if (tok_stdin_decode(tok, &newtok) != 0)
PyMem_FREE(newtok);
#endif
else if (tok->start != NULL) {
size_t start = tok->start - tok->buf;
size_t oldlen = tok->cur - tok->buf;
size_t newlen = oldlen + strlen(newtok);
char *buf = tok->buf;
buf = (char *)PyMem_REALLOC(buf, newlen+1);
tok->lineno++;
if (buf == NULL) {
PyMem_FREE(tok->buf);
tok->buf = NULL;
PyMem_FREE(newtok);
tok->done = E_NOMEM;
return EOF;
}
tok->buf = buf;
tok->cur = tok->buf + oldlen;
tok->line_start = tok->cur;
strcpy(tok->buf + oldlen, newtok);
PyMem_FREE(newtok);
tok->inp = tok->buf + newlen;
tok->end = tok->inp + 1;
tok->start = tok->buf + start;
}
else {
tok->lineno++;
if (tok->buf != NULL)
PyMem_FREE(tok->buf);
tok->buf = newtok;
tok->line_start = tok->buf;
tok->cur = tok->buf;
tok->line_start = tok->buf;
tok->inp = strchr(tok->buf, '\0');
tok->end = tok->inp + 1;
}
}
else {
int done = 0;
Py_ssize_t cur = 0;
char *pt;
if (tok->start == NULL) {
if (tok->buf == NULL) {
tok->buf = (char *)
PyMem_MALLOC(BUFSIZ);
if (tok->buf == NULL) {
tok->done = E_NOMEM;
return EOF;
}
tok->end = tok->buf + BUFSIZ;
}
if (decoding_fgets(tok->buf, (int)(tok->end - tok->buf),
tok) == NULL) {
tok->done = E_EOF;
done = 1;
}
else {
tok->done = E_OK;
tok->inp = strchr(tok->buf, '\0');
done = tok->inp[-1] == '\n';
}
}
else {
cur = tok->cur - tok->buf;
if (decoding_feof(tok)) {
tok->done = E_EOF;
done = 1;
}
else
tok->done = E_OK;
}
tok->lineno++;
/* Read until '\n' or EOF */
while (!done) {
Py_ssize_t curstart = tok->start == NULL ? -1 :
tok->start - tok->buf;
Py_ssize_t curvalid = tok->inp - tok->buf;
Py_ssize_t newsize = curvalid + BUFSIZ;
char *newbuf = tok->buf;
newbuf = (char *)PyMem_REALLOC(newbuf,
newsize);
if (newbuf == NULL) {
tok->done = E_NOMEM;
tok->cur = tok->inp;
return EOF;
}
tok->buf = newbuf;
tok->inp = tok->buf + curvalid;
tok->end = tok->buf + newsize;
tok->start = curstart < 0 ? NULL :
tok->buf + curstart;
if (decoding_fgets(tok->inp,
(int)(tok->end - tok->inp),
tok) == NULL) {
/* Break out early on decoding
errors, as tok->buf will be NULL
*/
if (tok->decoding_erred)
return EOF;
/* Last line does not end in \n,
fake one */
strcpy(tok->inp, "\n");
}
tok->inp = strchr(tok->inp, '\0');
done = tok->inp[-1] == '\n';
}
if (tok->buf != NULL) {
tok->cur = tok->buf + cur;
tok->line_start = tok->cur;
/* replace "\r\n" with "\n" */
/* For Mac leave the \r, giving a syntax error */
pt = tok->inp - 2;
if (pt >= tok->buf && *pt == '\r') {
*pt++ = '\n';
*pt = '\0';
tok->inp = pt;
}
}
}
if (tok->done != E_OK) {
if (tok->prompt != NULL)
PySys_WriteStderr("\n");
tok->cur = tok->inp;
return EOF;
}
}
/*NOTREACHED*/
}
/* Back-up one character */
static void
tok_backup(register struct tok_state *tok, register int c)
{
if (c != EOF) {
if (--tok->cur < tok->buf)
Py_FatalError("tok_backup: beginning of buffer");
if (*tok->cur != c)
*tok->cur = c;
}
}
/* Return the token corresponding to a single character */
int
PyToken_OneChar(int c)
{
switch (c) {
case '(': return LPAR;
case ')': return RPAR;
case '[': return LSQB;
case ']': return RSQB;
case ':': return COLON;
case ',': return COMMA;
case ';': return SEMI;
case '+': return PLUS;
case '-': return MINUS;
case '*': return STAR;
case '/': return SLASH;
case '|': return VBAR;
case '&': return AMPER;
case '<': return LESS;
case '>': return GREATER;
case '=': return EQUAL;
case '.': return DOT;
case '%': return PERCENT;
case '`': return BACKQUOTE;
case '{': return LBRACE;
case '}': return RBRACE;
case '^': return CIRCUMFLEX;
case '~': return TILDE;
case '@': return AT;
default: return OP;
}
}
int
PyToken_TwoChars(int c1, int c2)
{
switch (c1) {
case '=':
switch (c2) {
case '=': return EQEQUAL;
}
break;
case '!':
switch (c2) {
case '=': return NOTEQUAL;
}
break;
case '<':
switch (c2) {
case '>': return NOTEQUAL;
case '=': return LESSEQUAL;
case '<': return LEFTSHIFT;
}
break;
case '>':
switch (c2) {
case '=': return GREATEREQUAL;
case '>': return RIGHTSHIFT;
}
break;
case '+':
switch (c2) {
case '=': return PLUSEQUAL;
}
break;
case '-':
switch (c2) {
case '=': return MINEQUAL;
}
break;
case '*':
switch (c2) {
case '*': return DOUBLESTAR;
case '=': return STAREQUAL;
}
break;
case '/':
switch (c2) {
case '/': return DOUBLESLASH;
case '=': return SLASHEQUAL;
}
break;
case '|':
switch (c2) {
case '=': return VBAREQUAL;
}
break;
case '%':
switch (c2) {
case '=': return PERCENTEQUAL;
}
break;
case '&':
switch (c2) {
case '=': return AMPEREQUAL;
}
break;
case '^':
switch (c2) {
case '=': return CIRCUMFLEXEQUAL;
}
break;
}
return OP;
}
int
PyToken_ThreeChars(int c1, int c2, int c3)
{
switch (c1) {
case '<':
switch (c2) {
case '<':
switch (c3) {
case '=':
return LEFTSHIFTEQUAL;
}
break;
}
break;
case '>':
switch (c2) {
case '>':
switch (c3) {
case '=':
return RIGHTSHIFTEQUAL;
}
break;
}
break;
case '*':
switch (c2) {
case '*':
switch (c3) {
case '=':
return DOUBLESTAREQUAL;
}
break;
}
break;
case '/':
switch (c2) {
case '/':
switch (c3) {
case '=':
return DOUBLESLASHEQUAL;
}
break;
}
break;
}
return OP;
}
static int
indenterror(struct tok_state *tok)
{
if (tok->alterror) {
tok->done = E_TABSPACE;
tok->cur = tok->inp;
return 1;
}
if (tok->altwarning) {
PySys_WriteStderr("%s: inconsistent use of tabs and spaces "
"in indentation\n", tok->filename);
tok->altwarning = 0;
}
return 0;
}
/* Get next token, after space stripping etc. */
static int
tok_get(register struct tok_state *tok, char **p_start, char **p_end)
{
register int c;
int blankline;
*p_start = *p_end = NULL;
nextline:
tok->start = NULL;
blankline = 0;
/* Get indentation level */
if (tok->atbol) {
register int col = 0;
register int altcol = 0;
tok->atbol = 0;
for (;;) {
c = tok_nextc(tok);
if (c == ' ')
col++, altcol++;
else if (c == '\t') {
col = (col/tok->tabsize + 1) * tok->tabsize;
altcol = (altcol/tok->alttabsize + 1)
* tok->alttabsize;
}
else if (c == '\014') /* Control-L (formfeed) */
col = altcol = 0; /* For Emacs users */
else
break;
}
tok_backup(tok, c);
if (c == '#' || c == '\n') {
/* Lines with only whitespace and/or comments
shouldn't affect the indentation and are
not passed to the parser as NEWLINE tokens,
except *totally* empty lines in interactive
mode, which signal the end of a command group. */
if (col == 0 && c == '\n' && tok->prompt != NULL)
blankline = 0; /* Let it through */
else
blankline = 1; /* Ignore completely */
/* We can't jump back right here since we still
may need to skip to the end of a comment */
}
if (!blankline && tok->level == 0) {
if (col == tok->indstack[tok->indent]) {
/* No change */
if (altcol != tok->altindstack[tok->indent]) {
if (indenterror(tok))
return ERRORTOKEN;
}
}
else if (col > tok->indstack[tok->indent]) {
/* Indent -- always one */
if (tok->indent+1 >= MAXINDENT) {
tok->done = E_TOODEEP;
tok->cur = tok->inp;
return ERRORTOKEN;
}
if (altcol <= tok->altindstack[tok->indent]) {
if (indenterror(tok))
return ERRORTOKEN;
}
tok->pendin++;
tok->indstack[++tok->indent] = col;
tok->altindstack[tok->indent] = altcol;
}
else /* col < tok->indstack[tok->indent] */ {
/* Dedent -- any number, must be consistent */
while (tok->indent > 0 &&
col < tok->indstack[tok->indent]) {
tok->pendin--;
tok->indent--;
}
if (col != tok->indstack[tok->indent]) {
tok->done = E_DEDENT;
tok->cur = tok->inp;
return ERRORTOKEN;
}
if (altcol != tok->altindstack[tok->indent]) {
if (indenterror(tok))
return ERRORTOKEN;
}
}
}
}
tok->start = tok->cur;
/* Return pending indents/dedents */
if (tok->pendin != 0) {
if (tok->pendin < 0) {
tok->pendin++;
return DEDENT;
}
else {
tok->pendin--;
return INDENT;
}
}
again:
tok->start = NULL;
/* Skip spaces */
do {
c = tok_nextc(tok);
} while (c == ' ' || c == '\t' || c == '\014');
/* Set start of current token */
tok->start = tok->cur - 1;
/* Skip comment, while looking for tab-setting magic */
if (c == '#') {
static char *tabforms[] = {
"tab-width:", /* Emacs */
":tabstop=", /* vim, full form */
":ts=", /* vim, abbreviated form */
"set tabsize=", /* will vi never die? */
/* more templates can be added here to support other editors */
};
char cbuf[80];
char *tp, **cp;
tp = cbuf;
do {
*tp++ = c = tok_nextc(tok);
} while (c != EOF && c != '\n' &&
(size_t)(tp - cbuf + 1) < sizeof(cbuf));
*tp = '\0';
for (cp = tabforms;
cp < tabforms + sizeof(tabforms)/sizeof(tabforms[0]);
cp++) {
if ((tp = strstr(cbuf, *cp))) {
int newsize = atoi(tp + strlen(*cp));
if (newsize >= 1 && newsize <= 40) {
tok->tabsize = newsize;
if (Py_VerboseFlag)
PySys_WriteStderr(
"Tab size set to %d\n",
newsize);
}
}
}
while (c != EOF && c != '\n')
c = tok_nextc(tok);
}
/* Check for EOF and errors now */
if (c == EOF) {
return tok->done == E_EOF ? ENDMARKER : ERRORTOKEN;
}
/* Identifier (most frequent token!) */
if (Py_ISALPHA(c) || c == '_') {
/* Process r"", u"" and ur"" */
switch (c) {
case 'b':
case 'B':
c = tok_nextc(tok);
if (c == 'r' || c == 'R')
c = tok_nextc(tok);
if (c == '"' || c == '\'')
goto letter_quote;
break;
case 'r':
case 'R':
c = tok_nextc(tok);
if (c == '"' || c == '\'')
goto letter_quote;
break;
case 'u':
case 'U':
c = tok_nextc(tok);
if (c == 'r' || c == 'R')
c = tok_nextc(tok);
if (c == '"' || c == '\'')
goto letter_quote;
break;
}
while (c != EOF && (Py_ISALNUM(c) || c == '_')) {
c = tok_nextc(tok);
}
tok_backup(tok, c);
*p_start = tok->start;
*p_end = tok->cur;
return NAME;
}
/* Newline */
if (c == '\n') {
tok->atbol = 1;
if (blankline || tok->level > 0)
goto nextline;
*p_start = tok->start;
*p_end = tok->cur - 1; /* Leave '\n' out of the string */
tok->cont_line = 0;
return NEWLINE;
}
/* Period or number starting with period? */
if (c == '.') {
c = tok_nextc(tok);
if (isdigit(c)) {
goto fraction;
}
else {
tok_backup(tok, c);
*p_start = tok->start;
*p_end = tok->cur;
return DOT;
}
}
/* Number */
if (isdigit(c)) {
if (c == '0') {
/* Hex, octal or binary -- maybe. */
c = tok_nextc(tok);
if (c == '.')
goto fraction;
#ifndef WITHOUT_COMPLEX
if (c == 'j' || c == 'J')
goto imaginary;
#endif
if (c == 'x' || c == 'X') {
/* Hex */
c = tok_nextc(tok);
if (!isxdigit(c)) {
tok->done = E_TOKEN;
tok_backup(tok, c);
return ERRORTOKEN;
}
do {
c = tok_nextc(tok);
} while (isxdigit(c));
}
else if (c == 'o' || c == 'O') {
/* Octal */
c = tok_nextc(tok);
if (c < '0' || c >= '8') {
tok->done = E_TOKEN;
tok_backup(tok, c);
return ERRORTOKEN;
}
do {
c = tok_nextc(tok);
} while ('0' <= c && c < '8');
}
else if (c == 'b' || c == 'B') {
/* Binary */
c = tok_nextc(tok);
if (c != '0' && c != '1') {
tok->done = E_TOKEN;
tok_backup(tok, c);
return ERRORTOKEN;
}
do {
c = tok_nextc(tok);
} while (c == '0' || c == '1');
}
else {
int found_decimal = 0;
/* Octal; c is first char of it */
/* There's no 'isoctdigit' macro, sigh */
while ('0' <= c && c < '8') {
c = tok_nextc(tok);
}
if (isdigit(c)) {
found_decimal = 1;
do {
c = tok_nextc(tok);
} while (isdigit(c));
}
if (c == '.')
goto fraction;
else if (c == 'e' || c == 'E')
goto exponent;
#ifndef WITHOUT_COMPLEX
else if (c == 'j' || c == 'J')
goto imaginary;
#endif
else if (found_decimal) {
tok->done = E_TOKEN;
tok_backup(tok, c);
return ERRORTOKEN;
}
}
if (c == 'l' || c == 'L')
c = tok_nextc(tok);
}
else {
/* Decimal */
do {
c = tok_nextc(tok);
} while (isdigit(c));
if (c == 'l' || c == 'L')
c = tok_nextc(tok);
else {
/* Accept floating point numbers. */
if (c == '.') {
fraction:
/* Fraction */
do {
c = tok_nextc(tok);
} while (isdigit(c));
}
if (c == 'e' || c == 'E') {
exponent:
/* Exponent part */
c = tok_nextc(tok);
if (c == '+' || c == '-')
c = tok_nextc(tok);
if (!isdigit(c)) {
tok->done = E_TOKEN;
tok_backup(tok, c);
return ERRORTOKEN;
}
do {
c = tok_nextc(tok);
} while (isdigit(c));
}
#ifndef WITHOUT_COMPLEX
if (c == 'j' || c == 'J')
/* Imaginary part */
imaginary:
c = tok_nextc(tok);
#endif
}
}
tok_backup(tok, c);
*p_start = tok->start;
*p_end = tok->cur;
return NUMBER;
}
letter_quote:
/* String */
if (c == '\'' || c == '"') {
Py_ssize_t quote2 = tok->cur - tok->start + 1;
int quote = c;
int triple = 0;
int tripcount = 0;
for (;;) {
c = tok_nextc(tok);
if (c == '\n') {
if (!triple) {
tok->done = E_EOLS;
tok_backup(tok, c);
return ERRORTOKEN;
}
tripcount = 0;
tok->cont_line = 1; /* multiline string. */
}
else if (c == EOF) {
if (triple)
tok->done = E_EOFS;
else
tok->done = E_EOLS;
tok->cur = tok->inp;
return ERRORTOKEN;
}
else if (c == quote) {
tripcount++;
if (tok->cur - tok->start == quote2) {
c = tok_nextc(tok);
if (c == quote) {
triple = 1;
tripcount = 0;
continue;
}
tok_backup(tok, c);
}
if (!triple || tripcount == 3)
break;
}
else if (c == '\\') {
tripcount = 0;
c = tok_nextc(tok);
if (c == EOF) {
tok->done = E_EOLS;
tok->cur = tok->inp;
return ERRORTOKEN;
}
}
else
tripcount = 0;
}
*p_start = tok->start;
*p_end = tok->cur;
return STRING;
}
/* Line continuation */
if (c == '\\') {
c = tok_nextc(tok);
if (c != '\n') {
tok->done = E_LINECONT;
tok->cur = tok->inp;
return ERRORTOKEN;
}
tok->cont_line = 1;
goto again; /* Read next line */
}
/* Check for two-character token */
{
int c2 = tok_nextc(tok);
int token = PyToken_TwoChars(c, c2);
#ifndef PGEN
if (Py_Py3kWarningFlag && token == NOTEQUAL && c == '<') {
if (PyErr_WarnExplicit(PyExc_DeprecationWarning,
"<> not supported in 3.x; use !=",
tok->filename, tok->lineno,
NULL, NULL)) {
return ERRORTOKEN;
}
}
#endif
if (token != OP) {
int c3 = tok_nextc(tok);
int token3 = PyToken_ThreeChars(c, c2, c3);
if (token3 != OP) {
token = token3;
} else {
tok_backup(tok, c3);
}
*p_start = tok->start;
*p_end = tok->cur;
return token;
}
tok_backup(tok, c2);
}
/* Keep track of parentheses nesting level */
switch (c) {
case '(':
case '[':
case '{':
tok->level++;
break;
case ')':
case ']':
case '}':
tok->level--;
break;
}
/* Punctuation character */
*p_start = tok->start;
*p_end = tok->cur;
return PyToken_OneChar(c);
}
int
PyTokenizer_Get(struct tok_state *tok, char **p_start, char **p_end)
{
int result = tok_get(tok, p_start, p_end);
if (tok->decoding_erred) {
result = ERRORTOKEN;
tok->done = E_DECODE;
}
return result;
}
/* This function is only called from parsetok. However, it cannot live
there, as it must be empty for PGEN, and we can check for PGEN only
in this file. */
#if defined(PGEN) || !defined(Py_USING_UNICODE)
char*
PyTokenizer_RestoreEncoding(struct tok_state* tok, int len, int* offset)
{
return NULL;
}
#else
#ifdef Py_USING_UNICODE
static PyObject *
dec_utf8(const char *enc, const char *text, size_t len) {
PyObject *ret = NULL;
PyObject *unicode_text = PyUnicode_DecodeUTF8(text, len, "replace");
if (unicode_text) {
ret = PyUnicode_AsEncodedString(unicode_text, enc, "replace");
Py_DECREF(unicode_text);
}
if (!ret) {
PyErr_Clear();
}
return ret;
}
char *
PyTokenizer_RestoreEncoding(struct tok_state* tok, int len, int *offset)
{
char *text = NULL;
if (tok->encoding) {
/* convert source to original encondig */
PyObject *lineobj = dec_utf8(tok->encoding, tok->buf, len);
if (lineobj != NULL) {
int linelen = PyString_Size(lineobj);
const char *line = PyString_AsString(lineobj);
text = PyObject_MALLOC(linelen + 1);
if (text != NULL && line != NULL) {
if (linelen)
strncpy(text, line, linelen);
text[linelen] = '\0';
}
Py_DECREF(lineobj);
/* adjust error offset */
if (*offset > 1) {
PyObject *offsetobj = dec_utf8(tok->encoding,
tok->buf, *offset-1);
if (offsetobj) {
*offset = PyString_Size(offsetobj) + 1;
Py_DECREF(offsetobj);
}
}
}
}
return text;
}
#endif /* defined(Py_USING_UNICODE) */
#endif
#ifdef Py_DEBUG
void
tok_dump(int type, char *start, char *end)
{
printf("%s", _PyParser_TokenNames[type]);
if (type == NAME || type == NUMBER || type == STRING || type == OP)
printf("(%.*s)", (int)(end - start), start);
}
#endif
from_cpython/Parser/tokenizer.h 0 → 100644
View file @ 3fa45d83
#ifndef Py_TOKENIZER_H
#define Py_TOKENIZER_H
#ifdef __cplusplus
extern "C" {
#endif
#include "object.h"
/* Tokenizer interface */
#include "token.h" /* For token types */
#define MAXINDENT 100 /* Max indentation level */
/* Tokenizer state */
struct tok_state {
/* Input state; buf <= cur <= inp <= end */
/* NB an entire line is held in the buffer */
char *buf; /* Input buffer, or NULL; malloc'ed if fp != NULL */
char *cur; /* Next character in buffer */
char *inp; /* End of data in buffer */
char *end; /* End of input buffer if buf != NULL */
char *start; /* Start of current token if not NULL */
int done; /* E_OK normally, E_EOF at EOF, otherwise error code */
/* NB If done != E_OK, cur must be == inp!!! */
FILE *fp; /* Rest of input; NULL if tokenizing a string */
int tabsize; /* Tab spacing */
int indent; /* Current indentation index */
int indstack[MAXINDENT]; /* Stack of indents */
int atbol; /* Nonzero if at begin of new line */
int pendin; /* Pending indents (if > 0) or dedents (if < 0) */
char *prompt, *nextprompt; /* For interactive prompting */
int lineno; /* Current line number */
int level; /* () [] {} Parentheses nesting level */
/* Used to allow free continuations inside them */
/* Stuff for checking on different tab sizes */
const char *filename; /* For error messages */
int altwarning; /* Issue warning if alternate tabs don't match */
int alterror; /* Issue error if alternate tabs don't match */
int alttabsize; /* Alternate tab spacing */
int altindstack[MAXINDENT]; /* Stack of alternate indents */
/* Stuff for PEP 0263 */
int decoding_state; /* -1:decoding, 0:init, 1:raw */
int decoding_erred; /* whether erred in decoding */
int read_coding_spec; /* whether 'coding:...' has been read */
char *encoding;
int cont_line; /* whether we are in a continuation line. */
const char* line_start; /* pointer to start of current line */
#ifndef PGEN
PyObject *decoding_readline; /* codecs.open(...).readline */
PyObject *decoding_buffer;
#endif
const char* enc;
const char* str;
const char* input; /* Tokenizer's newline translated copy of the string. */
};
extern struct tok_state *PyTokenizer_FromString(const char *, int);
extern struct tok_state *PyTokenizer_FromFile(FILE *, char *, char *);
extern void PyTokenizer_Free(struct tok_state *);
extern int PyTokenizer_Get(struct tok_state *, char **, char **);
#if defined(PGEN) || defined(Py_USING_UNICODE)
extern char * PyTokenizer_RestoreEncoding(struct tok_state* tok,
int len, int *offset);
#endif
#ifdef __cplusplus
}
#endif
#endif /* !Py_TOKENIZER_H */
from_cpython/Parser/tokenizer_pgen.c 0 → 100644
View file @ 3fa45d83
#define PGEN
#include "tokenizer.c"
from_cpython/Python/Python-ast.c 0 → 100644
View file @ 3fa45d83
This source diff could not be displayed because it is too large. You can view the blob instead.
from_cpython/Python/asdl.c 0 → 100644
View file @ 3fa45d83
#include "Python.h"
#include "asdl.h"
asdl_seq *
asdl_seq_new(int size, PyArena *arena)
{
asdl_seq *seq = NULL;
size_t n = (size ? (sizeof(void *) * (size - 1)) : 0);
/* check size is sane */
if (size < 0 || size == INT_MIN ||
(size && ((size - 1) > (PY_SIZE_MAX / sizeof(void *))))) {
PyErr_NoMemory();
return NULL;
}
/* check if size can be added safely */
if (n > PY_SIZE_MAX - sizeof(asdl_seq)) {
PyErr_NoMemory();
return NULL;
}
n += sizeof(asdl_seq);
seq = (asdl_seq *)PyArena_Malloc(arena, n);
if (!seq) {
PyErr_NoMemory();
return NULL;
}
memset(seq, 0, n);
seq->size = size;
return seq;
}
asdl_int_seq *
asdl_int_seq_new(int size, PyArena *arena)
{
asdl_int_seq *seq = NULL;
size_t n = (size ? (sizeof(void *) * (size - 1)) : 0);
/* check size is sane */
if (size < 0 || size == INT_MIN ||
(size && ((size - 1) > (PY_SIZE_MAX / sizeof(void *))))) {
PyErr_NoMemory();
return NULL;
}
/* check if size can be added safely */
if (n > PY_SIZE_MAX - sizeof(asdl_seq)) {
PyErr_NoMemory();
return NULL;
}
n += sizeof(asdl_seq);
seq = (asdl_int_seq *)PyArena_Malloc(arena, n);
if (!seq) {
PyErr_NoMemory();
return NULL;
}
memset(seq, 0, n);
seq->size = size;
return seq;
}
from_cpython/Python/ast.c 0 → 100644
View file @ 3fa45d83
This source diff could not be displayed because it is too large. You can view the blob instead.
from_cpython/Python/graminit.c 0 → 100644
View file @ 3fa45d83
/* Generated by Parser/pgen */
#include "pgenheaders.h"
#include "grammar.h"
PyAPI_DATA(grammar) _PyParser_Grammar;
static arc arcs_0_0[3] = {
{2, 1},
{3, 1},
{4, 2},
};
static arc arcs_0_1[1] = {
{0, 1},
};
static arc arcs_0_2[1] = {
{2, 1},
};
static state states_0[3] = {
{3, arcs_0_0},
{1, arcs_0_1},
{1, arcs_0_2},
};
static arc arcs_1_0[3] = {
{2, 0},
{6, 0},
{7, 1},
};
static arc arcs_1_1[1] = {
{0, 1},
};
static state states_1[2] = {
{3, arcs_1_0},
{1, arcs_1_1},
};
static arc arcs_2_0[1] = {
{9, 1},
};
static arc arcs_2_1[2] = {
{2, 1},
{7, 2},
};
static arc arcs_2_2[1] = {
{0, 2},
};
static state states_2[3] = {
{1, arcs_2_0},
{2, arcs_2_1},
{1, arcs_2_2},
};
static arc arcs_3_0[1] = {
{11, 1},
};
static arc arcs_3_1[1] = {
{12, 2},
};
static arc arcs_3_2[2] = {
{13, 3},
{2, 4},
};
static arc arcs_3_3[2] = {
{14, 5},
{15, 6},
};
static arc arcs_3_4[1] = {
{0, 4},
};
static arc arcs_3_5[1] = {
{15, 6},
};
static arc arcs_3_6[1] = {
{2, 4},
};
static state states_3[7] = {
{1, arcs_3_0},
{1, arcs_3_1},
{2, arcs_3_2},
{2, arcs_3_3},
{1, arcs_3_4},
{1, arcs_3_5},
{1, arcs_3_6},
};
static arc arcs_4_0[1] = {
{10, 1},
};
static arc arcs_4_1[2] = {
{10, 1},
{0, 1},
};
static state states_4[2] = {
{1, arcs_4_0},
{2, arcs_4_1},
};
static arc arcs_5_0[1] = {
{16, 1},
};
static arc arcs_5_1[2] = {
{18, 2},
{19, 2},
};
static arc arcs_5_2[1] = {
{0, 2},
};
static state states_5[3] = {
{1, arcs_5_0},
{2, arcs_5_1},
{1, arcs_5_2},
};
static arc arcs_6_0[1] = {
{20, 1},
};
static arc arcs_6_1[1] = {
{21, 2},
};
static arc arcs_6_2[1] = {
{22, 3},
};
static arc arcs_6_3[1] = {
{23, 4},
};
static arc arcs_6_4[1] = {
{24, 5},
};
static arc arcs_6_5[1] = {
{0, 5},
};
static state states_6[6] = {
{1, arcs_6_0},
{1, arcs_6_1},
{1, arcs_6_2},
{1, arcs_6_3},
{1, arcs_6_4},
{1, arcs_6_5},
};
static arc arcs_7_0[1] = {
{13, 1},
};
static arc arcs_7_1[2] = {
{25, 2},
{15, 3},
};
static arc arcs_7_2[1] = {
{15, 3},
};
static arc arcs_7_3[1] = {
{0, 3},
};
static state states_7[4] = {
{1, arcs_7_0},
{2, arcs_7_1},
{1, arcs_7_2},
{1, arcs_7_3},
};
static arc arcs_8_0[3] = {
{26, 1},
{30, 2},
{31, 3},
};
static arc arcs_8_1[3] = {
{27, 4},
{29, 5},
{0, 1},
};
static arc arcs_8_2[1] = {
{21, 6},
};
static arc arcs_8_3[1] = {
{21, 7},
};
static arc arcs_8_4[1] = {
{28, 8},
};
static arc arcs_8_5[4] = {
{26, 1},
{30, 2},
{31, 3},
{0, 5},
};
static arc arcs_8_6[2] = {
{29, 9},
{0, 6},
};
static arc arcs_8_7[1] = {
{0, 7},
};
static arc arcs_8_8[2] = {
{29, 5},
{0, 8},
};
static arc arcs_8_9[1] = {
{31, 3},
};
static state states_8[10] = {
{3, arcs_8_0},
{3, arcs_8_1},
{1, arcs_8_2},
{1, arcs_8_3},
{1, arcs_8_4},
{4, arcs_8_5},
{2, arcs_8_6},
{1, arcs_8_7},
{2, arcs_8_8},
{1, arcs_8_9},
};
static arc arcs_9_0[2] = {
{21, 1},
{13, 2},
};
static arc arcs_9_1[1] = {
{0, 1},
};
static arc arcs_9_2[1] = {
{32, 3},
};
static arc arcs_9_3[1] = {
{15, 1},
};
static state states_9[4] = {
{2, arcs_9_0},
{1, arcs_9_1},
{1, arcs_9_2},
{1, arcs_9_3},
};
static arc arcs_10_0[1] = {
{26, 1},
};
static arc arcs_10_1[2] = {
{29, 2},
{0, 1},
};
static arc arcs_10_2[2] = {
{26, 1},
{0, 2},
};
static state states_10[3] = {
{1, arcs_10_0},
{2, arcs_10_1},
{2, arcs_10_2},
};
static arc arcs_11_0[2] = {
{3, 1},
{4, 1},
};
static arc arcs_11_1[1] = {
{0, 1},
};
static state states_11[2] = {
{2, arcs_11_0},
{1, arcs_11_1},
};
static arc arcs_12_0[1] = {
{33, 1},
};
static arc arcs_12_1[2] = {
{34, 2},
{2, 3},
};
static arc arcs_12_2[2] = {
{33, 1},
{2, 3},
};
static arc arcs_12_3[1] = {
{0, 3},
};
static state states_12[4] = {
{1, arcs_12_0},
{2, arcs_12_1},
{2, arcs_12_2},
{1, arcs_12_3},
};
static arc arcs_13_0[9] = {
{35, 1},
{36, 1},
{37, 1},
{38, 1},
{39, 1},
{40, 1},
{41, 1},
{42, 1},
{43, 1},
};
static arc arcs_13_1[1] = {
{0, 1},
};
static state states_13[2] = {
{9, arcs_13_0},
{1, arcs_13_1},
};
static arc arcs_14_0[1] = {
{9, 1},
};
static arc arcs_14_1[3] = {
{44, 2},
{27, 3},
{0, 1},
};
static arc arcs_14_2[2] = {
{45, 4},
{9, 4},
};
static arc arcs_14_3[2] = {
{45, 5},
{9, 5},
};
static arc arcs_14_4[1] = {
{0, 4},
};
static arc arcs_14_5[2] = {
{27, 3},
{0, 5},
};
static state states_14[6] = {
{1, arcs_14_0},
{3, arcs_14_1},
{2, arcs_14_2},
{2, arcs_14_3},
{1, arcs_14_4},
{2, arcs_14_5},
};
static arc arcs_15_0[12] = {
{46, 1},
{47, 1},
{48, 1},
{49, 1},
{50, 1},
{51, 1},
{52, 1},
{53, 1},
{54, 1},
{55, 1},
{56, 1},
{57, 1},
};
static arc arcs_15_1[1] = {
{0, 1},
};
static state states_15[2] = {
{12, arcs_15_0},
{1, arcs_15_1},
};
static arc arcs_16_0[1] = {
{58, 1},
};
static arc arcs_16_1[3] = {
{28, 2},
{59, 3},
{0, 1},
};
static arc arcs_16_2[2] = {
{29, 4},
{0, 2},
};
static arc arcs_16_3[1] = {
{28, 5},
};
static arc arcs_16_4[2] = {
{28, 2},
{0, 4},
};
static arc arcs_16_5[2] = {
{29, 6},
{0, 5},
};
static arc arcs_16_6[1] = {
{28, 7},
};
static arc arcs_16_7[2] = {
{29, 8},
{0, 7},
};
static arc arcs_16_8[2] = {
{28, 7},
{0, 8},
};
static state states_16[9] = {
{1, arcs_16_0},
{3, arcs_16_1},
{2, arcs_16_2},
{1, arcs_16_3},
{2, arcs_16_4},
{2, arcs_16_5},
{1, arcs_16_6},
{2, arcs_16_7},
{2, arcs_16_8},
};
static arc arcs_17_0[1] = {
{60, 1},
};
static arc arcs_17_1[1] = {
{61, 2},
};
static arc arcs_17_2[1] = {
{0, 2},
};
static state states_17[3] = {
{1, arcs_17_0},
{1, arcs_17_1},
{1, arcs_17_2},
};
static arc arcs_18_0[1] = {
{62, 1},
};
static arc arcs_18_1[1] = {
{0, 1},
};
static state states_18[2] = {
{1, arcs_18_0},
{1, arcs_18_1},
};
static arc arcs_19_0[5] = {
{63, 1},
{64, 1},
{65, 1},
{66, 1},
{67, 1},
};
static arc arcs_19_1[1] = {
{0, 1},
};
static state states_19[2] = {
{5, arcs_19_0},
{1, arcs_19_1},
};
static arc arcs_20_0[1] = {
{68, 1},
};
static arc arcs_20_1[1] = {
{0, 1},
};
static state states_20[2] = {
{1, arcs_20_0},
{1, arcs_20_1},
};
static arc arcs_21_0[1] = {
{69, 1},
};
static arc arcs_21_1[1] = {
{0, 1},
};
static state states_21[2] = {
{1, arcs_21_0},
{1, arcs_21_1},
};
static arc arcs_22_0[1] = {
{70, 1},
};
static arc arcs_22_1[2] = {
{9, 2},
{0, 1},
};
static arc arcs_22_2[1] = {
{0, 2},
};
static state states_22[3] = {
{1, arcs_22_0},
{2, arcs_22_1},
{1, arcs_22_2},
};
static arc arcs_23_0[1] = {
{45, 1},
};
static arc arcs_23_1[1] = {
{0, 1},
};
static state states_23[2] = {
{1, arcs_23_0},
{1, arcs_23_1},
};
static arc arcs_24_0[1] = {
{71, 1},
};
static arc arcs_24_1[2] = {
{28, 2},
{0, 1},
};
static arc arcs_24_2[2] = {
{29, 3},
{0, 2},
};
static arc arcs_24_3[1] = {
{28, 4},
};
static arc arcs_24_4[2] = {
{29, 5},
{0, 4},
};
static arc arcs_24_5[1] = {
{28, 6},
};
static arc arcs_24_6[1] = {
{0, 6},
};
static state states_24[7] = {
{1, arcs_24_0},
{2, arcs_24_1},
{2, arcs_24_2},
{1, arcs_24_3},
{2, arcs_24_4},
{1, arcs_24_5},
{1, arcs_24_6},
};
static arc arcs_25_0[2] = {
{72, 1},
{73, 1},
};
static arc arcs_25_1[1] = {
{0, 1},
};
static state states_25[2] = {
{2, arcs_25_0},
{1, arcs_25_1},
};
static arc arcs_26_0[1] = {
{74, 1},
};
static arc arcs_26_1[1] = {
{75, 2},
};
static arc arcs_26_2[1] = {
{0, 2},
};
static state states_26[3] = {
{1, arcs_26_0},
{1, arcs_26_1},
{1, arcs_26_2},
};
static arc arcs_27_0[1] = {
{76, 1},
};
static arc arcs_27_1[2] = {
{77, 2},
{12, 3},
};
static arc arcs_27_2[3] = {
{77, 2},
{12, 3},
{74, 4},
};
static arc arcs_27_3[1] = {
{74, 4},
};
static arc arcs_27_4[3] = {
{30, 5},
{13, 6},
{78, 5},
};
static arc arcs_27_5[1] = {
{0, 5},
};
static arc arcs_27_6[1] = {
{78, 7},
};
static arc arcs_27_7[1] = {
{15, 5},
};
static state states_27[8] = {
{1, arcs_27_0},
{2, arcs_27_1},
{3, arcs_27_2},
{1, arcs_27_3},
{3, arcs_27_4},
{1, arcs_27_5},
{1, arcs_27_6},
{1, arcs_27_7},
};
static arc arcs_28_0[1] = {
{21, 1},
};
static arc arcs_28_1[2] = {
{80, 2},
{0, 1},
};
static arc arcs_28_2[1] = {
{21, 3},
};
static arc arcs_28_3[1] = {
{0, 3},
};
static state states_28[4] = {
{1, arcs_28_0},
{2, arcs_28_1},
{1, arcs_28_2},
{1, arcs_28_3},
};
static arc arcs_29_0[1] = {
{12, 1},
};
static arc arcs_29_1[2] = {
{80, 2},
{0, 1},
};
static arc arcs_29_2[1] = {
{21, 3},
};
static arc arcs_29_3[1] = {
{0, 3},
};
static state states_29[4] = {
{1, arcs_29_0},
{2, arcs_29_1},
{1, arcs_29_2},
{1, arcs_29_3},
};
static arc arcs_30_0[1] = {
{79, 1},
};
static arc arcs_30_1[2] = {
{29, 2},
{0, 1},
};
static arc arcs_30_2[2] = {
{79, 1},
{0, 2},
};
static state states_30[3] = {
{1, arcs_30_0},
{2, arcs_30_1},
{2, arcs_30_2},
};
static arc arcs_31_0[1] = {
{81, 1},
};
static arc arcs_31_1[2] = {
{29, 0},
{0, 1},
};
static state states_31[2] = {
{1, arcs_31_0},
{2, arcs_31_1},
};
static arc arcs_32_0[1] = {
{21, 1},
};
static arc arcs_32_1[2] = {
{77, 0},
{0, 1},
};
static state states_32[2] = {
{1, arcs_32_0},
{2, arcs_32_1},
};
static arc arcs_33_0[1] = {
{82, 1},
};
static arc arcs_33_1[1] = {
{21, 2},
};
static arc arcs_33_2[2] = {
{29, 1},
{0, 2},
};
static state states_33[3] = {
{1, arcs_33_0},
{1, arcs_33_1},
{2, arcs_33_2},
};
static arc arcs_34_0[1] = {
{83, 1},
};
static arc arcs_34_1[1] = {
{84, 2},
};
static arc arcs_34_2[2] = {
{85, 3},
{0, 2},
};
static arc arcs_34_3[1] = {
{28, 4},
};
static arc arcs_34_4[2] = {
{29, 5},
{0, 4},
};
static arc arcs_34_5[1] = {
{28, 6},
};
static arc arcs_34_6[1] = {
{0, 6},
};
static state states_34[7] = {
{1, arcs_34_0},
{1, arcs_34_1},
{2, arcs_34_2},
{1, arcs_34_3},
{2, arcs_34_4},
{1, arcs_34_5},
{1, arcs_34_6},
};
static arc arcs_35_0[1] = {
{86, 1},
};
static arc arcs_35_1[1] = {
{28, 2},
};
static arc arcs_35_2[2] = {
{29, 3},
{0, 2},
};
static arc arcs_35_3[1] = {
{28, 4},
};
static arc arcs_35_4[1] = {
{0, 4},
};
static state states_35[5] = {
{1, arcs_35_0},
{1, arcs_35_1},
{2, arcs_35_2},
{1, arcs_35_3},
{1, arcs_35_4},
};
static arc arcs_36_0[8] = {
{87, 1},
{88, 1},
{89, 1},
{90, 1},
{91, 1},
{19, 1},
{18, 1},
{17, 1},
};
static arc arcs_36_1[1] = {
{0, 1},
};
static state states_36[2] = {
{8, arcs_36_0},
{1, arcs_36_1},
};
static arc arcs_37_0[1] = {
{92, 1},
};
static arc arcs_37_1[1] = {
{28, 2},
};
static arc arcs_37_2[1] = {
{23, 3},
};
static arc arcs_37_3[1] = {
{24, 4},
};
static arc arcs_37_4[3] = {
{93, 1},
{94, 5},
{0, 4},
};
static arc arcs_37_5[1] = {
{23, 6},
};
static arc arcs_37_6[1] = {
{24, 7},
};
static arc arcs_37_7[1] = {
{0, 7},
};
static state states_37[8] = {
{1, arcs_37_0},
{1, arcs_37_1},
{1, arcs_37_2},
{1, arcs_37_3},
{3, arcs_37_4},
{1, arcs_37_5},
{1, arcs_37_6},
{1, arcs_37_7},
};
static arc arcs_38_0[1] = {
{95, 1},
};
static arc arcs_38_1[1] = {
{28, 2},
};
static arc arcs_38_2[1] = {
{23, 3},
};
static arc arcs_38_3[1] = {
{24, 4},
};
static arc arcs_38_4[2] = {
{94, 5},
{0, 4},
};
static arc arcs_38_5[1] = {
{23, 6},
};
static arc arcs_38_6[1] = {
{24, 7},
};
static arc arcs_38_7[1] = {
{0, 7},
};
static state states_38[8] = {
{1, arcs_38_0},
{1, arcs_38_1},
{1, arcs_38_2},
{1, arcs_38_3},
{2, arcs_38_4},
{1, arcs_38_5},
{1, arcs_38_6},
{1, arcs_38_7},
};
static arc arcs_39_0[1] = {
{96, 1},
};
static arc arcs_39_1[1] = {
{61, 2},
};
static arc arcs_39_2[1] = {
{85, 3},
};
static arc arcs_39_3[1] = {
{9, 4},
};
static arc arcs_39_4[1] = {
{23, 5},
};
static arc arcs_39_5[1] = {
{24, 6},
};
static arc arcs_39_6[2] = {
{94, 7},
{0, 6},
};
static arc arcs_39_7[1] = {
{23, 8},
};
static arc arcs_39_8[1] = {
{24, 9},
};
static arc arcs_39_9[1] = {
{0, 9},
};
static state states_39[10] = {
{1, arcs_39_0},
{1, arcs_39_1},
{1, arcs_39_2},
{1, arcs_39_3},
{1, arcs_39_4},
{1, arcs_39_5},
{2, arcs_39_6},
{1, arcs_39_7},
{1, arcs_39_8},
{1, arcs_39_9},
};
static arc arcs_40_0[1] = {
{97, 1},
};
static arc arcs_40_1[1] = {
{23, 2},
};
static arc arcs_40_2[1] = {
{24, 3},
};
static arc arcs_40_3[2] = {
{98, 4},
{99, 5},
};
static arc arcs_40_4[1] = {
{23, 6},
};
static arc arcs_40_5[1] = {
{23, 7},
};
static arc arcs_40_6[1] = {
{24, 8},
};
static arc arcs_40_7[1] = {
{24, 9},
};
static arc arcs_40_8[4] = {
{98, 4},
{94, 10},
{99, 5},
{0, 8},
};
static arc arcs_40_9[1] = {
{0, 9},
};
static arc arcs_40_10[1] = {
{23, 11},
};
static arc arcs_40_11[1] = {
{24, 12},
};
static arc arcs_40_12[2] = {
{99, 5},
{0, 12},
};
static state states_40[13] = {
{1, arcs_40_0},
{1, arcs_40_1},
{1, arcs_40_2},
{2, arcs_40_3},
{1, arcs_40_4},
{1, arcs_40_5},
{1, arcs_40_6},
{1, arcs_40_7},
{4, arcs_40_8},
{1, arcs_40_9},
{1, arcs_40_10},
{1, arcs_40_11},
{2, arcs_40_12},
};
static arc arcs_41_0[1] = {
{100, 1},
};
static arc arcs_41_1[1] = {
{101, 2},
};
static arc arcs_41_2[2] = {
{29, 1},
{23, 3},
};
static arc arcs_41_3[1] = {
{24, 4},
};
static arc arcs_41_4[1] = {
{0, 4},
};
static state states_41[5] = {
{1, arcs_41_0},
{1, arcs_41_1},
{2, arcs_41_2},
{1, arcs_41_3},
{1, arcs_41_4},
};
static arc arcs_42_0[1] = {
{28, 1},
};
static arc arcs_42_1[2] = {
{80, 2},
{0, 1},
};
static arc arcs_42_2[1] = {
{84, 3},
};
static arc arcs_42_3[1] = {
{0, 3},
};
static state states_42[4] = {
{1, arcs_42_0},
{2, arcs_42_1},
{1, arcs_42_2},
{1, arcs_42_3},
};
static arc arcs_43_0[1] = {
{102, 1},
};
static arc arcs_43_1[2] = {
{28, 2},
{0, 1},
};
static arc arcs_43_2[3] = {
{80, 3},
{29, 3},
{0, 2},
};
static arc arcs_43_3[1] = {
{28, 4},
};
static arc arcs_43_4[1] = {
{0, 4},
};
static state states_43[5] = {
{1, arcs_43_0},
{2, arcs_43_1},
{3, arcs_43_2},
{1, arcs_43_3},
{1, arcs_43_4},
};
static arc arcs_44_0[2] = {
{3, 1},
{2, 2},
};
static arc arcs_44_1[1] = {
{0, 1},
};
static arc arcs_44_2[1] = {
{103, 3},
};
static arc arcs_44_3[1] = {
{6, 4},
};
static arc arcs_44_4[2] = {
{6, 4},
{104, 1},
};
static state states_44[5] = {
{2, arcs_44_0},
{1, arcs_44_1},
{1, arcs_44_2},
{1, arcs_44_3},
{2, arcs_44_4},
};
static arc arcs_45_0[1] = {
{106, 1},
};
static arc arcs_45_1[2] = {
{29, 2},
{0, 1},
};
static arc arcs_45_2[1] = {
{106, 3},
};
static arc arcs_45_3[2] = {
{29, 4},
{0, 3},
};
static arc arcs_45_4[2] = {
{106, 3},
{0, 4},
};
static state states_45[5] = {
{1, arcs_45_0},
{2, arcs_45_1},
{1, arcs_45_2},
{2, arcs_45_3},
{2, arcs_45_4},
};
static arc arcs_46_0[2] = {
{107, 1},
{108, 1},
};
static arc arcs_46_1[1] = {
{0, 1},
};
static state states_46[2] = {
{2, arcs_46_0},
{1, arcs_46_1},
};
static arc arcs_47_0[1] = {
{109, 1},
};
static arc arcs_47_1[2] = {
{25, 2},
{23, 3},
};
static arc arcs_47_2[1] = {
{23, 3},
};
static arc arcs_47_3[1] = {
{106, 4},
};
static arc arcs_47_4[1] = {
{0, 4},
};
static state states_47[5] = {
{1, arcs_47_0},
{2, arcs_47_1},
{1, arcs_47_2},
{1, arcs_47_3},
{1, arcs_47_4},
};
static arc arcs_48_0[2] = {
{107, 1},
{110, 2},
};
static arc arcs_48_1[2] = {
{92, 3},
{0, 1},
};
static arc arcs_48_2[1] = {
{0, 2},
};
static arc arcs_48_3[1] = {
{107, 4},
};
static arc arcs_48_4[1] = {
{94, 5},
};
static arc arcs_48_5[1] = {
{28, 2},
};
static state states_48[6] = {
{2, arcs_48_0},
{2, arcs_48_1},
{1, arcs_48_2},
{1, arcs_48_3},
{1, arcs_48_4},
{1, arcs_48_5},
};
static arc arcs_49_0[1] = {
{111, 1},
};
static arc arcs_49_1[2] = {
{112, 0},
{0, 1},
};
static state states_49[2] = {
{1, arcs_49_0},
{2, arcs_49_1},
};
static arc arcs_50_0[1] = {
{113, 1},
};
static arc arcs_50_1[2] = {
{114, 0},
{0, 1},
};
static state states_50[2] = {
{1, arcs_50_0},
{2, arcs_50_1},
};
static arc arcs_51_0[2] = {
{115, 1},
{116, 2},
};
static arc arcs_51_1[1] = {
{113, 2},
};
static arc arcs_51_2[1] = {
{0, 2},
};
static state states_51[3] = {
{2, arcs_51_0},
{1, arcs_51_1},
{1, arcs_51_2},
};
static arc arcs_52_0[1] = {
{84, 1},
};
static arc arcs_52_1[2] = {
{117, 0},
{0, 1},
};
static state states_52[2] = {
{1, arcs_52_0},
{2, arcs_52_1},
};
static arc arcs_53_0[10] = {
{118, 1},
{119, 1},
{120, 1},
{121, 1},
{122, 1},
{123, 1},
{124, 1},
{85, 1},
{115, 2},
{125, 3},
};
static arc arcs_53_1[1] = {
{0, 1},
};
static arc arcs_53_2[1] = {
{85, 1},
};
static arc arcs_53_3[2] = {
{115, 1},
{0, 3},
};
static state states_53[4] = {
{10, arcs_53_0},
{1, arcs_53_1},
{1, arcs_53_2},
{2, arcs_53_3},
};
static arc arcs_54_0[1] = {
{126, 1},
};
static arc arcs_54_1[2] = {
{127, 0},
{0, 1},
};
static state states_54[2] = {
{1, arcs_54_0},
{2, arcs_54_1},
};
static arc arcs_55_0[1] = {
{128, 1},
};
static arc arcs_55_1[2] = {
{129, 0},
{0, 1},
};
static state states_55[2] = {
{1, arcs_55_0},
{2, arcs_55_1},
};
static arc arcs_56_0[1] = {
{130, 1},
};
static arc arcs_56_1[2] = {
{131, 0},
{0, 1},
};
static state states_56[2] = {
{1, arcs_56_0},
{2, arcs_56_1},
};
static arc arcs_57_0[1] = {
{132, 1},
};
static arc arcs_57_1[3] = {
{133, 0},
{59, 0},
{0, 1},
};
static state states_57[2] = {
{1, arcs_57_0},
{3, arcs_57_1},
};
static arc arcs_58_0[1] = {
{134, 1},
};
static arc arcs_58_1[3] = {
{135, 0},
{136, 0},
{0, 1},
};
static state states_58[2] = {
{1, arcs_58_0},
{3, arcs_58_1},
};
static arc arcs_59_0[1] = {
{137, 1},
};
static arc arcs_59_1[5] = {
{30, 0},
{138, 0},
{139, 0},
{140, 0},
{0, 1},
};
static state states_59[2] = {
{1, arcs_59_0},
{5, arcs_59_1},
};
static arc arcs_60_0[4] = {
{135, 1},
{136, 1},
{141, 1},
{142, 2},
};
static arc arcs_60_1[1] = {
{137, 2},
};
static arc arcs_60_2[1] = {
{0, 2},
};
static state states_60[3] = {
{4, arcs_60_0},
{1, arcs_60_1},
{1, arcs_60_2},
};
static arc arcs_61_0[1] = {
{143, 1},
};
static arc arcs_61_1[3] = {
{144, 1},
{31, 2},
{0, 1},
};
static arc arcs_61_2[1] = {
{137, 3},
};
static arc arcs_61_3[1] = {
{0, 3},
};
static state states_61[4] = {
{1, arcs_61_0},
{3, arcs_61_1},
{1, arcs_61_2},
{1, arcs_61_3},
};
static arc arcs_62_0[7] = {
{13, 1},
{146, 2},
{149, 3},
{152, 4},
{21, 5},
{154, 5},
{155, 6},
};
static arc arcs_62_1[3] = {
{45, 7},
{145, 7},
{15, 5},
};
static arc arcs_62_2[2] = {
{147, 8},
{148, 5},
};
static arc arcs_62_3[2] = {
{150, 9},
{151, 5},
};
static arc arcs_62_4[1] = {
{153, 10},
};
static arc arcs_62_5[1] = {
{0, 5},
};
static arc arcs_62_6[2] = {
{155, 6},
{0, 6},
};
static arc arcs_62_7[1] = {
{15, 5},
};
static arc arcs_62_8[1] = {
{148, 5},
};
static arc arcs_62_9[1] = {
{151, 5},
};
static arc arcs_62_10[1] = {
{152, 5},
};
static state states_62[11] = {
{7, arcs_62_0},
{3, arcs_62_1},
{2, arcs_62_2},
{2, arcs_62_3},
{1, arcs_62_4},
{1, arcs_62_5},
{2, arcs_62_6},
{1, arcs_62_7},
{1, arcs_62_8},
{1, arcs_62_9},
{1, arcs_62_10},
};
static arc arcs_63_0[1] = {
{28, 1},
};
static arc arcs_63_1[3] = {
{156, 2},
{29, 3},
{0, 1},
};
static arc arcs_63_2[1] = {
{0, 2},
};
static arc arcs_63_3[2] = {
{28, 4},
{0, 3},
};
static arc arcs_63_4[2] = {
{29, 3},
{0, 4},
};
static state states_63[5] = {
{1, arcs_63_0},
{3, arcs_63_1},
{1, arcs_63_2},
{2, arcs_63_3},
{2, arcs_63_4},
};
static arc arcs_64_0[1] = {
{28, 1},
};
static arc arcs_64_1[3] = {
{157, 2},
{29, 3},
{0, 1},
};
static arc arcs_64_2[1] = {
{0, 2},
};
static arc arcs_64_3[2] = {
{28, 4},
{0, 3},
};
static arc arcs_64_4[2] = {
{29, 3},
{0, 4},
};
static state states_64[5] = {
{1, arcs_64_0},
{3, arcs_64_1},
{1, arcs_64_2},
{2, arcs_64_3},
{2, arcs_64_4},
};
static arc arcs_65_0[1] = {
{109, 1},
};
static arc arcs_65_1[2] = {
{25, 2},
{23, 3},
};
static arc arcs_65_2[1] = {
{23, 3},
};
static arc arcs_65_3[1] = {
{28, 4},
};
static arc arcs_65_4[1] = {
{0, 4},
};
static state states_65[5] = {
{1, arcs_65_0},
{2, arcs_65_1},
{1, arcs_65_2},
{1, arcs_65_3},
{1, arcs_65_4},
};
static arc arcs_66_0[3] = {
{13, 1},
{146, 2},
{77, 3},
};
static arc arcs_66_1[2] = {
{14, 4},
{15, 5},
};
static arc arcs_66_2[1] = {
{158, 6},
};
static arc arcs_66_3[1] = {
{21, 5},
};
static arc arcs_66_4[1] = {
{15, 5},
};
static arc arcs_66_5[1] = {
{0, 5},
};
static arc arcs_66_6[1] = {
{148, 5},
};
static state states_66[7] = {
{3, arcs_66_0},
{2, arcs_66_1},
{1, arcs_66_2},
{1, arcs_66_3},
{1, arcs_66_4},
{1, arcs_66_5},
{1, arcs_66_6},
};
static arc arcs_67_0[1] = {
{159, 1},
};
static arc arcs_67_1[2] = {
{29, 2},
{0, 1},
};
static arc arcs_67_2[2] = {
{159, 1},
{0, 2},
};
static state states_67[3] = {
{1, arcs_67_0},
{2, arcs_67_1},
{2, arcs_67_2},
};
static arc arcs_68_0[3] = {
{77, 1},
{28, 2},
{23, 3},
};
static arc arcs_68_1[1] = {
{77, 4},
};
static arc arcs_68_2[2] = {
{23, 3},
{0, 2},
};
static arc arcs_68_3[3] = {
{28, 5},
{160, 6},
{0, 3},
};
static arc arcs_68_4[1] = {
{77, 6},
};
static arc arcs_68_5[2] = {
{160, 6},
{0, 5},
};
static arc arcs_68_6[1] = {
{0, 6},
};
static state states_68[7] = {
{3, arcs_68_0},
{1, arcs_68_1},
{2, arcs_68_2},
{3, arcs_68_3},
{1, arcs_68_4},
{2, arcs_68_5},
{1, arcs_68_6},
};
static arc arcs_69_0[1] = {
{23, 1},
};
static arc arcs_69_1[2] = {
{28, 2},
{0, 1},
};
static arc arcs_69_2[1] = {
{0, 2},
};
static state states_69[3] = {
{1, arcs_69_0},
{2, arcs_69_1},
{1, arcs_69_2},
};
static arc arcs_70_0[1] = {
{84, 1},
};
static arc arcs_70_1[2] = {
{29, 2},
{0, 1},
};
static arc arcs_70_2[2] = {
{84, 1},
{0, 2},
};
static state states_70[3] = {
{1, arcs_70_0},
{2, arcs_70_1},
{2, arcs_70_2},
};
static arc arcs_71_0[1] = {
{28, 1},
};
static arc arcs_71_1[2] = {
{29, 2},
{0, 1},
};
static arc arcs_71_2[2] = {
{28, 1},
{0, 2},
};
static state states_71[3] = {
{1, arcs_71_0},
{2, arcs_71_1},
{2, arcs_71_2},
};
static arc arcs_72_0[1] = {
{28, 1},
};
static arc arcs_72_1[4] = {
{23, 2},
{157, 3},
{29, 4},
{0, 1},
};
static arc arcs_72_2[1] = {
{28, 5},
};
static arc arcs_72_3[1] = {
{0, 3},
};
static arc arcs_72_4[2] = {
{28, 6},
{0, 4},
};
static arc arcs_72_5[3] = {
{157, 3},
{29, 7},
{0, 5},
};
static arc arcs_72_6[2] = {
{29, 4},
{0, 6},
};
static arc arcs_72_7[2] = {
{28, 8},
{0, 7},
};
static arc arcs_72_8[1] = {
{23, 9},
};
static arc arcs_72_9[1] = {
{28, 10},
};
static arc arcs_72_10[2] = {
{29, 7},
{0, 10},
};
static state states_72[11] = {
{1, arcs_72_0},
{4, arcs_72_1},
{1, arcs_72_2},
{1, arcs_72_3},
{2, arcs_72_4},
{3, arcs_72_5},
{2, arcs_72_6},
{2, arcs_72_7},
{1, arcs_72_8},
{1, arcs_72_9},
{2, arcs_72_10},
};
static arc arcs_73_0[1] = {
{161, 1},
};
static arc arcs_73_1[1] = {
{21, 2},
};
static arc arcs_73_2[2] = {
{13, 3},
{23, 4},
};
static arc arcs_73_3[2] = {
{9, 5},
{15, 6},
};
static arc arcs_73_4[1] = {
{24, 7},
};
static arc arcs_73_5[1] = {
{15, 6},
};
static arc arcs_73_6[1] = {
{23, 4},
};
static arc arcs_73_7[1] = {
{0, 7},
};
static state states_73[8] = {
{1, arcs_73_0},
{1, arcs_73_1},
{2, arcs_73_2},
{2, arcs_73_3},
{1, arcs_73_4},
{1, arcs_73_5},
{1, arcs_73_6},
{1, arcs_73_7},
};
static arc arcs_74_0[3] = {
{162, 1},
{30, 2},
{31, 3},
};
static arc arcs_74_1[2] = {
{29, 4},
{0, 1},
};
static arc arcs_74_2[1] = {
{28, 5},
};
static arc arcs_74_3[1] = {
{28, 6},
};
static arc arcs_74_4[4] = {
{162, 1},
{30, 2},
{31, 3},
{0, 4},
};
static arc arcs_74_5[2] = {
{29, 7},
{0, 5},
};
static arc arcs_74_6[1] = {
{0, 6},
};
static arc arcs_74_7[2] = {
{162, 5},
{31, 3},
};
static state states_74[8] = {
{3, arcs_74_0},
{2, arcs_74_1},
{1, arcs_74_2},
{1, arcs_74_3},
{4, arcs_74_4},
{2, arcs_74_5},
{1, arcs_74_6},
{2, arcs_74_7},
};
static arc arcs_75_0[1] = {
{28, 1},
};
static arc arcs_75_1[3] = {
{157, 2},
{27, 3},
{0, 1},
};
static arc arcs_75_2[1] = {
{0, 2},
};
static arc arcs_75_3[1] = {
{28, 2},
};
static state states_75[4] = {
{1, arcs_75_0},
{3, arcs_75_1},
{1, arcs_75_2},
{1, arcs_75_3},
};
static arc arcs_76_0[2] = {
{156, 1},
{164, 1},
};
static arc arcs_76_1[1] = {
{0, 1},
};
static state states_76[2] = {
{2, arcs_76_0},
{1, arcs_76_1},
};
static arc arcs_77_0[1] = {
{96, 1},
};
static arc arcs_77_1[1] = {
{61, 2},
};
static arc arcs_77_2[1] = {
{85, 3},
};
static arc arcs_77_3[1] = {
{105, 4},
};
static arc arcs_77_4[2] = {
{163, 5},
{0, 4},
};
static arc arcs_77_5[1] = {
{0, 5},
};
static state states_77[6] = {
{1, arcs_77_0},
{1, arcs_77_1},
{1, arcs_77_2},
{1, arcs_77_3},
{2, arcs_77_4},
{1, arcs_77_5},
};
static arc arcs_78_0[1] = {
{92, 1},
};
static arc arcs_78_1[1] = {
{106, 2},
};
static arc arcs_78_2[2] = {
{163, 3},
{0, 2},
};
static arc arcs_78_3[1] = {
{0, 3},
};
static state states_78[4] = {
{1, arcs_78_0},
{1, arcs_78_1},
{2, arcs_78_2},
{1, arcs_78_3},
};
static arc arcs_79_0[2] = {
{157, 1},
{166, 1},
};
static arc arcs_79_1[1] = {
{0, 1},
};
static state states_79[2] = {
{2, arcs_79_0},
{1, arcs_79_1},
};
static arc arcs_80_0[1] = {
{96, 1},
};
static arc arcs_80_1[1] = {
{61, 2},
};
static arc arcs_80_2[1] = {
{85, 3},
};
static arc arcs_80_3[1] = {
{107, 4},
};
static arc arcs_80_4[2] = {
{165, 5},
{0, 4},
};
static arc arcs_80_5[1] = {
{0, 5},
};
static state states_80[6] = {
{1, arcs_80_0},
{1, arcs_80_1},
{1, arcs_80_2},
{1, arcs_80_3},
{2, arcs_80_4},
{1, arcs_80_5},
};
static arc arcs_81_0[1] = {
{92, 1},
};
static arc arcs_81_1[1] = {
{106, 2},
};
static arc arcs_81_2[2] = {
{165, 3},
{0, 2},
};
static arc arcs_81_3[1] = {
{0, 3},
};
static state states_81[4] = {
{1, arcs_81_0},
{1, arcs_81_1},
{2, arcs_81_2},
{1, arcs_81_3},
};
static arc arcs_82_0[1] = {
{28, 1},
};
static arc arcs_82_1[2] = {
{29, 0},
{0, 1},
};
static state states_82[2] = {
{1, arcs_82_0},
{2, arcs_82_1},
};
static arc arcs_83_0[1] = {
{21, 1},
};
static arc arcs_83_1[1] = {
{0, 1},
};
static state states_83[2] = {
{1, arcs_83_0},
{1, arcs_83_1},
};
static arc arcs_84_0[1] = {
{168, 1},
};
static arc arcs_84_1[2] = {
{9, 2},
{0, 1},
};
static arc arcs_84_2[1] = {
{0, 2},
};
static state states_84[3] = {
{1, arcs_84_0},
{2, arcs_84_1},
{1, arcs_84_2},
};
static dfa dfas[85] = {
{256, "single_input", 0, 3, states_0,
"\004\050\060\000\000\000\000\124\360\024\114\220\023\040\010\000\200\041\044\015\002\001"},
{257, "file_input", 0, 2, states_1,
"\204\050\060\000\000\000\000\124\360\024\114\220\023\040\010\000\200\041\044\015\002\001"},
{258, "eval_input", 0, 3, states_2,
"\000\040\040\000\000\000\000\000\000\000\000\000\000\040\010\000\200\041\044\015\000\000"},
{259, "decorator", 0, 7, states_3,
"\000\010\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"},
{260, "decorators", 0, 2, states_4,
"\000\010\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"},
{261, "decorated", 0, 3, states_5,
"\000\010\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"},
{262, "funcdef", 0, 6, states_6,
"\000\000\020\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"},
{263, "parameters", 0, 4, states_7,
"\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"},
{264, "varargslist", 0, 10, states_8,
"\000\040\040\300\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"},
{265, "fpdef", 0, 4, states_9,
"\000\040\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"},
{266, "fplist", 0, 3, states_10,
"\000\040\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"},
{267, "stmt", 0, 2, states_11,
"\000\050\060\000\000\000\000\124\360\024\114\220\023\040\010\000\200\041\044\015\002\001"},
{268, "simple_stmt", 0, 4, states_12,
"\000\040\040\000\000\000\000\124\360\024\114\000\000\040\010\000\200\041\044\015\000\001"},
{269, "small_stmt", 0, 2, states_13,
"\000\040\040\000\000\000\000\124\360\024\114\000\000\040\010\000\200\041\044\015\000\001"},
{270, "expr_stmt", 0, 6, states_14,
"\000\040\040\000\000\000\000\000\000\000\000\000\000\040\010\000\200\041\044\015\000\000"},
{271, "augassign", 0, 2, states_15,
"\000\000\000\000\000\300\377\003\000\000\000\000\000\000\000\000\000\000\000\000\000\000"},
{272, "print_stmt", 0, 9, states_16,
"\000\000\000\000\000\000\000\004\000\000\000\000\000\000\000\000\000\000\000\000\000\000"},
{273, "del_stmt", 0, 3, states_17,
"\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000\000\000\000\000\000\000\000"},
{274, "pass_stmt", 0, 2, states_18,
"\000\000\000\000\000\000\000\100\000\000\000\000\000\000\000\000\000\000\000\000\000\000"},
{275, "flow_stmt", 0, 2, states_19,
"\000\000\000\000\000\000\000\000\360\000\000\000\000\000\000\000\000\000\000\000\000\001"},
{276, "break_stmt", 0, 2, states_20,
"\000\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000\000\000\000\000\000\000"},
{277, "continue_stmt", 0, 2, states_21,
"\000\000\000\000\000\000\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000"},
{278, "return_stmt", 0, 3, states_22,
"\000\000\000\000\000\000\000\000\100\000\000\000\000\000\000\000\000\000\000\000\000\000"},
{279, "yield_stmt", 0, 2, states_23,
"\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\001"},
{280, "raise_stmt", 0, 7, states_24,
"\000\000\000\000\000\000\000\000\200\000\000\000\000\000\000\000\000\000\000\000\000\000"},
{281, "import_stmt", 0, 2, states_25,
"\000\000\000\000\000\000\000\000\000\024\000\000\000\000\000\000\000\000\000\000\000\000"},
{282, "import_name", 0, 3, states_26,
"\000\000\000\000\000\000\000\000\000\004\000\000\000\000\000\000\000\000\000\000\000\000"},
{283, "import_from", 0, 8, states_27,
"\000\000\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000\000\000\000\000\000"},
{284, "import_as_name", 0, 4, states_28,
"\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"},
{285, "dotted_as_name", 0, 4, states_29,
"\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"},
{286, "import_as_names", 0, 3, states_30,
"\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"},
{287, "dotted_as_names", 0, 2, states_31,
"\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"},
{288, "dotted_name", 0, 2, states_32,
"\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"},
{289, "global_stmt", 0, 3, states_33,
"\000\000\000\000\000\000\000\000\000\000\004\000\000\000\000\000\000\000\000\000\000\000"},
{290, "exec_stmt", 0, 7, states_34,
"\000\000\000\000\000\000\000\000\000\000\010\000\000\000\000\000\000\000\000\000\000\000"},
{291, "assert_stmt", 0, 5, states_35,
"\000\000\000\000\000\000\000\000\000\000\100\000\000\000\000\000\000\000\000\000\000\000"},
{292, "compound_stmt", 0, 2, states_36,
"\000\010\020\000\000\000\000\000\000\000\000\220\023\000\000\000\000\000\000\000\002\000"},
{293, "if_stmt", 0, 8, states_37,
"\000\000\000\000\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000\000\000\000"},
{294, "while_stmt", 0, 8, states_38,
"\000\000\000\000\000\000\000\000\000\000\000\200\000\000\000\000\000\000\000\000\000\000"},
{295, "for_stmt", 0, 10, states_39,
"\000\000\000\000\000\000\000\000\000\000\000\000\001\000\000\000\000\000\000\000\000\000"},
{296, "try_stmt", 0, 13, states_40,
"\000\000\000\000\000\000\000\000\000\000\000\000\002\000\000\000\000\000\000\000\000\000"},
{297, "with_stmt", 0, 5, states_41,
"\000\000\000\000\000\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000\000\000"},
{298, "with_item", 0, 4, states_42,
"\000\040\040\000\000\000\000\000\000\000\000\000\000\040\010\000\200\041\044\015\000\000"},
{299, "except_clause", 0, 5, states_43,
"\000\000\000\000\000\000\000\000\000\000\000\000\100\000\000\000\000\000\000\000\000\000"},
{300, "suite", 0, 5, states_44,
"\004\040\040\000\000\000\000\124\360\024\114\000\000\040\010\000\200\041\044\015\000\001"},
{301, "testlist_safe", 0, 5, states_45,
"\000\040\040\000\000\000\000\000\000\000\000\000\000\040\010\000\200\041\044\015\000\000"},
{302, "old_test", 0, 2, states_46,
"\000\040\040\000\000\000\000\000\000\000\000\000\000\040\010\000\200\041\044\015\000\000"},
{303, "old_lambdef", 0, 5, states_47,
"\000\000\000\000\000\000\000\000\000\000\000\000\000\040\000\000\000\000\000\000\000\000"},
{304, "test", 0, 6, states_48,
"\000\040\040\000\000\000\000\000\000\000\000\000\000\040\010\000\200\041\044\015\000\000"},
{305, "or_test", 0, 2, states_49,
"\000\040\040\000\000\000\000\000\000\000\000\000\000\000\010\000\200\041\044\015\000\000"},
{306, "and_test", 0, 2, states_50,
"\000\040\040\000\000\000\000\000\000\000\000\000\000\000\010\000\200\041\044\015\000\000"},
{307, "not_test", 0, 3, states_51,
"\000\040\040\000\000\000\000\000\000\000\000\000\000\000\010\000\200\041\044\015\000\000"},
{308, "comparison", 0, 2, states_52,
"\000\040\040\000\000\000\000\000\000\000\000\000\000\000\000\000\200\041\044\015\000\000"},
{309, "comp_op", 0, 4, states_53,
"\000\000\000\000\000\000\000\000\000\000\040\000\000\000\310\077\000\000\000\000\000\000"},
{310, "expr", 0, 2, states_54,
"\000\040\040\000\000\000\000\000\000\000\000\000\000\000\000\000\200\041\044\015\000\000"},
{311, "xor_expr", 0, 2, states_55,
"\000\040\040\000\000\000\000\000\000\000\000\000\000\000\000\000\200\041\044\015\000\000"},
{312, "and_expr", 0, 2, states_56,
"\000\040\040\000\000\000\000\000\000\000\000\000\000\000\000\000\200\041\044\015\000\000"},
{313, "shift_expr", 0, 2, states_57,
"\000\040\040\000\000\000\000\000\000\000\000\000\000\000\000\000\200\041\044\015\000\000"},
{314, "arith_expr", 0, 2, states_58,
"\000\040\040\000\000\000\000\000\000\000\000\000\000\000\000\000\200\041\044\015\000\000"},
{315, "term", 0, 2, states_59,
"\000\040\040\000\000\000\000\000\000\000\000\000\000\000\000\000\200\041\044\015\000\000"},
{316, "factor", 0, 3, states_60,
"\000\040\040\000\000\000\000\000\000\000\000\000\000\000\000\000\200\041\044\015\000\000"},
{317, "power", 0, 4, states_61,
"\000\040\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\044\015\000\000"},
{318, "atom", 0, 11, states_62,
"\000\040\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\044\015\000\000"},
{319, "listmaker", 0, 5, states_63,
"\000\040\040\000\000\000\000\000\000\000\000\000\000\040\010\000\200\041\044\015\000\000"},
{320, "testlist_comp", 0, 5, states_64,
"\000\040\040\000\000\000\000\000\000\000\000\000\000\040\010\000\200\041\044\015\000\000"},
{321, "lambdef", 0, 5, states_65,
"\000\000\000\000\000\000\000\000\000\000\000\000\000\040\000\000\000\000\000\000\000\000"},
{322, "trailer", 0, 7, states_66,
"\000\040\000\000\000\000\000\000\000\040\000\000\000\000\000\000\000\000\004\000\000\000"},
{323, "subscriptlist", 0, 3, states_67,
"\000\040\240\000\000\000\000\000\000\040\000\000\000\040\010\000\200\041\044\015\000\000"},
{324, "subscript", 0, 7, states_68,
"\000\040\240\000\000\000\000\000\000\040\000\000\000\040\010\000\200\041\044\015\000\000"},
{325, "sliceop", 0, 3, states_69,
"\000\000\200\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"},
{326, "exprlist", 0, 3, states_70,
"\000\040\040\000\000\000\000\000\000\000\000\000\000\000\000\000\200\041\044\015\000\000"},
{327, "testlist", 0, 3, states_71,
"\000\040\040\000\000\000\000\000\000\000\000\000\000\040\010\000\200\041\044\015\000\000"},
{328, "dictorsetmaker", 0, 11, states_72,
"\000\040\040\000\000\000\000\000\000\000\000\000\000\040\010\000\200\041\044\015\000\000"},
{329, "classdef", 0, 8, states_73,
"\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\002\000"},
{330, "arglist", 0, 8, states_74,
"\000\040\040\300\000\000\000\000\000\000\000\000\000\040\010\000\200\041\044\015\000\000"},
{331, "argument", 0, 4, states_75,
"\000\040\040\000\000\000\000\000\000\000\000\000\000\040\010\000\200\041\044\015\000\000"},
{332, "list_iter", 0, 2, states_76,
"\000\000\000\000\000\000\000\000\000\000\000\020\001\000\000\000\000\000\000\000\000\000"},
{333, "list_for", 0, 6, states_77,
"\000\000\000\000\000\000\000\000\000\000\000\000\001\000\000\000\000\000\000\000\000\000"},
{334, "list_if", 0, 4, states_78,
"\000\000\000\000\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000\000\000\000"},
{335, "comp_iter", 0, 2, states_79,
"\000\000\000\000\000\000\000\000\000\000\000\020\001\000\000\000\000\000\000\000\000\000"},
{336, "comp_for", 0, 6, states_80,
"\000\000\000\000\000\000\000\000\000\000\000\000\001\000\000\000\000\000\000\000\000\000"},
{337, "comp_if", 0, 4, states_81,
"\000\000\000\000\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000\000\000\000"},
{338, "testlist1", 0, 2, states_82,
"\000\040\040\000\000\000\000\000\000\000\000\000\000\040\010\000\200\041\044\015\000\000"},
{339, "encoding_decl", 0, 2, states_83,
"\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"},
{340, "yield_expr", 0, 3, states_84,
"\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\001"},
};
static label labels[169] = {
{0, "EMPTY"},
{256, 0},
{4, 0},
{268, 0},
{292, 0},
{257, 0},
{267, 0},
{0, 0},
{258, 0},
{327, 0},
{259, 0},
{50, 0},
{288, 0},
{7, 0},
{330, 0},
{8, 0},
{260, 0},
{261, 0},
{329, 0},
{262, 0},
{1, "def"},
{1, 0},
{263, 0},
{11, 0},
{300, 0},
{264, 0},
{265, 0},
{22, 0},
{304, 0},
{12, 0},
{16, 0},
{36, 0},
{266, 0},
{269, 0},
{13, 0},
{270, 0},
{272, 0},
{273, 0},
{274, 0},
{275, 0},
{281, 0},
{289, 0},
{290, 0},
{291, 0},
{271, 0},
{340, 0},
{37, 0},
{38, 0},
{39, 0},
{40, 0},
{41, 0},
{42, 0},
{43, 0},
{44, 0},
{45, 0},
{46, 0},
{47, 0},
{49, 0},
{1, "print"},
{35, 0},
{1, "del"},
{326, 0},
{1, "pass"},
{276, 0},
{277, 0},
{278, 0},
{280, 0},
{279, 0},
{1, "break"},
{1, "continue"},
{1, "return"},
{1, "raise"},
{282, 0},
{283, 0},
{1, "import"},
{287, 0},
{1, "from"},
{23, 0},
{286, 0},
{284, 0},
{1, "as"},
{285, 0},
{1, "global"},
{1, "exec"},
{310, 0},
{1, "in"},
{1, "assert"},
{293, 0},
{294, 0},
{295, 0},
{296, 0},
{297, 0},
{1, "if"},
{1, "elif"},
{1, "else"},
{1, "while"},
{1, "for"},
{1, "try"},
{299, 0},
{1, "finally"},
{1, "with"},
{298, 0},
{1, "except"},
{5, 0},
{6, 0},
{301, 0},
{302, 0},
{305, 0},
{303, 0},
{1, "lambda"},
{321, 0},
{306, 0},
{1, "or"},
{307, 0},
{1, "and"},
{1, "not"},
{308, 0},
{309, 0},
{20, 0},
{21, 0},
{28, 0},
{31, 0},
{30, 0},
{29, 0},
{29, 0},
{1, "is"},
{311, 0},
{18, 0},
{312, 0},
{33, 0},
{313, 0},
{19, 0},
{314, 0},
{34, 0},
{315, 0},
{14, 0},
{15, 0},
{316, 0},
{17, 0},
{24, 0},
{48, 0},
{32, 0},
{317, 0},
{318, 0},
{322, 0},
{320, 0},
{9, 0},
{319, 0},
{10, 0},
{26, 0},
{328, 0},
{27, 0},
{25, 0},
{338, 0},
{2, 0},
{3, 0},
{333, 0},
{336, 0},
{323, 0},
{324, 0},
{325, 0},
{1, "class"},
{331, 0},
{332, 0},
{334, 0},
{335, 0},
{337, 0},
{339, 0},
{1, "yield"},
};
grammar _PyParser_Grammar = {
85,
dfas,
{169, labels},
256
};
from_cpython/Python/pyarena.c 0 → 100644
View file @ 3fa45d83
#include "Python.h"
#include "pyarena.h"
/* A simple arena block structure.
Measurements with standard library modules suggest the average
allocation is about 20 bytes and that most compiles use a single
block.
TODO(jhylton): Think about a realloc API, maybe just for the last
allocation?
*/
#define DEFAULT_BLOCK_SIZE 8192
#define ALIGNMENT 8
#define ALIGNMENT_MASK (ALIGNMENT - 1)
#define ROUNDUP(x) (((x) + ALIGNMENT_MASK) & ~ALIGNMENT_MASK)
typedef struct _block {
/* Total number of bytes owned by this block available to pass out.
* Read-only after initialization. The first such byte starts at
* ab_mem.
*/
size_t ab_size;
/* Total number of bytes already passed out. The next byte available
* to pass out starts at ab_mem + ab_offset.
*/
size_t ab_offset;
/* An arena maintains a singly-linked, NULL-terminated list of
* all blocks owned by the arena. These are linked via the
* ab_next member.
*/
struct _block *ab_next;
/* Pointer to the first allocatable byte owned by this block. Read-
* only after initialization.
*/
void *ab_mem;
} block;
/* The arena manages two kinds of memory, blocks of raw memory
and a list of PyObject* pointers. PyObjects are decrefed
when the arena is freed.
*/
struct _arena {
/* Pointer to the first block allocated for the arena, never NULL.
It is used only to find the first block when the arena is
being freed.
*/
block *a_head;
/* Pointer to the block currently used for allocation. It's
ab_next field should be NULL. If it is not-null after a
call to block_alloc(), it means a new block has been allocated
and a_cur should be reset to point it.
*/
block *a_cur;
/* A Python list object containing references to all the PyObject
pointers associated with this area. They will be DECREFed
when the arena is freed.
*/
PyObject *a_objects;
#if defined(Py_DEBUG)
/* Debug output */
size_t total_allocs;
size_t total_size;
size_t total_blocks;
size_t total_block_size;
size_t total_big_blocks;
#endif
};
static block *
block_new(size_t size)
{
/* Allocate header and block as one unit.
ab_mem points just past header. */
block *b = (block *)malloc(sizeof(block) + size);
if (!b)
return NULL;
b->ab_size = size;
b->ab_mem = (void *)(b + 1);
b->ab_next = NULL;
b->ab_offset = ROUNDUP((Py_uintptr_t)(b->ab_mem)) -
(Py_uintptr_t)(b->ab_mem);
return b;
}
static void
block_free(block *b) {
while (b) {
block *next = b->ab_next;
free(b);
b = next;
}
}
static void *
block_alloc(block *b, size_t size)
{
void *p;
assert(b);
size = ROUNDUP(size);
if (b->ab_offset + size > b->ab_size) {
/* If we need to allocate more memory than will fit in
the default block, allocate a one-off block that is
exactly the right size. */
/* TODO(jhylton): Think about space waste at end of block */
block *newbl = block_new(
size < DEFAULT_BLOCK_SIZE ?
DEFAULT_BLOCK_SIZE : size);
if (!newbl)
return NULL;
assert(!b->ab_next);
b->ab_next = newbl;
b = newbl;
}
assert(b->ab_offset + size <= b->ab_size);
p = (void *)(((char *)b->ab_mem) + b->ab_offset);
b->ab_offset += size;
return p;
}
PyArena *
PyArena_New()
{
PyArena* arena = (PyArena *)malloc(sizeof(PyArena));
if (!arena)
return (PyArena*)PyErr_NoMemory();
arena->a_head = block_new(DEFAULT_BLOCK_SIZE);
arena->a_cur = arena->a_head;
if (!arena->a_head) {
free((void *)arena);
return (PyArena*)PyErr_NoMemory();
}
arena->a_objects = PyList_New(0);
if (!arena->a_objects) {
block_free(arena->a_head);
free((void *)arena);
return (PyArena*)PyErr_NoMemory();
}
#if defined(Py_DEBUG)
arena->total_allocs = 0;
arena->total_size = 0;
arena->total_blocks = 1;
arena->total_block_size = DEFAULT_BLOCK_SIZE;
arena->total_big_blocks = 0;
#endif
return arena;
}
void
PyArena_Free(PyArena *arena)
{
assert(arena);
#if defined(Py_DEBUG)
/*
fprintf(stderr,
"alloc=%d size=%d blocks=%d block_size=%d big=%d objects=%d\n",
arena->total_allocs, arena->total_size, arena->total_blocks,
arena->total_block_size, arena->total_big_blocks,
PyList_Size(arena->a_objects));
*/
#endif
block_free(arena->a_head);
/* This property normally holds, except when the code being compiled
is sys.getobjects(0), in which case there will be two references.
assert(arena->a_objects->ob_refcnt == 1);
*/
Py_DECREF(arena->a_objects);
free(arena);
}
void *
PyArena_Malloc(PyArena *arena, size_t size)
{
void *p = block_alloc(arena->a_cur, size);
if (!p)
return PyErr_NoMemory();
#if defined(Py_DEBUG)
arena->total_allocs++;
arena->total_size += size;
#endif
/* Reset cur if we allocated a new block. */
if (arena->a_cur->ab_next) {
arena->a_cur = arena->a_cur->ab_next;
#if defined(Py_DEBUG)
arena->total_blocks++;
arena->total_block_size += arena->a_cur->ab_size;
if (arena->a_cur->ab_size > DEFAULT_BLOCK_SIZE)
++arena->total_big_blocks;
#endif
}
return p;
}
int
PyArena_AddPyObject(PyArena *arena, PyObject *obj)
{
int r = PyList_Append(arena->a_objects, obj);
if (r >= 0) {
Py_DECREF(obj);
}
return r;
}
src/CMakeLists.txt
View file @ 3fa45d83
......@@ -38,6 +38,7 @@ add_library(PYSTON_OBJECTS OBJECT ${OPTIONAL_SRCS}
codegen/baseline_jit.cpp
codegen/codegen.cpp
codegen/compvars.cpp
codegen/cpython_ast.cpp
codegen/entry.cpp
codegen/gcbuilder.cpp
codegen/irgen.cpp
......
src/codegen/cpython_ast.cpp 0 → 100644
View file @ 3fa45d83
// Copyright (c) 2014-2015 Dropbox, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "codegen/cpython_ast.h"
namespace pyston {
AST_Module* cpythonToPystonAst(mod_ty mod) {
Py_FatalError("unimplemented");
}
}
src/codegen/cpython_ast.h 0 → 100644
View file @ 3fa45d83
// Copyright (c) 2014-2015 Dropbox, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef PYSTON_CODEGEN_CPYTHONAST_H
#define PYSTON_CODEGEN_CPYTHONAST_H
#include "Python.h"
#include "Python-ast.h"
#include "core/ast.h"
namespace pyston {
AST_Module* cpythonToPystonAst(mod_ty mod);
}
#endif
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7