Skip to content

P
Pyston
Commit 0abee106 authored by Marius Wachtler's avatar Marius Wachtler Committed by GitHub
Browse files
Options

Merge pull request #1337 from Daetalus/libpypa 

Remove libpypa parser
parents 76c7d503 a120b2ad
Show whitespace changes
Inline Side-by-side
Showing with 30 additions and 1482 deletions
.gitmodules
View file @ 0abee106
......@@ -22,9 +22,6 @@
path = build_deps/libunwind
url = https://github.com/pathscale/libunwind.git
ignore = all
[submodule "build_deps/libpypa"]
path = build_deps/libpypa
url = git://github.com/vinzenz/libpypa.git
[submodule "build_deps/lz4"]
path = build_deps/lz4
url = git://github.com/Cyan4973/lz4.git
......
CMakeLists.txt
View file @ 0abee106
......@@ -244,10 +244,6 @@ SET_SOURCE_FILES_PROPERTIES(
${CMAKE_BINARY_DIR}/build_deps/libunwind/include/libunwind.h PROPERTIES OBJECT_DEPENDS ${CMAKE_SOURCE_DIR}/build_deps/libunwind/pyston_patched
)
# libpypa
add_subdirectory(build_deps/libpypa EXCLUDE_FROM_ALL)
add_dependencies(pypa gitsubmodules)
# lz4
add_subdirectory(build_deps/lz4/cmake_unofficial EXCLUDE_FROM_ALL)
add_dependencies(lz4 gitsubmodules)
......@@ -317,13 +313,9 @@ add_custom_command(OUTPUT ${CMAKE_BINARY_DIR}/linkdeps_dummy.c COMMAND ${CMAKE_C
add_executable(pyston $<TARGET_OBJECTS:PYSTON_MAIN_OBJECT> $<TARGET_OBJECTS:PYSTON_OBJECTS> $<TARGET_OBJECTS:FROM_CPYTHON> linkdeps_dummy.c)
# Wrap the stdlib in --whole-archive to force all the symbols to be included and eventually exported
target_link_libraries(pyston -Wl,--whole-archive stdlib -Wl,--no-whole-archive pthread m z readline sqlite3 gmp mpfr ssl crypto unwind pypa liblz4 double-conversion util ${LLVM_LIBS} ${LIBLZMA_LIBRARIES} ${OPTIONAL_LIBRARIES} ${CMAKE_BINARY_DIR}/jemalloc/lib/libjemalloc.a)
target_link_libraries(pyston -Wl,--whole-archive stdlib -Wl,--no-whole-archive pthread m z readline sqlite3 gmp mpfr ssl crypto unwind liblz4 util ${LLVM_LIBS} ${LIBLZMA_LIBRARIES} ${OPTIONAL_LIBRARIES} ${CMAKE_BINARY_DIR}/jemalloc/lib/libjemalloc.a)
add_dependencies(pyston libjemalloc)
add_custom_target(astcompare COMMAND ${CMAKE_SOURCE_DIR}/tools/astprint_test.sh
DEPENDS astprint
COMMENT "Running libpypa vs CPython AST result comparison test")
# test
enable_testing()
set(TEST_THREADS 1 CACHE STRING "number of pyston test threads")
......@@ -339,9 +331,9 @@ endmacro()
# tests testname directory arguments
add_pyston_test(defaults tests --order-by-mtime -t50)
add_pyston_test(force_llvm tests -a=-n -a=-x -t90)
add_pyston_test(force_llvm tests -a=-n -t90)
if(${CMAKE_BUILD_TYPE} STREQUAL "Release")
add_pyston_test(max_compilation_tier tests -a=-O -a=-x -t50)
add_pyston_test(max_compilation_tier tests -a=-O -t50)
endif()
add_pyston_test(defaults cpython --exit-code-only --skip-failing -t100)
add_pyston_test(defaults integration --exit-code-only --skip-failing -t900)
......
Makefile
View file @ 0abee106
......@@ -130,7 +130,6 @@ COMMON_CXXFLAGS += -Woverloaded-virtual
COMMON_CXXFLAGS += -fexceptions -fno-rtti
COMMON_CXXFLAGS += -Wno-invalid-offsetof # allow the use of "offsetof", and we'll just have to make sure to only use it legally.
COMMON_CXXFLAGS += -DENABLE_INTEL_JIT_EVENTS=$(ENABLE_INTEL_JIT_EVENTS)
COMMON_CXXFLAGS += -I$(DEPS_DIR)/pypa-install/include
COMMON_CXXFLAGS += -I$(DEPS_DIR)/lz4-install/include
ifeq ($(ENABLE_VALGRIND),0)
......@@ -147,7 +146,6 @@ COMMON_CXXFLAGS += -DGITREV=$(shell git rev-parse HEAD | head -c 12) -DLLVMREV=$
# Use our "custom linker" that calls gold if available
COMMON_LDFLAGS := -B$(TOOLS_DIR)/build_system -L/usr/local/lib -lpthread -lm -lunwind -llzma -L$(DEPS_DIR)/gcc-4.8.2-install/lib64 -lreadline -lgmp -lssl -lcrypto -lsqlite3
COMMON_LDFLAGS += $(DEPS_DIR)/pypa-install/lib/libpypa.a
COMMON_LDFLAGS += $(DEPS_DIR)/lz4-install/lib/liblz4.a
# Conditionally add libtinfo if available - otherwise nothing will be added
......@@ -501,18 +499,6 @@ $(TOOLS_DIR)/publicize_release: $(TOOLS_DIR)/publicize.release.o $(LLVM_RELEASE_
$(ECHO) Linking $(TOOLS_DIR)/publicize_release
$(VERB) $(CXX) $< $(LDFLAGS_RELEASE) -o $@ -lLLVMBitWriter
$(TOOLS_DIR)/astprint: $(TOOLS_DIR)/astprint.cpp $(BUILD_SYSTEM_DEPS) $(LLVM_DEPS) $(ASTPRINT_OBJS)
$(ECHO) Linking $(TOOLS_DIR)/astprint
$(VERB) $(CXX) $< -o $@ $(LLVM_LIB_DEPS) $(ASTPRINT_OBJS) $(LDFLAGS) $(STDLIB_SRCS:.cpp=.o) $(CXXFLAGS_DBG)
.PHONY: astprint astcompare
astprint: $(TOOLS_DIR)/astprint
astcompare: astprint
$(ECHO) Running libpypa vs CPython AST result comparison test
$(TOOLS_DIR)/astprint_test.sh && echo "Success" || echo "Failure"
## END OF TOOLS
......@@ -777,7 +763,7 @@ check$1 test$1: $(PYTHON_EXE_DEPS) pyston$1
@# we pass -I to cpython tests and skip failing ones because they are sloooow otherwise
$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston$1 -j$(TEST_THREADS) -a=-S -k --exit-code-only --skip-failing -t50 $(TEST_DIR)/cpython $(ARGS)
$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston$1 -j$(TEST_THREADS) -k -a=-S --exit-code-only --skip-failing -t600 $(TEST_DIR)/integration $(ARGS)
$(PYTHON) $(TOOLS_DIR)/tester.py -a=-x -R pyston$1 -j$(TEST_THREADS) -a=-n -a=-S -t50 -k $(TESTS_DIR) $(ARGS)
$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston$1 -j$(TEST_THREADS) -a=-n -a=-S -t50 -k $(TESTS_DIR) $(ARGS)
$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston$1 -j$(TEST_THREADS) -a=-O -a=-S -k $(TESTS_DIR) $(ARGS)
.PHONY: run$1 dbg$1
......@@ -1074,7 +1060,7 @@ superlint: plugins/clang_linter.so
.PHONY: lint_%
lint_%: %.cpp plugins/clang_linter.so
$(ECHO) Linting $<
$(VERB) $(CLANG_CXX) -Xclang -load -Xclang plugins/clang_linter.so -Xclang -plugin -Xclang pyston-linter src/runtime/float.cpp $< -c -Isrc/ -Ifrom_cpython/Include -Ibuild/Debug/from_cpython/Include $(shell $(LLVM_BIN_DBG)/llvm-config --cxxflags) $(COMMON_CXXFLAGS) -no-pedantic -Wno-unused-variable -DNVALGRIND -Wno-invalid-offsetof -Wno-mismatched-tags -Wno-unused-function -Wno-unused-private-field -Wno-sign-compare -DLLVMREV=$(LLVM_REVISION) -Ibuild_deps/lz4/lib -DBINARY_SUFFIX= -DBINARY_STRIPPED_SUFFIX=_stripped -Ibuild_deps/libpypa/src/ -Wno-covered-switch-default -Ibuild/Debug/libunwind/include -Wno-extern-c-compat -Wno-unused-local-typedef -Wno-inconsistent-missing-override
$(VERB) $(CLANG_CXX) -Xclang -load -Xclang plugins/clang_linter.so -Xclang -plugin -Xclang pyston-linter src/runtime/float.cpp $< -c -Isrc/ -Ifrom_cpython/Include -Ibuild/Debug/from_cpython/Include $(shell $(LLVM_BIN_DBG)/llvm-config --cxxflags) $(COMMON_CXXFLAGS) -no-pedantic -Wno-unused-variable -DNVALGRIND -Wno-invalid-offsetof -Wno-mismatched-tags -Wno-unused-function -Wno-unused-private-field -Wno-sign-compare -DLLVMREV=$(LLVM_REVISION) -Ibuild_deps/lz4/lib -DBINARY_SUFFIX= -DBINARY_STRIPPED_SUFFIX=_stripped -Ibuild/Debug/libunwind/include -Wno-extern-c-compat -Wno-unused-local-typedef -Wno-inconsistent-missing-override
REFCOUNT_CHECKER_BUILD_PATH := $(CMAKE_DIR_DBG)/plugins/refcount_checker/llvm/bin/refcount_checker
REFCOUNT_CHECKER_RUN_PATH := $(CMAKE_DIR_DBG)/llvm/bin/refcount_checker
......
README.md
View file @ 0abee106
......@@ -136,9 +136,6 @@ Pyston-specific flags:
<dt>-r</dt>
<dd>Use a stripped stdlib. When running pyston_dbg, the default is to use a stdlib with full debugging symbols enabled. Passing -r changes this behavior to load a slimmer, stripped stdlib.</dd>
<dt>-x</dt>
<dd>Enable the pypa parser.</dd>
Standard Python flags:
<dt>-i</dt>
<dd>Go into the repl after executing the given script.</dd>
......
libpypa @ 40eb32aa
Subproject commit 40eb32aa846b21e122cfa1f10bf8972e67de14c8
from_cpython/Lib/test/test_bytes.py
View file @ 0abee106
# expected: fail
# this trips an assert in pypa-parser.cpp
"""Unit tests for the bytes and bytearray types.
XXX This is a mess. Common tests should be moved to buffer_tests.py,
......
from_cpython/Lib/test/test_complex.py
View file @ 0abee106
......@@ -474,10 +474,8 @@ class ComplexTest(unittest.TestCase):
self.assertEqual(repr(complex(0, -INF)), "-infj")
self.assertEqual(repr(complex(0, NAN)), "nanj")
# Pyston change: This is a libpypa bug, waiting for upstream fix it
# please refer libpypa#47
# def test_neg(self):
# self.assertEqual(-(1+6j), -1-6j)
def test_neg(self):
self.assertEqual(-(1+6j), -1-6j)
def test_file(self):
a = 3.33+4.43j
......
src/CMakeLists.txt
View file @ 0abee106
......@@ -7,7 +7,6 @@ set_source_files_properties(jit.cpp PROPERTIES COMPILE_DEFINITIONS "GITREV=${GIT
include_directories(${CMAKE_BINARY_DIR})
include_directories(${CMAKE_BINARY_DIR}/build_deps/libunwind/include)
include_directories(${CMAKE_SOURCE_DIR}/build_deps/libpypa/src)
include_directories(${CMAKE_SOURCE_DIR}/build_deps/lz4/lib)
if(ENABLE_GPERFTOOLS)
......@@ -60,7 +59,6 @@ add_library(PYSTON_OBJECTS OBJECT ${OPTIONAL_SRCS}
codegen/patchpoints.cpp
codegen/profiling/dumprof.cpp
codegen/profiling/profiling.cpp
codegen/pypa-parser.cpp
codegen/runtime_hooks.cpp
codegen/serialize_ast.cpp
codegen/stackmaps.cpp
......@@ -109,10 +107,10 @@ add_library(PYSTON_OBJECTS OBJECT ${OPTIONAL_SRCS}
runtime/util.cpp
)
add_dependencies(PYSTON_OBJECTS copy_stdlib libunwind_patched libunwind pypa ${LLVM_LIBS})
add_dependencies(PYSTON_OBJECTS copy_stdlib libunwind_patched libunwind ${LLVM_LIBS})
add_library(PYSTON_MAIN_OBJECT OBJECT jit.cpp)
add_dependencies(PYSTON_MAIN_OBJECT copy_stdlib libunwind_patched libunwind pypa liblz4 ${LLVM_LIBS})
add_dependencies(PYSTON_MAIN_OBJECT copy_stdlib libunwind_patched libunwind liblz4 ${LLVM_LIBS})
# build stdlib
add_subdirectory(runtime/inline)
......
src/codegen/parser.cpp
View file @ 0abee106
......@@ -27,7 +27,6 @@
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/Path.h"
#include "codegen/pypa-parser.h"
#include "codegen/serialize_ast.h"
#include "core/ast.h"
#include "core/options.h"
......@@ -42,6 +41,10 @@
namespace pyston {
bool DEBUG_PARSING = false;
const char* MAGIC = "a\nCR";
#define MAGIC_STRING_LENGTH 4
#define LENGTH_LENGTH sizeof(int)
#define CHECKSUM_LENGTH 1
class BufferedReader {
private:
......@@ -1001,11 +1004,6 @@ AST* readASTMisc(BufferedReader* reader) {
AST_Module* parse_string(const char* code, FutureFlags inherited_flags) {
inherited_flags &= ~(CO_NESTED | CO_FUTURE_DIVISION);
if (ENABLE_PYPA_PARSER) {
AST_Module* rtn = pypa_parse_string(code, inherited_flags);
RELEASE_ASSERT(rtn, "unknown parse error (possibly: '%s'?)", strerror(errno));
return rtn;
}
PyCompilerFlags cf;
cf.cf_flags = inherited_flags;
ArenaWrapper arena;
......@@ -1022,11 +1020,6 @@ AST_Module* parse_string(const char* code, FutureFlags inherited_flags) {
AST_Module* parse_file(const char* fn, FutureFlags inherited_flags) {
Timer _t("parsing");
if (ENABLE_PYPA_PARSER) {
AST_Module* rtn = pypa_parse(fn, inherited_flags);
RELEASE_ASSERT(rtn, "unknown parse error (possibly: '%s'?)", strerror(errno));
return rtn;
}
FileHandle fp(fn, "r");
PyCompilerFlags cf;
cf.cf_flags = inherited_flags;
......@@ -1040,33 +1033,16 @@ AST_Module* parse_file(const char* fn, FutureFlags inherited_flags) {
return rtn;
}
const char* getMagic() {
if (ENABLE_PYPA_PARSER)
return "a\ncR";
else
return "a\nCR";
}
#define MAGIC_STRING_LENGTH 4
#define LENGTH_LENGTH sizeof(int)
#define CHECKSUM_LENGTH 1
// Does at least one of: returns a valid file_data vector, or fills in 'module'
static std::vector<char> _reparse(const char* fn, const std::string& cache_fn, AST_Module*& module,
FutureFlags inherited_flags) {
inherited_flags &= ~(CO_NESTED | CO_FUTURE_DIVISION);
FileHandle cache_fp(cache_fn.c_str(), "w");
if (DEBUG_PARSING) {
fprintf(stderr, "_reparse('%s', '%s'), pypa=%d\n", fn, cache_fn.c_str(), ENABLE_PYPA_PARSER);
fprintf(stderr, "writing magic string: %d %d %d %d\n", getMagic()[0], getMagic()[1], getMagic()[2],
getMagic()[3]);
}
std::vector<char> file_data;
if (cache_fp)
fwrite(getMagic(), 1, MAGIC_STRING_LENGTH, cache_fp);
file_data.insert(file_data.end(), getMagic(), getMagic() + MAGIC_STRING_LENGTH);
fwrite(MAGIC, 1, MAGIC_STRING_LENGTH, cache_fp);
file_data.insert(file_data.end(), MAGIC, MAGIC + MAGIC_STRING_LENGTH);
int checksum_start = file_data.size();
......@@ -1084,10 +1060,6 @@ static std::vector<char> _reparse(const char* fn, const std::string& cache_fn, A
file_data.insert(file_data.end(), (char*)&checksum, (char*)&checksum + CHECKSUM_LENGTH);
checksum = 0;
if (ENABLE_PYPA_PARSER) {
module = pypa_parse(fn, inherited_flags);
RELEASE_ASSERT(module, "unknown parse error");
} else {
FileHandle fp(fn, "r");
PyCompilerFlags cf;
cf.cf_flags = inherited_flags;
......@@ -1098,7 +1070,6 @@ static std::vector<char> _reparse(const char* fn, const std::string& cache_fn, A
throwCAPIException();
assert(mod->kind != Interactive_kind);
module = static_cast<AST_Module*>(cpythonToPystonAST(mod, fn));
}
if (!cache_fp)
return std::vector<char>();
......@@ -1123,10 +1094,6 @@ static std::vector<char> _reparse(const char* fn, const std::string& cache_fn, A
// on the startup time (40ms -> 10ms).
AST_Module* caching_parse_file(const char* fn, FutureFlags inherited_flags) {
std::ostringstream oss;
if (DEBUG_PARSING) {
oss << "caching_parse_file() on " << fn << '\n';
fprintf(stderr, "caching_parse_file('%s'), pypa=%d\n", fn, ENABLE_PYPA_PARSER);
}
UNAVOIDABLE_STAT_TIMER(t0, "us_timer_caching_parse_file");
static StatCounter us_parsing("us_parsing");
......@@ -1182,12 +1149,12 @@ AST_Module* caching_parse_file(const char* fn, FutureFlags inherited_flags) {
}
if (good) {
if (strncmp(&file_data[0], getMagic(), MAGIC_STRING_LENGTH) != 0) {
if (strncmp(&file_data[0], MAGIC, MAGIC_STRING_LENGTH) != 0) {
oss << "magic string did not match\n";
if (VERBOSITY() >= 2 || tries == MAX_TRIES) {
fprintf(stderr, "Warning: corrupt or non-Pyston .pyc file found; ignoring\n");
fprintf(stderr, "%d %d %d %d\n", file_data[0], file_data[1], file_data[2], file_data[3]);
fprintf(stderr, "%d %d %d %d\n", getMagic()[0], getMagic()[1], getMagic()[2], getMagic()[3]);
fprintf(stderr, "%d %d %d %d\n", MAGIC[0], MAGIC[1], MAGIC[2], MAGIC[3]);
}
good = false;
}
......
src/codegen/pypa-parser.cpp deleted 100644 → 0
View file @ 76c7d503
// Copyright (c) 2014-2016 Dropbox, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "codegen/pypa-parser.h"
#include <cassert>
#include <cstdint>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <memory>
#include <pypa/ast/visitor.hh>
#include <pypa/parser/parser.hh>
#include <sys/stat.h>
#include "llvm/ADT/STLExtras.h"
#include "llvm/Support/SwapByteOrder.h"
#include "core/ast.h"
#include "core/options.h"
#include "core/stats.h"
#include "core/types.h"
#include "core/util.h"
#include "runtime/objmodel.h"
#include "runtime/types.h"
/* For cStringIO: */
#include "Python.h"
#include "cStringIO.h"
namespace pypa {
bool string_to_double(String const& s, double& result);
}
namespace pyston {
void location(AST* t, pypa::Ast& a) {
t->lineno = a.line;
assert(a.column < 100000);
t->col_offset = a.column;
}
AST_expr* readItem(pypa::AstExpression& e, InternedStringPool& interned_strings);
AST_slice* readItem(pypa::AstSliceType& e, InternedStringPool& interned_strings);
AST_stmt* readItem(pypa::AstStatement& s, InternedStringPool& interned_strings);
AST_ExceptHandler* readItem(pypa::AstExcept&, InternedStringPool& interned_strings);
AST_ExceptHandler* readItem(pypa::AstExceptPtr, InternedStringPool& interned_strings);
template <typename T>
auto readItem(std::shared_ptr<T>& t, InternedStringPool& interned_strings) -> decltype(readItem(*t, interned_strings)) {
if (t)
return readItem(*t, interned_strings);
return nullptr;
}
AST_TYPE::AST_TYPE readItem(pypa::AstBoolOpType type) {
switch (type) {
case pypa::AstBoolOpType::And:
return AST_TYPE::And;
case pypa::AstBoolOpType::Or:
return AST_TYPE::Or;
default:
break;
}
assert("Unknown AstBoolOpType" && false);
abort();
}
AST_TYPE::AST_TYPE readItem(pypa::AstBinOpType type) {
switch (type) {
case pypa::AstBinOpType::Add:
return AST_TYPE::Add;
case pypa::AstBinOpType::BitAnd:
return AST_TYPE::BitAnd;
case pypa::AstBinOpType::BitOr:
return AST_TYPE::BitOr;
case pypa::AstBinOpType::BitXor:
return AST_TYPE::BitXor;
case pypa::AstBinOpType::Div:
return AST_TYPE::Div;
case pypa::AstBinOpType::FloorDiv:
return AST_TYPE::FloorDiv;
case pypa::AstBinOpType::LeftShift:
return AST_TYPE::LShift;
case pypa::AstBinOpType::Mod:
return AST_TYPE::Mod;
case pypa::AstBinOpType::Mult:
return AST_TYPE::Mult;
case pypa::AstBinOpType::Power:
return AST_TYPE::Pow;
case pypa::AstBinOpType::RightShift:
return AST_TYPE::RShift;
case pypa::AstBinOpType::Sub:
return AST_TYPE::Sub;
default:
break;
}
assert("Unknown AstBinOpType" && false);
abort();
}
AST_TYPE::AST_TYPE readItem(pypa::AstUnaryOpType type) {
switch (type) {
case pypa::AstUnaryOpType::Add:
return AST_TYPE::UAdd;
case pypa::AstUnaryOpType::Invert:
return AST_TYPE::Invert;
case pypa::AstUnaryOpType::Not:
return AST_TYPE::Not;
case pypa::AstUnaryOpType::Sub:
return AST_TYPE::USub;
default:
break;
}
assert("Unknown AstUnaryOpType" && false);
abort();
}
AST_TYPE::AST_TYPE readItem(pypa::AstContext ctx) {
switch (ctx) {
case pypa::AstContext::Load:
return AST_TYPE::Load;
case pypa::AstContext::Store:
return AST_TYPE::Store;
case pypa::AstContext::AugLoad:
return AST_TYPE::Load;
case pypa::AstContext::AugStore:
return AST_TYPE::Store;
case pypa::AstContext::Param:
return AST_TYPE::Param;
case pypa::AstContext::Del:
return AST_TYPE::Del;
default:
break;
}
assert("Unknown AstContext" && false);
return AST_TYPE::Load;
}
AST_TYPE::AST_TYPE readItem(pypa::AstCompareOpType type) {
switch (type) {
case pypa::AstCompareOpType::Equals:
return AST_TYPE::Eq;
case pypa::AstCompareOpType::In:
return AST_TYPE::In;
case pypa::AstCompareOpType::Is:
return AST_TYPE::Is;
case pypa::AstCompareOpType::IsNot:
return AST_TYPE::IsNot;
case pypa::AstCompareOpType::Less:
return AST_TYPE::Lt;
case pypa::AstCompareOpType::LessEqual:
return AST_TYPE::LtE;
case pypa::AstCompareOpType::More:
return AST_TYPE::Gt;
case pypa::AstCompareOpType::MoreEqual:
return AST_TYPE::GtE;
case pypa::AstCompareOpType::NotEqual:
return AST_TYPE::NotEq;
case pypa::AstCompareOpType::NotIn:
return AST_TYPE::NotIn;
default:
break;
}
assert("Unknown AstCompareOpType" && false);
abort();
}
InternedString readName(pypa::AstExpression& e, InternedStringPool& interned_strings) {
assert(e.type == pypa::AstType::Name);
return interned_strings.get(static_cast<pypa::AstName&>(e).id);
}
InternedString readName(pypa::AstExpr& n, InternedStringPool& interned_strings) {
if (!n) {
return interned_strings.get(std::string());
}
return readName(*n, interned_strings);
}
AST_keyword* readItem(pypa::AstKeyword& k, InternedStringPool& interned_strings) {
AST_keyword* ptr = new AST_keyword();
location(ptr, k);
ptr->arg = readName(k.name, interned_strings);
ptr->value = readItem(k.value, interned_strings);
return ptr;
}
void readVector(std::vector<AST_keyword*>& t, pypa::AstExprList& items, InternedStringPool& interned_strings) {
for (auto& item : items) {
assert(item->type == pypa::AstType::Keyword);
t.push_back(readItem(static_cast<pypa::AstKeyword&>(*item), interned_strings));
}
}
template <typename T, typename U>
void readVector(std::vector<T*>& t, std::vector<U>& u, InternedStringPool& interned_strings) {
for (auto& item : u) {
if (!item) {
t.push_back(nullptr);
} else {
t.push_back(readItem(*item, interned_strings));
}
}
}
void readVector(std::vector<AST_expr*>& t, pypa::AstExpression& u, InternedStringPool& interned_strings) {
if (u.type == pypa::AstType::Tuple) {
pypa::AstTuple& e = static_cast<pypa::AstTuple&>(u);
for (auto& item : e.elements) {
assert(item);
t.push_back(readItem(*item, interned_strings));
}
} else {
t.push_back(readItem(u, interned_strings));
}
}
void readVector(std::vector<AST_slice*>& t, pypa::AstSliceType& u, InternedStringPool& interned_strings) {
t.push_back(readItem(u, interned_strings));
}
void readVector(std::vector<AST_stmt*>& t, pypa::AstStatement& u, InternedStringPool& interned_strings) {
if (u.type == pypa::AstType::Suite) {
pypa::AstSuite& e = static_cast<pypa::AstSuite&>(u);
for (auto& item : e.items) {
assert(item);
t.push_back(readItem(*item, interned_strings));
}
} else {
t.push_back(readItem(u, interned_strings));
}
}
AST_comprehension* readItem(pypa::AstComprehension& c, InternedStringPool& interned_strings) {
AST_comprehension* ptr = new AST_comprehension();
ptr->target = readItem(c.target, interned_strings);
ptr->iter = readItem(c.iter, interned_strings);
readVector(ptr->ifs, c.ifs, interned_strings);
return ptr;
}
AST_comprehension* readItem(pypa::AstComprPtr c, InternedStringPool& interned_strings) {
if (c)
return readItem(*c, interned_strings);
return nullptr;
}
void readVector(std::vector<AST_comprehension*>& t, pypa::AstExprList& u, InternedStringPool& interned_strings) {
for (auto& e : u) {
assert(e && e->type == pypa::AstType::Comprehension);
t.push_back(readItem(static_cast<pypa::AstComprehension&>(*e), interned_strings));
}
}
void readVector(std::vector<AST_slice*>& t, pypa::AstSliceTypePtr u, InternedStringPool& interned_strings) {
if (u) {
readVector(t, *u, interned_strings);
}
}
void readVector(std::vector<AST_stmt*>& t, pypa::AstStmt u, InternedStringPool& interned_strings) {
if (u) {
readVector(t, *u, interned_strings);
}
}
AST_ExceptHandler* readItem(pypa::AstExcept& e, InternedStringPool& interned_strings) {
AST_ExceptHandler* ptr = new AST_ExceptHandler();
location(ptr, e);
readVector(ptr->body, e.body, interned_strings);
ptr->name = readItem(e.name, interned_strings);
ptr->type = readItem(e.type, interned_strings);
return ptr;
}
AST_ExceptHandler* readItem(pypa::AstExceptPtr ptr, InternedStringPool& interned_strings) {
assert(ptr);
return readItem(*ptr, interned_strings);
}
AST_alias* readItem(pypa::AstAlias& a, InternedStringPool& interned_strings) {
return new AST_alias(readName(a.name, interned_strings), readName(a.as_name, interned_strings));
}
AST_arguments* readItem(pypa::AstArguments& a, InternedStringPool& interned_strings) {
AST_arguments* ptr = new AST_arguments();
location(ptr, a);
readVector(ptr->defaults, a.defaults, interned_strings);
ptr->defaults.erase(std::remove(ptr->defaults.begin(), ptr->defaults.end(), nullptr), ptr->defaults.end());
readVector(ptr->args, a.arguments, interned_strings);
InternedString kwarg_name = readName(a.kwargs, interned_strings);
if (kwarg_name.s().size())
ptr->kwarg = new AST_Name(kwarg_name, AST_TYPE::Store, -1, -1);
InternedString vararg_name = readName(a.args, interned_strings);
if (vararg_name.s().size())
ptr->vararg = new AST_Name(vararg_name, AST_TYPE::Store, -1, -1);
return ptr;
}
struct slice_dispatcher {
InternedStringPool& interned_strings;
slice_dispatcher(InternedStringPool& interned_strings) : interned_strings(interned_strings) {}
typedef AST_slice* ResultPtr;
template <typename T> ResultPtr operator()(std::shared_ptr<T> t) {
if (t)
return (*this)(*t);
return nullptr;
}
template <typename T> ResultPtr operator()(T& t) {
pypa::Ast& a = t;
return read(t);
}
template <typename T> ResultPtr read(T& item) {
pypa::Ast& a = item;
RELEASE_ASSERT(0, "Unhandled ast slice type caught: %d @%s\n", a.type, __PRETTY_FUNCTION__);
}
ResultPtr read(pypa::AstEllipsis& e) {
AST_Ellipsis* ptr = new AST_Ellipsis();
location(ptr, e);
return ptr;
}
ResultPtr read(pypa::AstExtSlice& e) {
AST_ExtSlice* ptr = new AST_ExtSlice();
location(ptr, e);
readVector(ptr->dims, e.dims, interned_strings);
return ptr;
}
ResultPtr read(pypa::AstIndex& i) {
AST_Index* ptr = new AST_Index();
location(ptr, i);
ptr->value = readItem(i.value, interned_strings);
return ptr;
}
ResultPtr read(pypa::AstSlice& s) {
AST_Slice* ptr = new AST_Slice();
location(ptr, s);
ptr->lower = readItem(s.lower, interned_strings);
ptr->upper = readItem(s.upper, interned_strings);
ptr->step = readItem(s.step, interned_strings);
return ptr;
}
};
struct expr_dispatcher {
InternedStringPool& interned_strings;
expr_dispatcher(InternedStringPool& interned_strings) : interned_strings(interned_strings) {}
typedef AST_expr* ResultPtr;
template <typename T> ResultPtr operator()(std::shared_ptr<T> t) {
if (t)
return (*this)(*t);
return nullptr;
}
template <typename T> ResultPtr operator()(T& t) {
pypa::Ast& a = t;
return read(t);
}
template <typename T> ResultPtr read(T& item) {
pypa::Ast& a = item;
RELEASE_ASSERT(0, "Unhandled ast expression type caught: %d @%s\n", a.type, __PRETTY_FUNCTION__);
}
ResultPtr read(pypa::AstAttribute& a) {
AST_Attribute* ptr = new AST_Attribute();
location(ptr, a);
ptr->value = readItem(a.value, interned_strings);
ptr->attr = readName(a.attribute, interned_strings);
ptr->ctx_type = readItem(a.context);
return ptr;
}
ResultPtr read(pypa::AstBoolOp& b) {
AST_BoolOp* ptr = new AST_BoolOp();
location(ptr, b);
ptr->op_type = readItem(b.op);
readVector(ptr->values, b.values, interned_strings);
return ptr;
}
ResultPtr read(pypa::AstBinOp& b) {
AST_BinOp* ptr = new AST_BinOp();
location(ptr, b);
ptr->op_type = readItem(b.op);
ptr->left = readItem(b.left, interned_strings);
ptr->right = readItem(b.right, interned_strings);
return ptr;
}
ResultPtr read(pypa::AstCall& c) {
AST_Call* ptr = new AST_Call();
location(ptr, c);
readVector(ptr->args, c.arglist.arguments, interned_strings);
readVector(ptr->keywords, c.arglist.keywords, interned_strings);
ptr->func = readItem(c.function, interned_strings);
ptr->starargs = readItem(c.arglist.args, interned_strings);
ptr->kwargs = readItem(c.arglist.kwargs, interned_strings);
return ptr;
}
ResultPtr read(pypa::AstCompare& c) {
AST_Compare* ptr = new AST_Compare();
location(ptr, c);
ptr->left = readItem(c.left, interned_strings);
ptr->ops.reserve(c.operators.size());
for (auto op : c.operators) {
ptr->ops.push_back(readItem(op));
}
readVector(ptr->comparators, c.comparators, interned_strings);
return ptr;
}
ResultPtr read(pypa::AstComplex& c) {
AST_Num* imag = new AST_Num();
location(imag, c);
imag->num_type = AST_Num::COMPLEX;
sscanf(c.imag.c_str(), "%lf", &imag->n_float);
if (!c.real)
return imag;
AST_BinOp* binop = new AST_BinOp();
location(binop, c);
binop->op_type = AST_TYPE::Add;
binop->left = readItem(c.real, interned_strings);
binop->right = imag;
return binop;
}
ResultPtr read(pypa::AstComprehension& c) {
assert("This should not be called" && false);
return nullptr;
}
ResultPtr read(pypa::AstDict& d) {
AST_Dict* ptr = new AST_Dict();
location(ptr, d);
readVector(ptr->keys, d.keys, interned_strings);
readVector(ptr->values, d.values, interned_strings);
return ptr;
}
ResultPtr read(pypa::AstDictComp& d) {
AST_DictComp* ptr = new AST_DictComp();
location(ptr, d);
ptr->key = readItem(d.key, interned_strings);
ptr->value = readItem(d.value, interned_strings);
readVector(ptr->generators, d.generators, interned_strings);
return ptr;
}
ResultPtr read(pypa::AstIfExpr& i) {
AST_IfExp* ptr = new AST_IfExp();
location(ptr, i);
ptr->body = readItem(i.body, interned_strings);
ptr->test = readItem(i.test, interned_strings);
ptr->orelse = readItem(i.orelse, interned_strings);
return ptr;
}
ResultPtr read(pypa::AstGenerator& g) {
AST_GeneratorExp* ptr = new AST_GeneratorExp();
location(ptr, g);
ptr->elt = readItem(g.element, interned_strings);
readVector(ptr->generators, g.generators, interned_strings);
return ptr;
}
ResultPtr read(pypa::AstLambda& l) {
AST_Lambda* ptr = new AST_Lambda();
location(ptr, l);
ptr->args = readItem(l.arguments, interned_strings);
ptr->body = readItem(l.body, interned_strings);
return ptr;
}
ResultPtr read(pypa::AstList& l) {
AST_List* ptr = new AST_List();
location(ptr, l);
readVector(ptr->elts, l.elements, interned_strings);
ptr->ctx_type = readItem(l.context);
return ptr;
}
ResultPtr read(pypa::AstListComp& l) {
AST_ListComp* ptr = new AST_ListComp();
location(ptr, l);
readVector(ptr->generators, l.generators, interned_strings);
ptr->elt = readItem(l.element, interned_strings);
return ptr;
}
ResultPtr read(pypa::AstName& a) {
AST_Name* ptr = new AST_Name(interned_strings.get(a.id), readItem(a.context), a.line, a.column);
return ptr;
}
ResultPtr read(pypa::AstNone& n) {
AST_Name* ptr = new AST_Name(interned_strings.get("None"), AST_TYPE::Load, n.line, n.column);
return ptr;
}
ResultPtr read(pypa::AstNumber& c) {
AST_Num* ptr = new AST_Num();
location(ptr, c);
switch (c.num_type) {
case pypa::AstNumber::Float:
ptr->num_type = AST_Num::FLOAT;
ptr->n_float = c.floating;
break;
case pypa::AstNumber::Long:
ptr->num_type = AST_Num::LONG;
ptr->n_long = c.str;
break;
default:
ptr->num_type = AST_Num::INT;
ptr->n_int = c.integer;
break;
}
return ptr;
}
ResultPtr read(pypa::AstRepr& r) {
AST_Repr* ptr = new AST_Repr();
location(ptr, r);
ptr->value = readItem(r.value, interned_strings);
return ptr;
}
ResultPtr read(pypa::AstSet& s) {
AST_Set* ptr = new AST_Set();
location(ptr, s);
readVector(ptr->elts, s.elements, interned_strings);
return ptr;
}
ResultPtr read(pypa::AstSetComp& l) {
AST_SetComp* ptr = new AST_SetComp();
location(ptr, l);
readVector(ptr->generators, l.generators, interned_strings);
ptr->elt = readItem(l.element, interned_strings);
return ptr;
}
ResultPtr read(pypa::AstStr& s) {
AST_Str* ptr = new AST_Str();
location(ptr, s);
ptr->str_type = s.unicode ? AST_Str::UNICODE : AST_Str::STR;
ptr->str_data = s.value;
return ptr;
}
ResultPtr read(pypa::AstSubscript& s) {
AST_Subscript* ptr = new AST_Subscript();
location(ptr, s);
ptr->value = readItem(s.value, interned_strings);
ptr->ctx_type = readItem(s.context);
ptr->slice = readItem(s.slice, interned_strings);
return ptr;
}
ResultPtr read(pypa::AstTuple& t) {
AST_Tuple* ptr = new AST_Tuple();
location(ptr, t);
readVector(ptr->elts, t.elements, interned_strings);
ptr->ctx_type = readItem(t.context);
return ptr;
}
ResultPtr read(pypa::AstUnaryOp& b) {
AST_UnaryOp* ptr = new AST_UnaryOp();
location(ptr, b);
ptr->op_type = readItem(b.op);
ptr->operand = readItem(b.operand, interned_strings);
return ptr;
}
ResultPtr read(pypa::AstYieldExpr& e) {
AST_Yield* ptr = new AST_Yield();
location(ptr, e);
ptr->value = readItem(e.args, interned_strings);
return ptr;
}
};
struct stmt_dispatcher {
InternedStringPool& interned_strings;
stmt_dispatcher(InternedStringPool& interned_strings) : interned_strings(interned_strings) {}
typedef AST_stmt* ResultPtr;
template <typename T> ResultPtr operator()(std::shared_ptr<T> t) {
if (t)
return (*this)(*t);
return nullptr;
}
template <typename T> ResultPtr operator()(T& t) {
pypa::Ast& a = t;
return read(t);
}
template <typename T> ResultPtr read(T& item) {
pypa::Ast& a = item;
fprintf(stderr, "Unhandled ast statement type caught: %d @%s\n", a.type, __PRETTY_FUNCTION__);
return nullptr;
}
ResultPtr read(pypa::AstAssign& a) {
AST_Assign* ptr = new AST_Assign();
location(ptr, a);
readVector(ptr->targets, a.targets, interned_strings);
ptr->value = readItem(a.value, interned_strings);
return ptr;
}
ResultPtr read(pypa::AstAssert& a) {
AST_Assert* ptr = new AST_Assert();
location(ptr, a);
ptr->msg = readItem(a.expression, interned_strings);
ptr->test = readItem(a.test, interned_strings);
return ptr;
}
ResultPtr read(pypa::AstAugAssign& a) {
AST_AugAssign* ptr = new AST_AugAssign();
location(ptr, a);
ptr->op_type = readItem(a.op);
ptr->target = readItem(a.target, interned_strings);
ptr->value = readItem(a.value, interned_strings);
return ptr;
}
ResultPtr read(pypa::AstBreak& b) {
AST_Break* ptr = new AST_Break();
location(ptr, b);
return ptr;
}
ResultPtr read(pypa::AstClassDef& c) {
AST_ClassDef* ptr = new AST_ClassDef();
location(ptr, c);
if (c.bases)
readVector(ptr->bases, *c.bases, interned_strings);
readVector(ptr->decorator_list, c.decorators, interned_strings);
readVector(ptr->body, c.body, interned_strings);
ptr->name = readName(c.name, interned_strings);
return ptr;
}
ResultPtr read(pypa::AstContinue& c) {
AST_Continue* ptr = new AST_Continue();
location(ptr, c);
return ptr;
}
ResultPtr read(pypa::AstDelete& d) {
AST_Delete* ptr = new AST_Delete();
location(ptr, d);
readVector(ptr->targets, *d.targets, interned_strings);
return ptr;
}
ResultPtr read(pypa::AstExec& e) {
AST_Exec* ptr = new AST_Exec();
location(ptr, e);
ptr->body = readItem(e.body, interned_strings);
if (e.globals)
ptr->globals = readItem(e.globals, interned_strings);
else
ptr->globals = NULL;
if (e.locals)
ptr->locals = readItem(e.locals, interned_strings);
else
ptr->locals = NULL;
assert(ptr->globals || !ptr->locals);
return ptr;
}
ResultPtr read(pypa::AstExpressionStatement& e) {
AST_Expr* ptr = new AST_Expr();
location(ptr, e);
ptr->value = readItem(e.expr, interned_strings);
return ptr;
}
ResultPtr read(pypa::AstFor& f) {
AST_For* ptr = new AST_For();
location(ptr, f);
ptr->target = readItem(f.target, interned_strings);
if (f.iter)
ptr->iter = readItem(f.iter, interned_strings);
if (f.body)
readVector(ptr->body, *f.body, interned_strings);
if (f.orelse)
readVector(ptr->orelse, *f.orelse, interned_strings);
return ptr;
}
ResultPtr read(pypa::AstFunctionDef& f) {
AST_FunctionDef* ptr = new AST_FunctionDef();
location(ptr, f);
readVector(ptr->decorator_list, f.decorators, interned_strings);
ptr->name = readName(f.name, interned_strings);
ptr->args = readItem(f.args, interned_strings);
readVector(ptr->body, f.body, interned_strings);
return ptr;
}
ResultPtr read(pypa::AstGlobal& g) {
AST_Global* ptr = new AST_Global();
location(ptr, g);
ptr->names.resize(g.names.size());
for (size_t i = 0; i < g.names.size(); ++i) {
ptr->names[i] = readName(*g.names[i], interned_strings);
}
return ptr;
}
ResultPtr read(pypa::AstIf& i) {
AST_If* ptr = new AST_If();
location(ptr, i);
readVector(ptr->body, i.body, interned_strings);
ptr->test = readItem(i.test, interned_strings);
assert(ptr->test != 0);
readVector(ptr->orelse, i.orelse, interned_strings);
return ptr;
}
ResultPtr read(pypa::AstImport& i) {
AST_Import* ptr = new AST_Import();
location(ptr, i);
if (i.names->type == pypa::AstType::Tuple) {
for (auto& name : static_cast<pypa::AstTuple&>(*i.names).elements) {
assert(name->type == pypa::AstType::Alias);
ptr->names.push_back(readItem(static_cast<pypa::AstAlias&>(*name), interned_strings));
}
} else {
assert(i.names->type == pypa::AstType::Alias);
ptr->names.push_back(readItem(static_cast<pypa::AstAlias&>(*i.names), interned_strings));
}
return ptr;
}
ResultPtr read(pypa::AstImportFrom& i) {
AST_ImportFrom* ptr = new AST_ImportFrom();
location(ptr, i);
ptr->module = readName(i.module, interned_strings);
if (i.names->type == pypa::AstType::Tuple) {
for (auto& name : static_cast<pypa::AstTuple&>(*i.names).elements) {
assert(name->type == pypa::AstType::Alias);
ptr->names.push_back(readItem(static_cast<pypa::AstAlias&>(*name), interned_strings));
}
} else {
assert(i.names->type == pypa::AstType::Alias);
ptr->names.push_back(readItem(static_cast<pypa::AstAlias&>(*i.names), interned_strings));
}
ptr->level = i.level;
return ptr;
}
ResultPtr read(pypa::AstPass& p) {
AST_Pass* ptr = new AST_Pass();
location(ptr, p);
return ptr;
}
ResultPtr read(pypa::AstPrint& p) {
AST_Print* ptr = new AST_Print();
location(ptr, p);
ptr->dest = readItem(p.destination, interned_strings);
ptr->nl = p.newline;
readVector(ptr->values, p.values, interned_strings);
return ptr;
}
ResultPtr read(pypa::AstRaise& r) {
AST_Raise* ptr = new AST_Raise();
location(ptr, r);
ptr->arg0 = readItem(r.arg0, interned_strings);
ptr->arg1 = readItem(r.arg1, interned_strings);
ptr->arg2 = readItem(r.arg2, interned_strings);
return ptr;
}
ResultPtr read(pypa::AstSuite& s) { return nullptr; }
ResultPtr read(pypa::AstReturn& r) {
AST_Return* ptr = new AST_Return();
location(ptr, r);
ptr->value = readItem(r.value, interned_strings);
return ptr;
}
ResultPtr read(pypa::AstTryExcept& t) {
AST_TryExcept* ptr = new AST_TryExcept();
location(ptr, t);
readVector(ptr->body, t.body, interned_strings);
readVector(ptr->orelse, t.orelse, interned_strings);
readVector(ptr->handlers, t.handlers, interned_strings);
return ptr;
}
ResultPtr read(pypa::AstTryFinally& t) {
AST_TryFinally* ptr = new AST_TryFinally();
location(ptr, t);
readVector(ptr->body, t.body, interned_strings);
readVector(ptr->finalbody, t.final_body, interned_strings);
return ptr;
}
ResultPtr read(pypa::AstWith& w) {
AST_With* ptr = new AST_With();
location(ptr, w);
ptr->optional_vars = readItem(w.optional, interned_strings);
ptr->context_expr = readItem(w.context, interned_strings);
readVector(ptr->body, w.body, interned_strings);
return ptr;
}
ResultPtr read(pypa::AstWhile& w) {
AST_While* ptr = new AST_While();
location(ptr, w);
ptr->test = readItem(w.test, interned_strings);
readVector(ptr->body, w.body, interned_strings);
readVector(ptr->orelse, w.orelse, interned_strings);
return ptr;
}
ResultPtr read(pypa::AstYield& w) {
AST_Expr* ptr = new AST_Expr();
location(ptr, w);
ptr->value = readItem(w.yield, interned_strings);
return ptr;
}
ResultPtr read(pypa::AstDocString& d) {
AST_Expr* ptr = new AST_Expr();
location(ptr, d);
AST_Str* str = new AST_Str();
location(str, d);
ptr->value = str;
str->str_type = d.unicode ? AST_Str::UNICODE : AST_Str::STR;
str->str_data = d.doc;
return ptr;
}
};
AST_expr* readItem(pypa::AstExpression& e, InternedStringPool& interned_strings) {
return pypa::visit<AST_expr*>(expr_dispatcher(interned_strings), e);
}
AST_slice* readItem(pypa::AstSliceType& e, InternedStringPool& interned_strings) {
return pypa::visit<AST_slice*>(slice_dispatcher(interned_strings), e);
}
AST_stmt* readItem(pypa::AstStatement& s, InternedStringPool& interned_strings) {
return pypa::visit<AST_stmt*>(stmt_dispatcher(interned_strings), s);
}
AST_Module* readModule(pypa::AstModule& t) {
if (VERBOSITY("PYPA parsing") >= 2) {
printf("PYPA reading module\n");
}
AST_Module* mod = new AST_Module(llvm::make_unique<InternedStringPool>());
readVector(mod->body, t.body->items, *mod->interned_strings);
return mod;
}
void pypaErrorHandler(pypa::Error e) {
if (e.type != pypa::ErrorType::SyntaxWarning) {
raiseSyntaxError(e.message.c_str(), e.cur.line, e.cur.column, e.file_name, std::string());
}
}
static PyObject* decode_utf8(const char** sPtr, const char* end, const char* encoding) noexcept {
#ifndef Py_USING_UNICODE
Py_FatalError("decode_utf8 should not be called in this build.");
return NULL;
#else
PyObject* u, *v;
const char* s, *t;
t = s = (const char*)*sPtr;
/* while (s < end && *s != '\\') s++; */ /* inefficient for u".." */
while (s < end && (*s & 0x80))
s++;
*sPtr = s;
u = PyUnicode_DecodeUTF8(t, s - t, NULL);
if (u == NULL)
return NULL;
v = PyUnicode_AsEncodedString(u, encoding, NULL);
Py_DECREF(u);
return v;
#endif
}
#ifdef Py_USING_UNICODE
static PyObject* decode_unicode(const char* s, size_t len, int rawmode, const char* encoding) noexcept {
PyObject* v;
PyObject* u = NULL;
char* buf;
char* p;
const char* end;
if (encoding != NULL && strcmp(encoding, "iso-8859-1")) {
/* check for integer overflow */
if (len > PY_SIZE_MAX / 6)
return NULL;
/* "<C3><A4>" (2 bytes) may become "\U000000E4" (10 bytes), or 1:5
"\ä" (3 bytes) may become "\u005c\U000000E4" (16 bytes), or ~1:6 */
u = PyString_FromStringAndSize((char*)NULL, len * 6);
if (u == NULL)
return NULL;
p = buf = PyString_AsString(u);
end = s + len;
while (s < end) {
if (*s == '\\') {
*p++ = *s++;
if (*s & 0x80) {
strcpy(p, "u005c");
p += 5;
}
}
if (*s & 0x80) { /* XXX inefficient */
PyObject* w;
char* r;
Py_ssize_t rn, i;
w = decode_utf8(&s, end, "utf-32-be");
if (w == NULL) {
Py_DECREF(u);
return NULL;
}
r = PyString_AsString(w);
rn = PyString_Size(w);
assert(rn % 4 == 0);
for (i = 0; i < rn; i += 4) {
sprintf(p, "\\U%02x%02x%02x%02x", r[i + 0] & 0xFF, r[i + 1] & 0xFF, r[i + 2] & 0xFF,
r[i + 3] & 0xFF);
p += 10;
}
Py_DECREF(w);
} else {
*p++ = *s++;
}
}
len = p - buf;
s = buf;
}
if (rawmode)
v = PyUnicode_DecodeRawUnicodeEscape(s, len, NULL);
else
v = PyUnicode_DecodeUnicodeEscape(s, len, NULL);
Py_XDECREF(u);
return v;
}
#endif
pypa::String pypaEscapeDecoder(const pypa::String& s, const pypa::String& encoding, bool unicode, bool raw_prefix,
bool& error) {
try {
error = false;
if (unicode) {
PyObject* str = decode_unicode(s.c_str(), s.size(), raw_prefix, encoding.c_str());
if (!str)
throwCAPIException();
AUTO_DECREF(str);
BoxedString* str_utf8 = (BoxedString*)PyUnicode_AsUTF8String(str);
AUTO_DECREF(str_utf8);
assert(str_utf8->cls == str_cls);
if (!str_utf8)
throwCAPIException();
return str_utf8->s().str();
}
bool need_encoding = encoding != "utf-8" && encoding != "iso-8859-1";
if (raw_prefix || s.find('\\') == pypa::String::npos) {
if (need_encoding) {
PyObject* u = PyUnicode_DecodeUTF8(s.c_str(), s.size(), NULL);
if (!u)
throwCAPIException();
AUTO_DECREF(u);
BoxedString* str = (BoxedString*)PyUnicode_AsEncodedString(u, encoding.c_str(), NULL);
AUTO_DECREF(str);
assert(str->cls == str_cls);
return str->s().str();
} else {
return s;
}
}
BoxedString* decoded = (BoxedString*)PyString_DecodeEscape(s.c_str(), s.size(), NULL, false,
need_encoding ? encoding.c_str() : NULL);
if (!decoded)
throwCAPIException();
AUTO_DECREF(decoded);
assert(decoded->cls == str_cls);
return decoded->s().str();
} catch (ExcInfo e) {
error = true;
BoxedString* error_message = str(e.value);
e.clear();
AUTO_DECREF(error_message);
if (error_message && error_message->cls == str_cls)
return std::string(error_message->s());
return "Encountered an unknown error inside pypaEscapeDecoder";
}
}
class PystonReader : public pypa::Reader {
public:
PystonReader();
~PystonReader() override;
bool set_encoding(const std::string& coding) override;
std::string get_line() override;
unsigned get_line_number() const override { return line_number; }
virtual char next() = 0;
virtual PyObject* open_python_file() noexcept = 0;
bool eof() const override { return is_eof; }
void set_eof() { is_eof = true; }
private:
bool is_eof;
PyObject* file;
PyObject* reader;
PyObject* readline;
unsigned line_number;
};
PystonReader::PystonReader() : is_eof(false), file(nullptr), reader(nullptr), readline(nullptr), line_number(0) {
}
PystonReader::~PystonReader() {
Py_CLEAR(readline);
Py_CLEAR(reader);
Py_CLEAR(file);
}
bool PystonReader::set_encoding(const std::string& coding) {
assert(!file && !reader && !readline);
file = open_python_file();
if (file == NULL)
return false;
reader = PyCodec_StreamReader(coding.c_str(), file, NULL);
if (reader == NULL)
return false;
readline = PyObject_GetAttrString(reader, "readline");
if (readline == NULL)
return false;
return true;
}
std::string PystonReader::get_line() {
if (eof())
return std::string();
if (!readline) {
std::string line;
char c;
do {
c = next();
if (eof())
break;
line.push_back(c);
} while (c != '\n' && c != '\x0c');
// check for UTF8 BOM
if (line_number == 0 && line[0] == '\xEF' && line[1] == '\xBB' && line[2] == '\xBF') {
set_encoding("utf-8");
line.erase(0, 3);
}
++line_number;
return line;
}
BoxedString* line = (BoxedString*)runtimeCall(readline, ArgPassSpec(0), 0, 0, 0, 0, 0);
if (line->cls == unicode_cls) {
AUTO_DECREF(line);
line = (BoxedString*)PyUnicode_AsUTF8String(line);
if (line == NULL) {
is_eof = true;
return std::string();
}
}
AUTO_DECREF(line);
assert(line->cls == str_cls);
if (!line->size())
is_eof = true;
++line_number;
return line->s();
}
class PystonFileReader : public PystonReader {
public:
PystonFileReader(FILE* file, std::string file_path);
~PystonFileReader() override;
PyObject* open_python_file() noexcept override;
std::string get_filename() const override { return file_path; }
static std::unique_ptr<PystonFileReader> create(const char* path);
private:
char next() override;
FILE* file;
std::string file_path;
};
PystonFileReader::PystonFileReader(FILE* file, std::string file_path) : file(file), file_path(std::move(file_path)) {
}
PystonFileReader::~PystonFileReader() {
if (file)
fclose(file);
file = nullptr;
file_path.clear();
set_eof();
}
std::unique_ptr<PystonFileReader> PystonFileReader::create(const char* path) {
FILE* f = fopen(path, "r");
if (!f)
return nullptr;
return llvm::make_unique<PystonFileReader>(f, path);
}
PyObject* PystonFileReader::open_python_file() noexcept {
return PyFile_FromFile(file, file_path.c_str(), "rb", NULL);
}
char PystonFileReader::next() {
if (eof())
return 0;
int c = fgetc(file);
if (c == EOF) {
set_eof();
return 0;
}
return c;
}
class PystonStringReader : public PystonReader {
public:
PystonStringReader(const char* str) : str(str), position(0) {}
~PystonStringReader() override {}
std::string get_filename() const override { return "<stdin>"; }
private:
char next() override;
PyObject* open_python_file() noexcept override;
const char* str;
int position;
};
static AST_Module* parse_with_reader(std::unique_ptr<pypa::Reader> reader, FutureFlags future_flags) {
pypa::Lexer lexer(std::move(reader));
pypa::SymbolTablePtr symbols;
pypa::AstModulePtr module;
pypa::ParserOptions options;
options.perform_inline_optimizations = true;
options.printerrors = false;
options.python3allowed = false;
options.python3only = false;
options.handle_future_errors = false;
options.error_handler = pypaErrorHandler;
options.escape_handler = pypaEscapeDecoder;
if (future_flags & CO_FUTURE_PRINT_FUNCTION) {
future_flags &= ~CO_FUTURE_PRINT_FUNCTION;
options.initial_future_features.print_function = true;
}
if (future_flags & CO_FUTURE_DIVISION) {
future_flags &= ~CO_FUTURE_DIVISION;
options.initial_future_features.division = true;
}
if (future_flags & CO_FUTURE_ABSOLUTE_IMPORT) {
future_flags &= ~CO_FUTURE_ABSOLUTE_IMPORT;
options.initial_future_features.absolute_imports = true;
}
if (future_flags & CO_FUTURE_WITH_STATEMENT) {
future_flags &= ~CO_FUTURE_WITH_STATEMENT;
options.initial_future_features.with_statement = true;
}
if (future_flags & CO_FUTURE_UNICODE_LITERALS) {
future_flags &= ~CO_FUTURE_UNICODE_LITERALS;
options.initial_future_features.unicode_literals = true;
}
// Strip out some flags:
future_flags &= ~(CO_NESTED);
RELEASE_ASSERT(!future_flags, "0x%x", future_flags);
if (pypa::parse(lexer, module, symbols, options) && module) {
return readModule(*module);
}
return nullptr;
}
char PystonStringReader::next() {
char c = str[position];
if (c)
position++;
else
set_eof();
return c;
}
PyObject* PystonStringReader::open_python_file() noexcept {
PycString_IMPORT;
PyObject* s = PyString_FromString(str + position);
if (!s)
return s;
AUTO_DECREF(s);
return PycStringIO->NewInput(s);
}
AST_Module* pypa_parse(char const* file_path, FutureFlags future_flags) {
auto reader = PystonFileReader::create(file_path);
if (!reader)
return nullptr;
return parse_with_reader(std::move(reader), future_flags);
}
AST_Module* pypa_parse_string(char const* str, FutureFlags future_flags) {
auto reader = llvm::make_unique<PystonStringReader>(str);
return parse_with_reader(std::move(reader), future_flags);
}
}
src/codegen/pypa-parser.h deleted 100644 → 0
View file @ 76c7d503
// Copyright (c) 2014-2016 Dropbox, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef PYSTON_CODEGEN_PYPAPARSER_H
#define PYSTON_CODEGEN_PYPAPARSER_H
#include <cstdio>
#include "core/types.h"
namespace pyston {
class AST_Module;
AST_Module* pypa_parse(char const* file_path, FutureFlags future_flags);
AST_Module* pypa_parse_string(char const* str, FutureFlags future_flags);
}
#endif // PYSTON_CODEGEN_PYPAPARSER_H
src/core/cfg.cpp
View file @ 0abee106
......@@ -2362,21 +2362,6 @@ public:
bool visit_tryexcept(AST_TryExcept* node) override {
assert(curblock);
// The pypa parser will generate a tryexcept node inside a try-finally block with
// no except clauses
if (node->handlers.size() == 0) {
assert(ENABLE_PYPA_PARSER);
assert(node->orelse.size() == 0);
for (AST_stmt* subnode : node->body) {
subnode->accept(this);
if (!curblock)
break;
}
return true;
}
assert(node->handlers.size() > 0);
CFGBlock* exc_handler_block = cfg->addDeferredBlock();
......
src/core/options.cpp
View file @ 0abee106
......@@ -38,7 +38,6 @@ bool PROFILE = false;
bool DUMPJIT = false;
bool TRAP = false;
bool USE_STRIPPED_STDLIB = true; // always true
bool ENABLE_PYPA_PARSER = false;
bool USE_REGALLOC_BASIC = false;
bool PAUSE_AT_ABORT = false;
bool ENABLE_TRACEBACKS = true;
......
src/core/options.h
View file @ 0abee106
......@@ -36,8 +36,8 @@ extern int SPECULATION_THRESHOLD;
extern int MAX_OBJECT_CACHE_ENTRIES;
extern bool SHOW_DISASM, FORCE_INTERPRETER, FORCE_OPTIMIZE, PROFILE, DUMPJIT, TRAP, USE_STRIPPED_STDLIB,
CONTINUE_AFTER_FATAL, ENABLE_INTERPRETER, ENABLE_BASELINEJIT, ENABLE_PYPA_PARSER, USE_REGALLOC_BASIC,
PAUSE_AT_ABORT, ENABLE_TRACEBACKS, FORCE_LLVM_CAPI_CALLS, FORCE_LLVM_CAPI_THROWS;
CONTINUE_AFTER_FATAL, ENABLE_INTERPRETER, ENABLE_BASELINEJIT, USE_REGALLOC_BASIC, PAUSE_AT_ABORT, ENABLE_TRACEBACKS,
FORCE_LLVM_CAPI_CALLS, FORCE_LLVM_CAPI_THROWS;
extern bool LOG_IC_ASSEMBLY, LOG_BJIT_ASSEMBLY;
......
src/jit.cpp
View file @ 0abee106
......@@ -219,8 +219,6 @@ int handleArg(char code) {
USE_STRIPPED_STDLIB = true;
} else if (code == 'b') {
USE_REGALLOC_BASIC = false;
} else if (code == 'x') {
ENABLE_PYPA_PARSER = true;
} else if (code == 'E') {
Py_IgnoreEnvironmentFlag = 1;
} else if (code == 'P') {
......
test/CPYTHON_TEST_NOTES.md
View file @ 0abee106
......@@ -10,7 +10,6 @@ The CPython tests I've included fail for various reasons. Recurring issues inclu
- missing `__name__` attributes on capifuncs
- missing `sys.__stdout__` attribute
- `serialize_ast.cpp`: `writeColOffset: assert(v < 100000 || v == -1)` gets tripped
- `pypa-parser.cpp`: readName: `assert(e.type == pypa::AstType::Name)`
- `src/runtime/util.cpp`: `parseSlice`: `assert(isSubclass(start->cls, int_cls) || start->cls == none_cls)`
## List of files & why they're failing
......
test/tests/property.py
View file @ 0abee106
# I think pypa has an issue parsing decorator expressions if they aren't simple names
# https://github.com/vinzenz/libpypa/issues/15
class C(object):
def fget(self):
return 5
......
test/unittests/CMakeLists.txt
View file @ 0abee106
......@@ -9,7 +9,7 @@ add_custom_target(unittests)
macro(add_unittest unittest)
add_executable(${unittest}_unittest EXCLUDE_FROM_ALL ${unittest}.cpp $<TARGET_OBJECTS:PYSTON_OBJECTS> $<TARGET_OBJECTS:FROM_CPYTHON>)
target_link_libraries(${unittest}_unittest stdlib z sqlite3 gmp mpfr ssl crypto readline pypa liblz4 double-conversion unwind gtest gtest_main util ${LLVM_LIBS} ${LIBLZMA_LIBRARIES})
target_link_libraries(${unittest}_unittest stdlib z sqlite3 gmp mpfr ssl crypto readline liblz4 unwind gtest gtest_main util ${LLVM_LIBS} ${LIBLZMA_LIBRARIES})
add_dependencies(unittests ${unittest}_unittest)
endmacro()
......
tools/CMakeLists.txt
View file @ 0abee106
......@@ -5,8 +5,3 @@ llvm_map_components_to_libnames(LLVM_LIBS_EXTRA BitWriter)
add_executable(publicize publicize.cpp)
target_link_libraries(publicize curses dl pthread ${LLVM_LIBS} ${LLVM_LIBS_EXTRA})
include_directories(${CMAKE_SOURCE_DIR}/src)
add_executable(astprint EXCLUDE_FROM_ALL astprint.cpp $<TARGET_OBJECTS:PYSTON_OBJECTS> $<TARGET_OBJECTS:FROM_CPYTHON>)
target_link_libraries(astprint stdlib gmp ssl crypto readline pypa double-conversion unwind ${LLVM_LIBS} ${LIBLZMA_LIBRARIES})
tools/astprint.cpp deleted 100644 → 0
View file @ 76c7d503
#include "analysis/scoping_analysis.h"
#include "codegen/parser.h"
#include "codegen/entry.h"
#include "core/cfg.h"
#include "runtime/objmodel.h"
#include "runtime/types.h"
using namespace pyston;
int main(int argc, char const ** argv) {
GLOBAL_VERBOSITY=0;
threading::registerMainThread();
threading::GLReadRegion _glock;
initCodegen();
if(argc > 2 && argv[1][0] == '-' && argv[1][1] == 'x') {
ENABLE_PYPA_PARSER = false;
}
std::string fn = argv[1 + int(argc > 2)];
try {
AST_Module* m = caching_parse(fn.c_str());
PrintVisitor* visitor = new PrintVisitor(4);
visitor->visit_module(m);
} catch (Box* b) {
std::string msg = formatException(b);
printLastTraceback();
fprintf(stderr, "%s\n", msg.c_str());
return 1;
}
return 0;
}
tools/astprint_test.sh deleted 100755 → 0
View file @ 76c7d503
#! /bin/sh
TOTAL_RESULT=0
pushd `dirname $0` > /dev/null
SCRIPTPATH=`pwd -P`
popd > /dev/null
pad=$(printf '%0.1s' " "{1..90})
TESTSDIR=`realpath $SCRIPTPATH/../test/tests`
mkdir -p results
for test_script in `find $TESTSDIR -name "*.py"`
do
resultspath="./results/${test_script#$TESTSDIR}"
echo $resultspath | grep "encoding" && continue
echo $resultspath | grep "codec" && continue
$SCRIPTPATH/astprint $test_script 2>&1 > /dev/null || (echo $test_script "[SKIPPED]" && continue)
mkdir -p `dirname $resultspath`
touch $resultspath.python $resultspath.pypa
$SCRIPTPATH/astprint $test_script > $resultspath.python
grep "Warning: converting unicode literal to str" $resultspath.python 2>&1 > /dev/null && rm -f $resultspath.python && continue
$SCRIPTPATH/astprint -x $test_script > $resultspath.pypa
diff -q $resultspath.python $resultspath.pypa 2>&1 > /dev/null && result=`echo -e "[\033[0;32mSUCCESS\033[0m]"` && rm -f $resultspath.pypa $resultspath.python || result=`echo -e "[\033[0;31mFAILED\033[0m]"` && TOTALRESULT=1
reltestscript=$(perl -MFile::Spec -e "print File::Spec->abs2rel(q($test_script),q($SCRIPTPATH))")
echo "$reltestscript${pad:${#test_script}} " $result
done
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7