Skip to content

P
Pyston
Commit 3a811c26 authored by Kevin Modzelewski's avatar Kevin Modzelewski Committed by Kevin Modzelewski
Browse files
Options

Add documentation about some structs/classes

parent 8e2a0949
Show whitespace changes
Inline Side-by-side
Showing with 167 additions and 38 deletions
src/codegen/irgen/hooks.cpp
View file @ 3a811c26
......@@ -772,11 +772,11 @@ CompiledFunction::CompiledFunction(llvm::Function* func, FunctionSpecialization*
ExceptionStyle exception_style, const OSREntryDescriptor* entry_descriptor)
: clfunc(NULL),
func(func),
effort(effort),
exception_style(exception_style),
spec(spec),
entry_descriptor(entry_descriptor),
code(code),
effort(effort),
exception_style(exception_style),
times_called(0),
times_speculation_failed(0),
location_map(nullptr) {
......
src/core/stats.h
View file @ 3a811c26
......@@ -26,8 +26,10 @@
namespace pyston {
// Set this to 1 to disable all stats-related operations. Shouldn't usually be necessary.
#define DISABLE_STATS 0
// Enable certain expensive stat collections:
#define STAT_ALLOCATIONS (0 && !DISABLE_STATS)
#define STAT_ALLOCATION_TYPES (0 && !DISABLE_STATS)
#define STAT_CALLATTR_DESCR_ABORTS (0 && !DISABLE_STATS)
......@@ -53,6 +55,8 @@ namespace pyston {
#define STAT_TIMER_NAME(id) _st##id
#if !DISABLE_STATS
// The class that stores and manages stats collection. For normal stats collections purposes,
// you shouldn't have to use this class, and will usually want to use StatCounter instead.
struct Stats {
private:
static std::unordered_map<uint64_t*, std::string>* names;
......@@ -75,6 +79,22 @@ public:
static void endOfInit();
};
// A helper class for efficient stats collections. Typical usage:
//
// static StatCounter my_stat_counter("my_informative_stat_name");
// void myFunction() {
// my_stat_counter.log();
// }
//
// The current convention for stat names is underscore_case, such as `num_cxa_throw`,
// though at some point we'd like to move to a period-delimited convention.
// (Run `./pyston -s` to see the list of current stats we log.)
// For single stats, usually `num_foo` is a decent name. If there are many stats in a
// single category, you can drop the `num_`.
// If a stat name is a prefix of another, the event it is counting should be a superset.
// For instance, `ic_rewrites` counts a superset of the events that `ic_rewrites_aborted`
// counts, which itself is a superset of the events that `ic_rewrites_aborted_assemblyfail`
// counts.
struct StatCounter {
private:
uint64_t* counter;
......@@ -85,6 +105,11 @@ public:
void log(uint64_t count = 1) { *counter += count; }
};
// Similar to StatCounter, but should be allocated as:
//
// static thread_local StatPerThreadCounter my_stat_counter("cool_stat_name");
//
// This will automatically add the thread id to the stat name.
struct StatPerThreadCounter {
private:
uint64_t* counter = 0;
......@@ -117,6 +142,9 @@ struct StatPerThreadCounter {
#endif
#if STAT_TIMERS
// StatTimers are for a specific type of profiling investigation. Until we make this more usable,
// there probably shouldn't be more changes or uses of this class.
class StatTimer {
private:
static __thread StatTimer* stack;
......@@ -226,6 +254,8 @@ public:
static void assertActive() { ASSERT(stack && !stack->isPaused(), ""); }
};
// Helper class around a StatTimer
class ScopedStatTimer {
private:
StatTimer timer;
......@@ -237,6 +267,7 @@ public:
}
~ScopedStatTimer() { timer.popNonTopLevel(); }
};
#else
struct StatTimer {
StatTimer(uint64_t*) {}
......
src/core/threading.h
View file @ 3a811c26
......@@ -138,6 +138,7 @@ void demoteGL();
// Helper macro for creating a RAII wrapper around two functions.
#define MAKE_REGION(name, start, end) \
class name { \
public: \
......
src/core/types.h
View file @ 3a811c26
......@@ -41,16 +41,35 @@ namespace pyston {
using gc::GCVisitor;
// The "effort" which we will put into compiling a Python function. This applies to the LLVM tier,
// where it can affect things such as the amount of type analysis we do, whether or not to run expensive
// LLVM optimization passes, etc.
// Currently (Nov '15) these are mostly unused, and we only use MAXIMAL. There used to be two other levels
// as well but we stopped using them too.
enum class EffortLevel {
MODERATE = 2,
MAXIMAL = 3,
};
// Pyston supports two ways of implementing Python exceptions: by using return-code-based exceptions ("CAPI"
// style since this is what the CPython C API uses), or our custom C++-based exceptions ("CXX" style). CAPI
// is faster when an exception is thrown, and CXX is faster when an exception is not thrown, so depending on
// the situation it can be beneficial to use one or the other. The JIT will use some light profiling to
// determine when to emit code in one style or the other.
// Many runtime functions support being called in either style, and can get passed one of these enum values
// as a template parameter to switch between them.
enum ExceptionStyle {
CAPI,
CXX,
};
// Much of our runtime supports "rewriting" aka tracing itself. Our tracing JIT requires support from the
// functions to be traced, so our runtime functions will contain checks to see if the tracer is currently
// activated, and then will do additional work.
// When the tracer isn't active, these extra "is the tracer active" checks can become expensive. So when
// the caller knows that the tracer is not active, they can call a special version of the function where
// all of the "is the tracer active" checks are hardcoded to false. This is possible by passing "NOT_REWRITEABLE"
// as a template argument to the called function.
enum Rewritable {
NOT_REWRITABLE,
REWRITABLE,
......@@ -88,6 +107,8 @@ public:
template <ExceptionStyle S> R call(Args... args) noexcept(S == CAPI) { return this->template get<S>()(args...); }
};
// CompilerType (and a specific kind of CompilerType, the ConcreteCompilerType) are the way that the LLVM JIT represents
// type information. See src/codegen/compvars.h for more information.
class CompilerType;
template <class V> class ValuedCompilerType;
typedef ValuedCompilerType<llvm::Value*> ConcreteCompilerType;
......@@ -98,6 +119,8 @@ extern ConcreteCompilerType* UNBOXED_INT, *BOXED_INT, *LONG, *UNBOXED_FLOAT, *BO
*BOXED_COMPLEX, *FRAME_INFO;
extern CompilerType* UNDEF, *INT, *FLOAT, *UNBOXED_SLICE;
// CompilerVariables are the way that the LLVM JIT tracks variables, which are a CompilerType combined with some sort
// of value (the type of value depends on the type of CompilerType).
class CompilerVariable;
template <class V> class ValuedCompilerVariable;
typedef ValuedCompilerVariable<llvm::Value*> ConcreteCompilerVariable;
......@@ -125,6 +148,10 @@ class ScopingAnalysis;
class CLFunction;
class OSREntryDescriptor;
// Pyston's internal calling convention is to pass around arguments in as unprocessed a form as possible,
// which lets the callee decide how they would like to receive their arguments. In addition to the actual
// argument parameters, functions will often receive an ArgPassSpec struct which specifies the meaning of
// the raw pointer values, such as whether they were positional arguments or keyword arguments, etc.
struct ArgPassSpec {
bool has_starargs : 1;
bool has_kwargs : 1;
......@@ -196,7 +223,8 @@ private:
ParamNames() : takes_param_names(false), vararg_name(NULL), kwarg_name(NULL) {}
};
// Probably overkill to copy this from ArgPassSpec
// Similar to ArgPassSpec, this struct is how functions specify what their parameter signature is.
// (Probably overkill to copy this from ArgPassSpec)
struct ParamReceiveSpec {
bool takes_varargs : 1;
bool takes_kwargs : 1;
......@@ -228,6 +256,14 @@ struct ParamReceiveSpec {
int kwargsIndex() { return num_args + (takes_varargs ? 1 : 0); }
};
// Inline-caches contain fastpath code, and need to know that their fastpath is valid for a particular set
// of arguments. This is usually done with guards: conditional checks that will avoid the fastpath if the
// assumptions failed. This can also be done using invalidation: no checks will be emitted into the generated
// assembly, but instead if the assumption is invalidated, the IC will get erased.
// This is useful for cases where we expect the assumption to overwhelmingly be true, or cases where it
// is not possible to use guards. It is more difficult to use invalidation because it is much easier to
// get it wrong by forgetting to invalidate in all places that are necessary (whereas it is easier to be
// conservative about guarding).
class ICInvalidator {
private:
int64_t cur_version;
......@@ -259,17 +295,31 @@ class ICInfo;
class LocationMap;
class JitCodeBlock;
// A specific compilation of a CLFunction. Usually these will be created by the LLVM JIT, which will take a CLFunction
// and some compilation settings, and produce a CompiledFunction
// CompiledFunctions can also be created from raw function pointers, using boxRTFunction.
// A single CLFunction can have multiple CompiledFunctions associated with it, if they have different settings.
// Typically, this will happen due to specialization on the argument types (ie we will generate a separate versions
// of a function that are faster but only apply to specific argument types).
struct CompiledFunction {
private:
public:
CLFunction* clfunc;
llvm::Function* func; // the llvm IR object
// Some compilation settings:
EffortLevel effort;
ExceptionStyle exception_style;
FunctionSpecialization* spec;
// If this compilation was due to an OSR, `entry_descriptor` contains metadata about the OSR.
// Otherwise this field is NULL.
const OSREntryDescriptor* entry_descriptor;
// Pointers that were written directly into the code, which the GC should be aware of.
std::vector<const void*> pointers_in_code;
// The function pointer to the generated code. For convenience, it can be accessed
// as one of many different types.
union {
Box* (*call)(Box*, Box*, Box*, Box**);
Box* (*closure_call)(BoxedClosure*, Box*, Box*, Box*, Box**);
......@@ -283,14 +333,20 @@ public:
};
int code_size;
EffortLevel effort;
ExceptionStyle exception_style;
// Some simple profiling stats:
int64_t times_called, times_speculation_failed;
// A list of ICs that depend on various properties of this CompiledFunction.
// These will get invalidated in situations such as: we compiled a higher-effort version of
// this function so we want to get old callsites to use the newer and better version, or
// we noticed that we compiled the function with speculations that kept on failing and
// we want to generate a more conservative version.
ICInvalidator dependent_callsites;
// Metadata that lets us find local variables from the C stack fram.
LocationMap* location_map;
// List of metadata objects for ICs inside this compilation
std::vector<ICInfo*> ics;
CompiledFunction(llvm::Function* func, FunctionSpecialization* spec, void* code, EffortLevel effort,
......@@ -316,9 +372,12 @@ class BoxedModule;
class ScopeInfo;
class InternedStringPool;
class LivenessAnalysis;
// Data about a single textual function definition.
class SourceInfo {
private:
BoxedString* fn; // equivalent of code.co_filename
public:
BoxedModule* parent_module;
ScopingAnalysis* scoping;
......@@ -354,16 +413,24 @@ private:
typedef std::vector<CompiledFunction*> FunctionList;
struct CallRewriteArgs;
// A BoxedCode is our implementation of the Python "code" object (such as function.func_code).
// It is implemented as a wrapper around a CLFunction.
class BoxedCode;
class CLFunction {
private:
// The Python-level "code" object corresponding to this CLFunction. We store it in the CLFunction
// so that multiple attempts to translate from CLFunction->BoxedCode will always return the same
// BoxedCode object.
// Callers should use getCode()
BoxedCode* code_obj;
public:
int num_args;
bool takes_varargs, takes_kwargs;
//ParamReceiveSpec paramspec;
std::unique_ptr<SourceInfo> source;
std::unique_ptr<SourceInfo> source; // source can be NULL for functions defined in the C/C++ runtime
ParamNames param_names;
FunctionList
......@@ -371,6 +438,7 @@ public:
CompiledFunction* always_use_version; // if this version is set, always use it (for unboxed cases)
std::unordered_map<const OSREntryDescriptor*, CompiledFunction*> osr_versions;
// Profiling counter:
int propagated_cxx_exceptions = 0;
// For use by the interpreter/baseline jit:
......@@ -426,6 +494,9 @@ typedef int64_t i64;
const char* getNameOfClass(BoxedClass* cls);
std::string getFullNameOfClass(BoxedClass* cls);
std::string getFullTypeName(Box* o);
const char* getTypeName(Box* b);
class Rewriter;
class RewriterVar;
......@@ -491,6 +562,8 @@ inline void internStringMortalInplace(BoxedString*& s) noexcept {
PyString_InternInPlace((PyObject**)&s);
}
// The data structure definition for hidden-class-based attributes. Consists of a
// pointer to the hidden class object, and a pointer to a variable-size attributes array.
struct HCAttrs {
public:
struct AttrList {
......@@ -507,12 +580,13 @@ static_assert(sizeof(HCAttrs) == sizeof(struct _hcattrs), "");
class BoxedDict;
class BoxedString;
// In Pyston, this is the same type as CPython's PyObject (they are interchangeable, but we
// use Box in Pyston wherever possible as a convention).
// "Box" is the base class of any C++ type that implements a Python type. For example,
// BoxedString is the data structure that implements Python's str type, and BoxedString
// inherits from Box.
//
// Other types on Pyston inherit from Box (e.g. BoxedString is a Box). Why is this class not
// polymorphic? Because of C extension support -- having virtual methods would change the layout
// of the object.
// This is the same as CPython's PyObject (and they are interchangeable), with the difference
// since we are in C++ (whereas CPython is in C) we can use C++ inheritance to implement
// Python inheritance, and avoid the raw pointer casts that CPython needs everywhere.
class Box {
private:
BoxedDict** getDictPtr();
......@@ -573,6 +647,12 @@ static_assert(offsetof(Box, cls) == offsetof(struct _object, ob_type), "");
// Our default for tp_alloc:
extern "C" PyObject* PystonType_GenericAlloc(BoxedClass* cls, Py_ssize_t nitems) noexcept;
// These are some macros for tying the C++ type hiercharchy to the Pyston type hiercharchy.
// Classes that inherit from Box have a special operator new() that takes a class object (as
// a BoxedClass*) since the class is necessary for object allocation.
// To enable expressions such as `new BoxedString()` instead of having to type
// `new (str_cls) BoxedString()` everywhere, we need to tell C++ what the default class is.
// We can do this by putting `DEFAULT_CLASS(str_cls);` anywhere in the definition of BoxedString.
#define DEFAULT_CLASS(default_cls) \
void* operator new(size_t size, BoxedClass * cls) __attribute__((visibility("default"))) { \
assert(cls->tp_itemsize == 0); \
......@@ -615,6 +695,11 @@ extern "C" PyObject* PystonType_GenericAlloc(BoxedClass* cls, Py_ssize_t nitems)
#endif
// In the simple cases, we can inline the fast paths of the following methods and improve allocation speed quite a bit:
// - Box::operator new
// - cls->tp_alloc
// - PystonType_GenericAlloc
// - PyObject_Init
// The restrictions on when you can use the SIMPLE (ie fast) variant are encoded as
// asserts in the 1-arg operator new function:
#define DEFAULT_CLASS_SIMPLE(default_cls) \
......@@ -623,12 +708,6 @@ extern "C" PyObject* PystonType_GenericAlloc(BoxedClass* cls, Py_ssize_t nitems)
} \
void* operator new(size_t size) __attribute__((visibility("default"))) { \
ALLOC_STATS(default_cls); \
/* In the simple cases, we can inline the following methods and simplify things a lot: \
* - Box::operator new \
* - cls->tp_alloc \
* - PystonType_GenericAlloc \
* - PyObject_Init \
*/ \
assert(default_cls->tp_alloc == PystonType_GenericAlloc); \
assert(default_cls->tp_itemsize == 0); \
assert(default_cls->tp_basicsize == size); \
......@@ -650,6 +729,22 @@ extern "C" PyObject* PystonType_GenericAlloc(BoxedClass* cls, Py_ssize_t nitems)
/* TODO: there should be a way to not have to do this nested inlining by hand */ \
}
// This corresponds to CPython's PyVarObject, for objects with a variable number of "items" that are stored inline.
// For example, strings and tuples store their data in line in the main object allocation, so are BoxVars. Lists,
// since they have a changeable size, store their elements in a separate array, and their main object is a fixed
// size and so aren't BoxVar.
class BoxVar : public Box {
public:
// This field gets initialized in operator new.
Py_ssize_t ob_size;
BoxVar() {}
void* operator new(size_t size, BoxedClass* cls, size_t nitems) __attribute__((visibility("default")));
};
static_assert(offsetof(BoxVar, ob_size) == offsetof(struct _varobject, ob_size), "");
// This is the variant of DEFAULT_CLASS that applies to BoxVar objects.
#define DEFAULT_CLASS_VAR(default_cls, itemsize) \
static_assert(itemsize > 0, ""); \
/* asserts that the class in question is a subclass of BoxVar */ \
......@@ -693,21 +788,6 @@ extern "C" PyObject* PystonType_GenericAlloc(BoxedClass* cls, Py_ssize_t nitems)
return rtn; \
}
// CPython C API compatibility class:
class BoxVar : public Box {
public:
// This field gets initialized in operator new.
Py_ssize_t ob_size;
BoxVar() {}
void* operator new(size_t size, BoxedClass* cls, size_t nitems) __attribute__((visibility("default")));
};
static_assert(offsetof(BoxVar, ob_size) == offsetof(struct _varobject, ob_size), "");
std::string getFullTypeName(Box* o);
const char* getTypeName(Box* b);
class BoxedClass;
// TODO these shouldn't be here
......@@ -733,6 +813,7 @@ void raiseSyntaxError(const char* msg, int lineno, int col_offset, llvm::StringR
void raiseSyntaxErrorHelper(llvm::StringRef file, llvm::StringRef func, AST* node_at, const char* msg, ...)
__attribute__((format(printf, 4, 5)));
// A data structure used for storing information for tracebacks.
struct LineInfo {
public:
int line, column;
......@@ -742,6 +823,7 @@ public:
: line(line), column(column), file(file), func(func) {}
};
// A data structure to simplify passing around all the data about a thrown exception.
struct ExcInfo {
Box* type, *value, *traceback;
......@@ -750,7 +832,10 @@ struct ExcInfo {
void printExcAndTraceback() const;
};
// Our object that implements Python's "frame" object:
class BoxedFrame;
// Our internal data structure for storing certain information about a stack frame.
struct FrameInfo {
// Note(kmod): we have a number of fields here that all have independent
// initialization rules. We could potentially save time on every function-entry
......@@ -779,6 +864,7 @@ struct FrameInfo {
void gcVisit(GCVisitor* visitor);
};
// callattr() takes a number of flags and arguments, and for performance we pack them into a single register:
struct CallattrFlags {
bool cls_only : 1;
bool null_on_nonexistent : 1;
......@@ -788,7 +874,7 @@ struct CallattrFlags {
};
static_assert(sizeof(CallattrFlags) == sizeof(uint64_t), "");
// A C++-style way of handling a PyArena*
// A C++-style RAII way of handling a PyArena*
class ArenaWrapper {
private:
PyArena* arena;
......@@ -803,6 +889,7 @@ public:
operator PyArena*() const { return arena; }
};
// A C++-style RAII way of handling a FILE*
class FileHandle {
private:
FILE* file;
......@@ -841,6 +928,7 @@ int binarySearch(T needle, RandomAccessIterator start, RandomAccessIterator end,
}
}
// We need to override these functions so that our GC can know about them.
namespace std {
template <> std::pair<pyston::Box**, std::ptrdiff_t> get_temporary_buffer<pyston::Box*>(std::ptrdiff_t count) noexcept;
template <> void return_temporary_buffer<pyston::Box*>(pyston::Box** p);
......
src/runtime/types.h
View file @ 3a811c26
......@@ -307,6 +307,10 @@ protected:
friend void setupThread();
};
// Corresponds to PyHeapTypeObject. Very similar to BoxedClass, but allocates some extra space for
// structures that otherwise might get allocated statically. For instance, tp_as_number for builtin
// types will usually point to a `static PyNumberMethods` object, but for a heap-allocated class it
// will point to `this->as_number`.
class BoxedHeapClass : public BoxedClass {
public:
PyNumberMethods as_number;
......@@ -336,6 +340,7 @@ private:
friend void setupThread();
};
// Assert that our data structures have the same layout as the C API ones with which they need to be interchangeable.
static_assert(sizeof(pyston::Box) == sizeof(struct _object), "");
static_assert(offsetof(pyston::Box, cls) == offsetof(struct _object, ob_type), "");
......@@ -675,7 +680,7 @@ public:
// CPython declares ob_item (their version of elts) to have 1 element. We want to
// copy that behavior so that the sizes of the objects match, but we want to also
// have a zero-length array in there since we have some extra compiler warnings turned
// on. _elts[1] will throw an error, but elts[1] will not.
// on: _elts[1] will throw an error, but elts[1] will not.
union {
Box* elts[0];
Box* _elts[1];
......@@ -687,6 +692,7 @@ static_assert(offsetof(BoxedTuple, elts) == offsetof(PyTupleObject, ob_item), ""
extern BoxedString* characters[UCHAR_MAX + 1];
// C++ functor objects that implement Python semantics.
struct PyHasher {
size_t operator()(Box* b) const {
if (b->cls == str_cls) {
......@@ -719,8 +725,11 @@ struct PyLt {
// llvm::DenseMap doesn't store the original hash values, choosing to instead
// check for equality more often. This is probably a good tradeoff when the keys
// are pointers and comparison is cheap, but we want to make sure that keys with
// different hash values don't get compared.
// are pointers and comparison is cheap, but when the equality function is user-defined
// it can be much faster to avoid Python function invocations by doing some integer
// comparisons.
// This also has a user-visible behavior difference of how many times the hash function
// and equality functions get called.
struct BoxAndHash {
Box* value;
size_t hash;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7