Basic support for Generators

Currently does not support arguments and only a single generator at a time is allowed

In addition added 'StopIteration' iteration support

src/analysis/function_analysis.cpp

@@ -31,6 +31,25 @@
 
 namespace pyston {
 
+class YieldVisitor : public NoopASTVisitor {
+public:
+    YieldVisitor() : containsYield(false) {}
+
+    virtual bool visit_yield(AST_Yield*) {
+        containsYield = true;
+        return true;
+    }
+
+    bool containsYield;
+};
+
+bool containsYield(AST* ast) {
+    YieldVisitor visitor;
+    ast->accept(&visitor);
+    return visitor.containsYield;
+}
+
+
 class LivenessBBVisitor : public NoopASTVisitor {
 public:
     typedef llvm::SmallSet<std::string, 4> StrSet;

src/analysis/function_analysis.h

@@ -77,6 +77,8 @@ public:
     bool isPotentiallyUndefinedAfter(const std::string& name, CFGBlock* block);
 };
 
+bool containsYield(AST* ast);
+
 LivenessAnalysis* computeLivenessInfo(CFG*);
 PhiAnalysis* computeRequiredPhis(const SourceInfo::ArgNames&, CFG*, LivenessAnalysis*, ScopeInfo* scope_Info);
 }

src/analysis/scoping_analysis.cpp

@@ -200,6 +200,7 @@ public:
     virtual bool visit_unaryop(AST_UnaryOp* node) { return false; }
     virtual bool visit_while(AST_While* node) { return false; }
     virtual bool visit_with(AST_With* node) { return false; }
+    virtual bool visit_yield(AST_Yield* node) { return false; }
 
     virtual bool visit_branch(AST_Branch* node) { return false; }
     virtual bool visit_jump(AST_Jump* node) { return false; }

src/analysis/type_analysis.cpp

@@ -439,6 +439,7 @@ private:
         return attr_type->callType(ArgPassSpec(0), arg_types, NULL);
     }
 
+    virtual void* visit_yield(AST_Yield*) { return UNKNOWN; }
 
 
     virtual void visit_assert(AST_Assert* node) {

src/codegen/codegen.h

@@ -70,7 +70,7 @@ struct GlobalState {
     llvm::Type* llvm_flavor_type, *llvm_flavor_type_ptr;
     llvm::Type* llvm_opaque_type;
     llvm::Type* llvm_str_type_ptr;
-    llvm::Type* llvm_clfunction_type_ptr, *llvm_closure_type_ptr;
+    llvm::Type* llvm_clfunction_type_ptr, *llvm_closure_type_ptr, *llvm_func_type_ptr;
     llvm::Type* llvm_module_type_ptr, *llvm_bool_type_ptr;
     llvm::Type* i1, *i8, *i8_ptr, *i32, *i64, *void_, *double_;
     llvm::Type* vector_ptr;

src/codegen/irgen.cpp

@@ -513,7 +513,7 @@ static void emitBBs(IRGenState* irstate, const char* bb_type, GuardList& out_gua
             assert(strcmp("opt", bb_type) == 0);
 
             if (ENABLE_REOPT && effort < EffortLevel::MAXIMAL && source->ast != NULL
-                && source->ast->type != AST_TYPE::Module) {
+                && source->ast->type != AST_TYPE::Module && !containsYield(source->ast)) {
                 llvm::BasicBlock* preentry_bb
                     = llvm::BasicBlock::Create(g.context, "pre_entry", irstate->getLLVMFunction(),
                                                llvm_entry_blocks[source->cfg->getStartingBlock()]);

src/codegen/irgen/irgenerator.cpp

@@ -27,7 +27,9 @@
 #include "codegen/type_recording.h"
 #include "core/ast.h"
 #include "core/cfg.h"
+#include "core/types.h"
 #include "core/util.h"
+#include "runtime/generator.h"
 #include "runtime/objmodel.h"
 #include "runtime/types.h"
 
@@ -1015,6 +1017,33 @@ private:
         }
     }
 
+    CompilerVariable* evalYield(AST_Yield* node, ExcInfo exc_info) {
+        assert(state != PARTIAL);
+        /*
+                AST_Name genName;
+                genName.id = "!__GENERATOR__";
+                genName.ctx_type = AST_TYPE::Load;
+
+                CompilerVariable* generator = evalName(&genName, exc_info);
+                ConcreteCompilerVariable* convertedGenerator = generator->makeConverted(emitter,
+           generator->getBoxType());
+                generator->decvref(emitter);
+        */
+        CompilerVariable* value = node->value ? evalExpr(node->value, exc_info) : getNone();
+        ConcreteCompilerVariable* convertedValue = value->makeConverted(emitter, value->getBoxType());
+        value->decvref(emitter);
+
+        llvm::Value* rtn = emitter.createCall2(exc_info, g.funcs.yield, convertedValue->getValue(),
+                                               convertedValue->getValue()).getInstruction();
+        // convertedGenerator->decvref(emitter);
+        convertedValue->decvref(emitter);
+
+        return new ConcreteCompilerVariable(UNKNOWN, rtn, true);
+
+        // CompilerVariable* value = evalExpr(node->value, exc_info);
+        // return getNone();
+    }
+
     ConcreteCompilerVariable* unboxVar(ConcreteCompilerType* t, llvm::Value* v, bool grabbed) {
         assert(state != PARTIAL);
 
@@ -1097,6 +1126,10 @@ private:
                 case AST_TYPE::UnaryOp:
                     rtn = evalUnaryOp(ast_cast<AST_UnaryOp>(node), exc_info);
                     break;
+                case AST_TYPE::Yield:
+                    rtn = evalYield(ast_cast<AST_Yield>(node), exc_info);
+                    break;
+
                 case AST_TYPE::ClsAttribute:
                     rtn = evalClsAttribute(ast_cast<AST_ClsAttribute>(node), exc_info);
                     break;
@@ -1555,10 +1588,13 @@ private:
             d->decvref(emitter);
         }
 
-        // llvm::Type* boxCLFuncArgType = g.funcs.boxCLFunction->arg_begin()->getType();
-        // llvm::Value *boxed = emitter.getBuilder()->CreateCall(g.funcs.boxCLFunction, embedConstantPtr(cl,
-        // boxCLFuncArgType));
-        // CompilerVariable *func = new ConcreteCompilerVariable(typeFromClass(function_cls), boxed, true);
+        if (containsYield(node)) {
+            ConcreteCompilerVariable* converted = func->makeConverted(emitter, func->getBoxType());
+            CLFunction* clFunc = boxRTFunction((void*)createGenerator, UNKNOWN, 1, 1, false, false);
+            func = makeFunction(emitter, clFunc, NULL, { converted });
+            converted->decvref(emitter);
+        }
+
         return func;
     }
 
@@ -1883,7 +1919,8 @@ private:
 
         llvm::BasicBlock* target = entry_blocks[node->target];
 
-        if (ENABLE_OSR && node->target->idx < myblock->idx && irstate->getEffortLevel() < EffortLevel::MAXIMAL) {
+        if (ENABLE_OSR && node->target->idx < myblock->idx && irstate->getEffortLevel() < EffortLevel::MAXIMAL
+            && !containsYield(irstate->getSourceInfo()->ast)) {
             assert(node->target->predecessors.size() > 1);
             doOSRExit(target, node);
         } else {

src/codegen/parser.cpp

@@ -677,6 +677,16 @@ AST_With* read_with(BufferedReader* reader) {
     return rtn;
 }
 
+AST_Yield* read_yield(BufferedReader* reader) {
+    AST_Yield* rtn = new AST_Yield();
+
+    rtn->col_offset = readColOffset(reader);
+    rtn->lineno = reader->readULL();
+    rtn->value = readASTExpr(reader);
+
+    return rtn;
+}
+
 AST_expr* readASTExpr(BufferedReader* reader) {
     uint8_t type = reader->readByte();
     if (VERBOSITY("parsing") >= 2)
@@ -728,6 +738,8 @@ AST_expr* readASTExpr(BufferedReader* reader) {
             return read_tuple(reader);
         case AST_TYPE::UnaryOp:
             return read_unaryop(reader);
+        case AST_TYPE::Yield:
+            return read_yield(reader);
         default:
             fprintf(stderr, "Unknown expr node type (parser.cpp:" STRINGIFY(__LINE__) "): %d\n", type);
             abort();

src/codegen/runtime_hooks.cpp

@@ -34,6 +34,7 @@
 #include "core/types.h"
 #include "runtime/float.h"
 #include "runtime/gc_runtime.h"
+#include "runtime/generator.h"
 #include "runtime/inline/boxing.h"
 #include "runtime/int.h"
 #include "runtime/objmodel.h"
@@ -141,6 +142,9 @@ void initGlobalFuncs(GlobalState& g) {
     g.llvm_closure_type_ptr = g.stdlib_module->getTypeByName("class.pyston::BoxedClosure")->getPointerTo();
     assert(g.llvm_closure_type_ptr);
 
+    g.llvm_func_type_ptr = g.stdlib_module->getTypeByName("class.pyston::BoxedFunction")->getPointerTo();
+    assert(g.llvm_func_type_ptr);
+
 #define GET(N) g.funcs.N = getFunc((void*)N, STRINGIFY(N))
 
     g.funcs.printf = addFunc((void*)printf, g.i8_ptr, true);
@@ -166,6 +170,7 @@ void initGlobalFuncs(GlobalState& g) {
     GET(createDict);
     GET(createSlice);
     GET(createClosure);
+    GET(createGenerator);
 
     GET(getattr);
     GET(setattr);
@@ -184,6 +189,7 @@ void initGlobalFuncs(GlobalState& g) {
     GET(import);
     GET(repr);
     GET(isinstance);
+    GET(yield);
 
     GET(checkUnpackingLength);
     GET(raiseAttributeError);

src/codegen/runtime_hooks.h

@@ -32,9 +32,9 @@ struct GlobalFuncs {
 
     llvm::Value* boxInt, *unboxInt, *boxFloat, *unboxFloat, *boxStringPtr, *boxCLFunction, *unboxCLFunction,
         *boxInstanceMethod, *boxBool, *unboxBool, *createTuple, *createDict, *createList, *createSlice,
-        *createUserClass, *createClosure;
+        *createUserClass, *createClosure, *createGenerator;
     llvm::Value* getattr, *setattr, *print, *nonzero, *binop, *compare, *augbinop, *unboxedLen, *getitem, *getclsattr,
-        *getGlobal, *setitem, *delitem, *unaryop, *import, *repr, *isinstance;
+        *yield, *getGlobal, *setitem, *delitem, *unaryop, *import, *repr, *isinstance;
 
     llvm::Value* checkUnpackingLength, *raiseAttributeError, *raiseAttributeErrorStr, *raiseNotIterableError,
         *assertNameDefined, *assertFail;

src/core/ast.cpp

@@ -836,6 +836,18 @@ void AST_With::accept_stmt(StmtVisitor* v) {
     v->visit_with(this);
 }
 
+void AST_Yield::accept(ASTVisitor* v) {
+    bool skip = v->visit_yield(this);
+    if (skip)
+        return;
+
+    if (value)
+        value->accept(v);
+}
+
+void* AST_Yield::accept_expr(ExprVisitor* v) {
+    return v->visit_yield(this);
+}
 
 void AST_Branch::accept(ASTVisitor* v) {
     bool skip = v->visit_branch(this);
@@ -1602,6 +1614,13 @@ bool PrintVisitor::visit_with(AST_With* node) {
     return true;
 }
 
+bool PrintVisitor::visit_yield(AST_Yield* node) {
+    printf("yield ");
+    if (node->value)
+        node->value->accept(this);
+    return true;
+}
+
 bool PrintVisitor::visit_branch(AST_Branch* node) {
     printf("if ");
     node->test->accept(this);
@@ -1835,6 +1854,10 @@ public:
         output->push_back(node);
         return false;
     }
+    virtual bool visit_yield(AST_Yield* node) {
+        output->push_back(node);
+        return false;
+    }
 
     virtual bool visit_branch(AST_Branch* node) {
         output->push_back(node);

src/core/ast.h

@@ -797,6 +797,18 @@ public:
     static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::With;
 };
 
+class AST_Yield : public AST_expr {
+public:
+    AST_expr* value;
+
+    virtual void accept(ASTVisitor* v);
+    virtual void* accept_expr(ExprVisitor* v);
+
+    AST_Yield() : AST_expr(AST_TYPE::Yield) {}
+
+    static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::Yield;
+};
+
 
 // AST pseudo-nodes that will get added during CFG-construction.  These don't exist in the input AST, but adding them in
 // lets us avoid creating a completely new IR for this phase
@@ -953,6 +965,7 @@ public:
     virtual bool visit_unreachable(AST_Unreachable* node) { RELEASE_ASSERT(0, ""); }
     virtual bool visit_while(AST_While* node) { RELEASE_ASSERT(0, ""); }
     virtual bool visit_with(AST_With* node) { RELEASE_ASSERT(0, ""); }
+    virtual bool visit_yield(AST_Yield* node) { RELEASE_ASSERT(0, ""); }
 
     virtual bool visit_branch(AST_Branch* node) { RELEASE_ASSERT(0, ""); }
     virtual bool visit_jump(AST_Jump* node) { RELEASE_ASSERT(0, ""); }
@@ -1016,6 +1029,7 @@ public:
     virtual bool visit_unreachable(AST_Unreachable* node) { return false; }
     virtual bool visit_while(AST_While* node) { return false; }
     virtual bool visit_with(AST_With* node) { return false; }
+    virtual bool visit_yield(AST_Yield* node) { return false; }
 
     virtual bool visit_branch(AST_Branch* node) { return false; }
     virtual bool visit_jump(AST_Jump* node) { return false; }
@@ -1049,6 +1063,7 @@ public:
     virtual void* visit_subscript(AST_Subscript* node) { RELEASE_ASSERT(0, ""); }
     virtual void* visit_tuple(AST_Tuple* node) { RELEASE_ASSERT(0, ""); }
     virtual void* visit_unaryop(AST_UnaryOp* node) { RELEASE_ASSERT(0, ""); }
+    virtual void* visit_yield(AST_Yield* node) { RELEASE_ASSERT(0, ""); }
 };
 
 class StmtVisitor {
@@ -1148,6 +1163,7 @@ public:
     virtual bool visit_unreachable(AST_Unreachable* node);
     virtual bool visit_while(AST_While* node);
     virtual bool visit_with(AST_With* node);
+    virtual bool visit_yield(AST_Yield* node);
 
     virtual bool visit_branch(AST_Branch* node);
     virtual bool visit_jump(AST_Jump* node);

src/core/cfg.cpp

@@ -717,6 +717,14 @@ private:
         return rtn;
     }
 
+    AST_expr* remapYield(AST_Yield* node) {
+        AST_Yield* rtn = new AST_Yield();
+        rtn->lineno = node->lineno;
+        rtn->col_offset = node->col_offset;
+        rtn->value = remapExpr(node->value);
+        return rtn;
+    }
+
     AST_expr* remapExpr(AST_expr* node, bool wrap_with_assign = true) {
         if (node == NULL)
             return NULL;
@@ -787,6 +795,9 @@ private:
             case AST_TYPE::UnaryOp:
                 rtn = remapUnaryOp(ast_cast<AST_UnaryOp>(node));
                 break;
+            case AST_TYPE::Yield:
+                rtn = remapYield(ast_cast<AST_Yield>(node));
+                break;
             default:
                 RELEASE_ASSERT(0, "%d", node->type);
         }

src/runtime/generator.cpp

+// Copyright (c) 2014 Dropbox, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "runtime/generator.h"
+
+#include <algorithm>
+#include <cstddef>
+#include <cstring>
+#include <ucontext.h>
+
+#include "codegen/compvars.h"
+#include "core/ast.h"
+#include "core/common.h"
+#include "core/stats.h"
+#include "core/types.h"
+#include "gc/collector.h"
+#include "runtime/gc_runtime.h"
+#include "runtime/objmodel.h"
+#include "runtime/types.h"
+#include "runtime/util.h"
+
+namespace pyston {
+
+BoxedGenerator* g_gen; // HACK REMOVE ME!!!
+
+static void generatorEntry(BoxedGenerator* self) {
+    assert(self->cls == generator_cls);
+    assert(self->function->cls == function_cls);
+
+    try {
+        // call body of the generator
+        runtimeCall(self->function, ArgPassSpec(0), 0, 0, 0, 0, 0);
+    } catch (Box* e) {
+        // unhandled exception: propagate the exception to the caller
+        self->exception = e;
+    }
+
+    // we returned from the body of the generator. next/send/throw will notify the caller
+    self->entryExited = true;
+    swapcontext(&self->context, &self->returnContext);
+}
+
+Box* generatorIter(Box* s) {
+    return s;
+}
+
+Box* generatorSend(Box* s, Box* v) {
+    assert(s->cls == generator_cls);
+    BoxedGenerator* self = static_cast<BoxedGenerator*>(s);
+
+    // check if the generator already exited
+    if (self->entryExited)
+        raiseExcHelper(StopIteration, "");
+
+    self->returnValue = v;
+    swapcontext(&self->returnContext, &self->context);
+
+    // propagate exception to the caller
+    if (self->exception)
+        raiseExc(self->exception);
+
+    // throw StopIteration if the generator exited
+    if (self->entryExited)
+        raiseExcHelper(StopIteration, "");
+
+    return self->returnValue;
+}
+
+Box* generatorThrow(Box* s, BoxedClass* e) {
+    assert(s->cls == generator_cls);
+    assert(isSubclass(e, Exception));
+    BoxedGenerator* self = static_cast<BoxedGenerator*>(s);
+    self->exception = exceptionNew1(e);
+    return generatorSend(self, None);
+}
+
+Box* generatorNext(Box* s) {
+    return generatorSend(s, None);
+}
+
+extern "C" Box* yield(Box* obj, Box* value) {
+    obj = g_gen;
+    assert(obj->cls == generator_cls);
+    BoxedGenerator* self = static_cast<BoxedGenerator*>(obj);
+    self->returnValue = value;
+
+    swapcontext(&self->context, &self->returnContext);
+
+    // if the generator receives a exception from the caller we have throw it
+    if (self->exception) {
+        Box* exception = self->exception;
+        self->exception = nullptr;
+        raiseExc(exception);
+    }
+    return self->returnValue;
+}
+
+extern "C" BoxedGenerator* createGenerator(BoxedFunction* function) {
+    assert(function);
+    assert(function->cls == function_cls);
+    return new BoxedGenerator(function);
+}
+
+extern "C" BoxedGenerator::BoxedGenerator(BoxedFunction* function)
+    : Box(&generator_flavor, generator_cls), function(function), entryExited(false), returnValue(nullptr),
+      exception(nullptr) {
+
+    giveAttr("__name__", boxString(function->f->source->getName()));
+
+    getcontext(&context);
+    context.uc_link = 0;
+    context.uc_stack.ss_sp = stack;
+    context.uc_stack.ss_size = STACK_SIZE;
+    makecontext(&context, (void (*)(void))generatorEntry, 1, this);
+
+    g_gen = this;
+}
+
+
+void setupGenerator() {
+    generator_cls = new BoxedClass(object_cls, offsetof(BoxedGenerator, attrs), sizeof(BoxedGenerator), false);
+    generator_cls->giveAttr("__name__", boxStrConstant("generator"));
+    generator_cls->giveAttr("__iter__",
+                            new BoxedFunction(boxRTFunction((void*)generatorIter, typeFromClass(generator_cls), 1)));
+
+    generator_cls->giveAttr("next", new BoxedFunction(boxRTFunction((void*)generatorNext, UNKNOWN, 1)));
+    generator_cls->giveAttr("send", new BoxedFunction(boxRTFunction((void*)generatorSend, UNKNOWN, 2)));
+    generator_cls->giveAttr("throw", new BoxedFunction(boxRTFunction((void*)generatorThrow, UNKNOWN, 2)));
+
+    gc::registerStaticRootObj(generator_cls);
+    generator_cls->freeze();
+}
+}

src/runtime/generator.h

+// Copyright (c) 2014 Dropbox, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef PYSTON_RUNTIME_GENERATOR_H
+#define PYSTON_RUNTIME_GENERATOR_H
+
+#include "core/types.h"
+#include "runtime/types.h"
+
+namespace pyston {
+
+extern BoxedClass* generator_cls;
+extern "C" const ObjectFlavor generator_flavor;
+
+void setupGenerator();
+
+extern "C" Box* yield(Box* obj, Box* value);
+extern "C" BoxedGenerator* createGenerator(BoxedFunction* function);
+}
+
+#endif

src/runtime/inline/link_forcer.cpp

@@ -19,6 +19,7 @@
 #include "gc/heap.h"
 #include "runtime/float.h"
 #include "runtime/gc_runtime.h"
+#include "runtime/generator.h"
 #include "runtime/inline/boxing.h"
 #include "runtime/int.h"
 #include "runtime/list.h"
@@ -58,6 +59,7 @@ void force() {
     FORCE(createSlice);
     FORCE(createUserClass);
     FORCE(createClosure);
+    FORCE(createGenerator);
 
     FORCE(getattr);
     FORCE(setattr);
@@ -76,6 +78,7 @@ void force() {
     FORCE(import);
     FORCE(repr);
     FORCE(isinstance);
+    FORCE(yield);
 
     FORCE(checkUnpackingLength);
     FORCE(raiseAttributeError);

src/runtime/objmodel.h

@@ -27,6 +27,7 @@ class BoxedClass;
 class BoxedInt;
 class BoxedList;
 class BoxedString;
+class BoxedGenerator;
 
 void raiseExc(Box*) __attribute__((__noreturn__));
 void raiseExcHelper(BoxedClass*, const char* fmt, ...) __attribute__((__noreturn__));

src/runtime/types.cpp

@@ -44,11 +44,23 @@ BoxIterator& BoxIterator::operator++() {
     static std::string next_str("next");
 
     Box* hasnext = callattrInternal(iter, &hasnext_str, CLASS_ONLY, NULL, ArgPassSpec(0), NULL, NULL, NULL, NULL, NULL);
-    if (nonzero(hasnext)) {
-        value = callattrInternal(iter, &next_str, CLASS_ONLY, NULL, ArgPassSpec(0), NULL, NULL, NULL, NULL, NULL);
+    if (hasnext) {
+        if (nonzero(hasnext)) {
+            value = callattrInternal(iter, &next_str, CLASS_ONLY, NULL, ArgPassSpec(0), NULL, NULL, NULL, NULL, NULL);
+        } else {
+            iter = nullptr;
+            value = nullptr;
+        }
     } else {
-        iter = nullptr;
-        value = nullptr;
+        try {
+            value = callattrInternal(iter, &next_str, CLASS_ONLY, NULL, ArgPassSpec(0), NULL, NULL, NULL, NULL, NULL);
+        } catch (Box* e) {
+            if ((e == StopIteration) || isSubclass(e->cls, StopIteration)) {
+                iter = nullptr;
+                value = nullptr;
+            } else
+                throw;
+        }
     }
     return *this;
 }
@@ -261,10 +273,26 @@ extern "C" void closureGCHandler(GCVisitor* v, void* p) {
         v->visit(c->parent);
 }
 
+extern "C" void generatorGCHandler(GCVisitor* v, void* p) {
+    boxGCHandler(v, p);
+
+    BoxedGenerator* g = (BoxedGenerator*)p;
+    if (g->function)
+        v->visit(g->function);
+    if (g->returnValue)
+        v->visit(g->returnValue);
+    if (g->exception)
+        v->visit(g->exception);
+    v->visitPotentialRange((void**)&g->context, ((void**)&g->context) + sizeof(g->context) / sizeof(void*));
+    v->visitPotentialRange((void**)&g->returnContext,
+                           ((void**)&g->returnContext) + sizeof(g->returnContext) / sizeof(void*));
+    v->visitPotentialRange((void**)&g->stack[0], (void**)&g->stack[BoxedGenerator::STACK_SIZE]);
+}
+
 extern "C" {
 BoxedClass* object_cls, *type_cls, *none_cls, *bool_cls, *int_cls, *float_cls, *str_cls, *function_cls,
     *instancemethod_cls, *list_cls, *slice_cls, *module_cls, *dict_cls, *tuple_cls, *file_cls, *member_cls,
-    *closure_cls;
+    *closure_cls, *generator_cls;
 
 const ObjectFlavor object_flavor(&boxGCHandler, NULL);
 const ObjectFlavor type_flavor(&typeGCHandler, NULL);
@@ -283,6 +311,7 @@ const ObjectFlavor tuple_flavor(&tupleGCHandler, NULL);
 const ObjectFlavor file_flavor(&boxGCHandler, NULL);
 const ObjectFlavor member_flavor(&boxGCHandler, NULL);
 const ObjectFlavor closure_flavor(&closureGCHandler, NULL);
+const ObjectFlavor generator_flavor(&generatorGCHandler, NULL);
 
 const AllocationKind untracked_kind(NULL, NULL);
 const AllocationKind hc_kind(&hcGCHandler, NULL);
@@ -573,6 +602,7 @@ void setupRuntime() {
     setupSet();
     setupTuple();
     setupFile();
+    setupGenerator();
 
     function_cls->giveAttr("__name__", boxStrConstant("function"));
     function_cls->giveAttr("__repr__", new BoxedFunction(boxRTFunction((void*)functionRepr, STR, 1)));

src/runtime/types.h

@@ -15,6 +15,8 @@
 #ifndef PYSTON_RUNTIME_TYPES_H
 #define PYSTON_RUNTIME_TYPES_H
 
+#include <ucontext.h>
+
 #include "core/threading.h"
 #include "core/types.h"
 
@@ -28,6 +30,7 @@ class BoxedDict;
 class BoxedTuple;
 class BoxedFile;
 class BoxedClosure;
+class BoxedGenerator;
 
 void setupInt();
 void teardownInt();
@@ -51,6 +54,7 @@ void setupFile();
 void teardownFile();
 void setupCAPI();
 void teardownCAPI();
+void setupGenerator();
 
 void setupSys();
 void setupBuiltins();
@@ -64,12 +68,12 @@ BoxedList* getSysPath();
 extern "C" {
 extern BoxedClass* object_cls, *type_cls, *bool_cls, *int_cls, *float_cls, *str_cls, *function_cls, *none_cls,
     *instancemethod_cls, *list_cls, *slice_cls, *module_cls, *dict_cls, *tuple_cls, *file_cls, *xrange_cls, *member_cls,
-    *closure_cls;
+    *closure_cls, *generator_cls;
 }
 extern "C" {
 extern const ObjectFlavor object_flavor, type_flavor, bool_flavor, int_flavor, float_flavor, str_flavor,
     function_flavor, none_flavor, instancemethod_flavor, list_flavor, slice_flavor, module_flavor, dict_flavor,
-    tuple_flavor, file_flavor, xrange_flavor, member_flavor, closure_flavor;
+    tuple_flavor, file_flavor, xrange_flavor, member_flavor, closure_flavor, generator_flavor;
 }
 extern "C" { extern Box* None, *NotImplemented, *True, *False; }
 extern "C" {
@@ -319,6 +323,23 @@ public:
     BoxedClosure(BoxedClosure* parent) : Box(&closure_flavor, closure_cls), parent(parent) {}
 };
 
+class BoxedGenerator : public Box {
+public:
+    enum { STACK_SIZE = SIGSTKSZ * 5 };
+
+    HCAttrs attrs;
+    BoxedFunction* function;
+
+    bool entryExited;
+    Box* returnValue;
+    Box* exception;
+
+    ucontext_t context, returnContext;
+    char stack[STACK_SIZE];
+
+    BoxedGenerator(BoxedFunction* function);
+};
+
 extern "C" void boxGCHandler(GCVisitor* v, void* p);
 
 Box* exceptionNew1(BoxedClass* cls);

test/tests/generators.py

+def G1():
+    i = 0
+    while True:
+        yield i
+        i += i
+
+g1 = G1();
+for i in range(5):
+    print g1.next()
+
+
+
+def G2():
+    yield 1
+    yield 2
+    yield 3
+g2 = G2()
+print list(g2)
+print list(g2)
+print list(G2())
+
+
+
+def G3():
+    i = 0
+    while True:
+        got = (yield i**2)
+        print "i=", i, "got=", got
+        i += 1
+
+g3 = G3();
+g3.send(None)
+for i in range(5):
+    r = g3.send(i)
+    print "received= ", r
+
+
+
+def G4():
+    1/0
+    while True:
+        print "unreachable"
+
+try:
+    print list(G4())
+except ZeroDivisionError:
+    print "catched a ZeroDivisionError"
+    
+
+def G5():
+    i = 0
+    try:
+        while True:
+            yield i
+            i += 1
+    except:
+        print "catched a ZeroDivisionError inside G5"
+    yield 42
+
+g5 = G5()
+for i in range(5):
+    print g5.next()
+print g5.throw(ZeroDivisionError)

test/tests/stop_iterable.py

+class C(object):
+  i = 0
+  def next(self):
+    if self.i<5:
+        self.i += 1
+        return self.i
+    else:
+        raise StopIteration
+  def __iter__(self):
+    return self
+
+print list(C())
+
+