Merge branch 'future' of https://github.com/tjhance/pyston

Conflicts:
	src/core/ast.cpp
	src/core/ast.h

src/codegen/compvars.cpp

@@ -1037,9 +1037,12 @@ public:
         if (op_type == AST_TYPE::Mod) {
             v = emitter.createCall2(info.exc_info, g.funcs.mod_float_float, var->getValue(),
                                     converted_right->getValue()).getInstruction();
-        } else if (op_type == AST_TYPE::Div || op_type == AST_TYPE::FloorDiv) {
+        } else if (op_type == AST_TYPE::Div || op_type == AST_TYPE::TrueDiv) {
             v = emitter.createCall2(info.exc_info, g.funcs.div_float_float, var->getValue(),
                                     converted_right->getValue()).getInstruction();
+        } else if (op_type == AST_TYPE::FloorDiv) {
+            v = emitter.createCall2(info.exc_info, g.funcs.floordiv_float_float, var->getValue(),
+                                    converted_right->getValue()).getInstruction();
         } else if (op_type == AST_TYPE::Pow) {
             v = emitter.createCall2(info.exc_info, g.funcs.pow_float_float, var->getValue(),
                                     converted_right->getValue()).getInstruction();

src/codegen/irgen/future.cpp

+// Copyright (c) 2014 Dropbox, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "future.h"
+
+namespace pyston {
+
+struct FutureOption {
+    int optional_version_hex;
+    int mandatory_version_hex;
+    int ff_mask;
+};
+
+const std::map<std::string, FutureOption> future_options
+    = { { "absolute_import", { version_hex(2, 5, 0), version_hex(3, 0, 0), FF_ABSOLUTE_IMPORT } },
+        { "division", { version_hex(2, 2, 0), version_hex(3, 0, 0), FF_DIVISION } },
+        { "generators", { version_hex(2, 2, 0), version_hex(3, 0, 0), FF_GENERATOR } },
+        { "unicode_literals", { version_hex(2, 6, 0), version_hex(3, 0, 0), FF_UNICODE_LITERALS } },
+        { "print_functions", { version_hex(2, 6, 0), version_hex(3, 0, 0), FF_PRINT_FUNCTIONS } },
+        { "nested_scopes", { version_hex(2, 1, 0), version_hex(2, 2, 0), FF_NESTED_SCOPES } },
+        { "with_statement", { version_hex(2, 5, 0), version_hex(3, 6, 0), FF_WITH_STATEMENT } } };
+
+// Helper function:
+void raiseSyntaxError(const char* file, AST* node_at, const char* msg, ...) {
+    va_list ap;
+    va_start(ap, msg);
+
+    char buf[1024];
+    vsnprintf(buf, sizeof(buf), msg, ap);
+
+
+    // TODO I'm not sure that it's safe to raise an exception here, since I think
+    // there will be things that end up not getting cleaned up.
+    // Then again, there are a huge number of things that don't get cleaned up even
+    // if an exception doesn't get thrown...
+
+    // TODO output is still a little wrong, should be, for example
+    //
+    //  File "../test/tests/future_non_existent.py", line 1
+    //    from __future__ import rvalue_references # should cause syntax error
+    //
+    // but instead it is
+    //
+    // Traceback (most recent call last):
+    //  File "../test/tests/future_non_existent.py", line -1, in :
+    //    from __future__ import rvalue_references # should cause syntax error
+    ::pyston::raiseSyntaxError(buf, node_at->lineno, node_at->col_offset, file, "");
+}
+
+void raiseFutureImportErrorNotFound(const char* file, AST* node, const char* name) {
+    raiseSyntaxError(file, node, "future feature %s is not defined", name);
+}
+
+void raiseFutureImportErrorNotBeginning(const char* file, AST* node) {
+    raiseSyntaxError(file, node, "from __future__ imports must occur at the beginning of the file");
+}
+
+class BadFutureImportVisitor : public NoopASTVisitor {
+public:
+    virtual bool visit_importfrom(AST_ImportFrom* node) {
+        if (node->module == "__future__") {
+            raiseFutureImportErrorNotBeginning(file, node);
+        }
+        return true;
+    }
+
+    // TODO optimization: have it skip things like expressions that you know
+    // there is no need to descend into
+
+    BadFutureImportVisitor(const char* file) : file(file) {}
+    const char* file;
+};
+
+inline bool is_stmt_string(AST_stmt* stmt) {
+    return stmt->type == AST_TYPE::Expr && static_cast<AST_Expr*>(stmt)->value->type == AST_TYPE::Str;
+}
+
+FutureFlags getFutureFlags(AST_Module* m, const char* file) {
+    FutureFlags ff = 0;
+
+    // Set the defaults for the future flags depending on what version we are
+    for (const std::pair<std::string, FutureOption>& p : future_options) {
+        if (PYTHON_VERSION_HEX >= p.second.mandatory_version_hex) {
+            ff |= p.second.ff_mask;
+        }
+    }
+
+    // Find all the __future__ imports, raising an error for those that do not
+    // occur at the beginning of the file.
+    bool future_import_allowed = true;
+    BadFutureImportVisitor import_visitor(file);
+    for (int i = 0; i < m->body.size(); i++) {
+        AST_stmt* stmt = m->body[i];
+
+        if (stmt->type == AST_TYPE::ImportFrom && static_cast<AST_ImportFrom*>(stmt)->module == "__future__") {
+            if (future_import_allowed) {
+                // We have a `from __future__` import statement, and we are
+                // still at the top of the file, so just set the appropriate
+                // future flag for each imported option.
+
+                for (AST_alias* alias : static_cast<AST_ImportFrom*>(stmt)->names) {
+                    const std::string& option_name = alias->name;
+                    auto iter = future_options.find(option_name);
+                    if (iter == future_options.end()) {
+                        // If it's not one of the available options, throw an error.
+                        // Note: the __future__ module also exposes "all_feature_names",
+                        // but you still can't import that using a from-import, so we
+                        // don't need to worry about that here.
+                        raiseFutureImportErrorNotFound(file, alias, option_name.c_str());
+                    } else {
+                        const FutureOption& fo = iter->second;
+                        if (PYTHON_VERSION_HEX >= fo.optional_version_hex) {
+                            ff |= fo.ff_mask;
+                        } else {
+                            raiseFutureImportErrorNotFound(file, alias, option_name.c_str());
+                        }
+                    }
+                }
+            } else {
+                raiseFutureImportErrorNotBeginning(file, stmt);
+            }
+        } else {
+            // A docstring is allowed at the beginning of a module; otherwise,
+            // we cannot permit any __future__ import after this point.
+            if (i > 0 || !is_stmt_string(stmt)) {
+                // Recurse on the node and throw an error if it has any
+                // `from __future__` import statement.
+                stmt->accept(&import_visitor);
+
+                future_import_allowed = false;
+            }
+        }
+    }
+
+    return ff;
+}
+}

src/codegen/irgen/future.h

+// Copyright (c) 2014 Dropbox, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef PYSTON_CODEGEN_IRGEN_FUTURE_H
+#define PYSTON_CODEGEN_IRGEN_FUTURE_H
+
+#include <map>
+
+#include "core/ast.h"
+#include "core/options.h"
+#include "core/types.h"
+
+namespace pyston {
+
+#define FF_ABSOLUTE_IMPORT 0x01
+#define FF_DIVISION 0x02
+#define FF_GENERATOR 0x04
+#define FF_UNICODE_LITERALS 0x08
+#define FF_PRINT_FUNCTIONS 0x10
+#define FF_NESTED_SCOPES 0x20
+#define FF_WITH_STATEMENT 0x40
+
+typedef int FutureFlags;
+
+// Loop through import statements to find __future__ imports throwing errors for
+// bad __future__ imports. Returns the futures that are turned on. This is used
+// for irgeneration; the parser still has to handle some futures on its own,
+// when they are relevant for the parser.
+FutureFlags getFutureFlags(AST_Module* m, const char* file);
+}
+
+#endif

src/codegen/irgen/hooks.cpp

@@ -23,6 +23,7 @@
 #include "codegen/codegen.h"
 #include "codegen/compvars.h"
 #include "codegen/irgen.h"
+#include "codegen/irgen/future.h"
 #include "codegen/irgen/util.h"
 #include "codegen/llvm_interpreter.h"
 #include "codegen/osrentry.h"
@@ -236,6 +237,8 @@ void compileAndRunModule(AST_Module* m, BoxedModule* bm) {
 
         Timer _t("for compileModule()");
 
+        bm->future_flags = getFutureFlags(m, bm->fn.c_str());
+
         ScopingAnalysis* scoping = runScopingAnalysis(m);
 
         SourceInfo* si = new SourceInfo(bm, scoping, m, m->body);

src/codegen/irgen/irgenerator.cpp

@@ -250,12 +250,6 @@ private:
 
     OpInfo getEmptyOpInfo(ExcInfo exc_info) { return OpInfo(irstate->getEffortLevel(), NULL, exc_info); }
 
-    // Helper function:
-    void raiseSyntaxError(const char* msg, AST* node_at) {
-        ::pyston::raiseSyntaxError(msg, node_at->lineno, node_at->col_offset,
-                                   irstate->getSourceInfo()->parent_module->fn, irstate->getLLVMFunction()->getName());
-    }
-
     void createExprTypeGuard(llvm::Value* check_val, AST_expr* node, CompilerVariable* node_value) {
         assert(check_val->getType() == g.i1);
 
@@ -429,6 +423,10 @@ private:
         assert(left);
         assert(right);
 
+        if (type == AST_TYPE::Div && (irstate->getSourceInfo()->parent_module->future_flags & FF_DIVISION)) {
+            type = AST_TYPE::TrueDiv;
+        }
+
         return left->binexp(emitter, getOpInfoForNode(node, exc_info), right, type, exp_type);
     }
 
@@ -1597,14 +1595,6 @@ private:
         if (state == PARTIAL)
             return;
 
-        if (node->module == "__future__") {
-            // TODO I'm not sure that it's safe to raise an exception here, since I think
-            // there will be things that end up not getting cleaned up.
-            // Then again, there are a huge number of things that don't get cleaned up even
-            // if an exception doesn't get thrown...
-            raiseSyntaxError("the __future__ module is not supported yet", node);
-        }
-
         assert(node->level == 0);
 
         llvm::Value* imported_v

src/codegen/runtime_hooks.cpp

@@ -236,6 +236,7 @@ void initGlobalFuncs(GlobalState& g) {
     GET(raise3);
 
     GET(div_float_float);
+    GET(floordiv_float_float);
     GET(mod_float_float);
     GET(pow_float_float);
 }

src/codegen/runtime_hooks.h

@@ -49,7 +49,7 @@ struct GlobalFuncs {
     llvm::Value* __cxa_begin_catch, *__cxa_end_catch;
     llvm::Value* raise0, *raise3;
 
-    llvm::Value* div_float_float, *mod_float_float, *pow_float_float;
+    llvm::Value* div_float_float, *floordiv_float_float, *mod_float_float, *pow_float_float;
 };
 }
 

src/core/ast.cpp

@@ -22,8 +22,6 @@
 
 #include "core/cfg.h"
 
-#define FUTURE_DIVISION 0
-
 namespace pyston {
 
 llvm::StringRef getOpSymbol(int op_type) {
@@ -37,6 +35,7 @@ llvm::StringRef getOpSymbol(int op_type) {
         case AST_TYPE::BitXor:
             return "^";
         case AST_TYPE::Div:
+        case AST_TYPE::TrueDiv:
             return "/";
         case AST_TYPE::DivMod:
             return "divmod()";
@@ -113,10 +112,9 @@ const std::string& getOpName(int op_type) {
         case AST_TYPE::BitXor:
             return strBitXor;
         case AST_TYPE::Div:
-            if (FUTURE_DIVISION)
-                return strTrueDiv;
-            else
-                return strDiv;
+            return strDiv;
+        case AST_TYPE::TrueDiv:
+            return strTrueDiv;
         case AST_TYPE::DivMod:
             return strDivMod;
         case AST_TYPE::Eq:

src/core/ast.h

@@ -126,9 +126,10 @@ enum AST_TYPE {
     LangPrimitive = 205,
     Unreachable = 206,
 
-    // This isn't a real AST type, but since we use AST types to represent binexp types
-    // and divmod is essentially a type of binop, we add it here (at least for now):
+    // These aren't real AST types, but since we use AST types to represent binexp types
+    // and divmod+truediv are essentially types of binops, we add them here (at least for now):
     DivMod = 250,
+    TrueDiv = 251,
 };
 };
 

src/runtime/float.cpp

@@ -52,6 +52,11 @@ extern "C" double div_float_float(double lhs, double rhs) {
     return lhs / rhs;
 }
 
+extern "C" double floordiv_float_float(double lhs, double rhs) {
+    raiseDivZeroExcIfZero(rhs);
+    return floor(lhs / rhs);
+}
+
 extern "C" Box* floatAddFloat(BoxedFloat* lhs, BoxedFloat* rhs) {
     assert(lhs->cls == float_cls);
     assert(rhs->cls == float_cls);
@@ -100,6 +105,17 @@ extern "C" Box* floatDiv(BoxedFloat* lhs, Box* rhs) {
     }
 }
 
+extern "C" Box* floatTruediv(BoxedFloat* lhs, Box* rhs) {
+    assert(lhs->cls == float_cls);
+    if (rhs->cls == int_cls) {
+        return floatDivInt(lhs, static_cast<BoxedInt*>(rhs));
+    } else if (rhs->cls == float_cls) {
+        return floatDivFloat(lhs, static_cast<BoxedFloat*>(rhs));
+    } else {
+        return NotImplemented;
+    }
+}
+
 extern "C" Box* floatRDivFloat(BoxedFloat* lhs, BoxedFloat* rhs) {
     assert(lhs->cls == float_cls);
     assert(rhs->cls == float_cls);
@@ -563,6 +579,7 @@ void setupFloat() {
     _addFunc("__div__", BOXED_FLOAT, (void*)floatDivFloat, (void*)floatDivInt, (void*)floatDiv);
     _addFunc("__rdiv__", BOXED_FLOAT, (void*)floatRDivFloat, (void*)floatRDivInt, (void*)floatRDiv);
     float_cls->giveAttr("__floordiv__", new BoxedFunction(boxRTFunction((void*)floatFloorDiv, UNKNOWN, 2)));
+    _addFunc("__truediv__", BOXED_FLOAT, (void*)floatDivFloat, (void*)floatDivInt, (void*)floatTruediv);
 
     _addFunc("__eq__", BOXED_BOOL, (void*)floatEqFloat, (void*)floatEqInt, (void*)floatEq);
     _addFunc("__ge__", BOXED_BOOL, (void*)floatGeFloat, (void*)floatGeInt, (void*)floatGe);

src/runtime/float.h

@@ -19,6 +19,7 @@ namespace pyston {
 
 extern "C" double mod_float_float(double lhs, double rhs);
 extern "C" double div_float_float(double lhs, double rhs);
+extern "C" double floordiv_float_float(double lhs, double rhs);
 extern "C" double pow_float_float(double lhs, double rhs);
 
 class BoxedFloat;

src/runtime/inline/link_forcer.cpp

@@ -111,6 +111,7 @@ void force() {
     FORCE(pow_i64_i64);
 
     FORCE(div_float_float);
+    FORCE(floordiv_float_float);
     FORCE(mod_float_float);
     FORCE(pow_float_float);
 

src/runtime/int.cpp

@@ -317,6 +317,37 @@ extern "C" Box* intFloordiv(BoxedInt* lhs, Box* rhs) {
     }
 }
 
+extern "C" Box* intTruedivInt(BoxedInt* lhs, BoxedInt* rhs) {
+    assert(lhs->cls == int_cls);
+    assert(rhs->cls == int_cls);
+
+    if (rhs->n == 0) {
+        raiseExcHelper(ZeroDivisionError, "division by zero");
+    }
+    return boxFloat(lhs->n / (float)rhs->n);
+}
+
+extern "C" Box* intTruedivFloat(BoxedInt* lhs, BoxedFloat* rhs) {
+    assert(lhs->cls == int_cls);
+    assert(rhs->cls == float_cls);
+
+    if (rhs->d == 0) {
+        raiseExcHelper(ZeroDivisionError, "division by zero");
+    }
+    return boxFloat(lhs->n / rhs->d);
+}
+
+extern "C" Box* intTruediv(BoxedInt* lhs, Box* rhs) {
+    assert(lhs->cls == int_cls);
+    if (rhs->cls == int_cls) {
+        return intTruedivInt(lhs, static_cast<BoxedInt*>(rhs));
+    } else if (rhs->cls == float_cls) {
+        return intTruedivFloat(lhs, static_cast<BoxedFloat*>(rhs));
+    } else {
+        return NotImplemented;
+    }
+}
+
 extern "C" Box* intEqInt(BoxedInt* lhs, BoxedInt* rhs) {
     assert(lhs->cls == int_cls);
     assert(rhs->cls == int_cls);
@@ -685,6 +716,7 @@ void setupInt() {
     _addFuncIntFloatUnknown("__sub__", (void*)intSubInt, (void*)intSubFloat, (void*)intSub);
     _addFuncIntFloatUnknown("__div__", (void*)intDivInt, (void*)intDivFloat, (void*)intDiv);
     _addFuncIntFloatUnknown("__floordiv__", (void*)intFloordivInt, (void*)intFloordivFloat, (void*)intFloordiv);
+    _addFuncIntFloatUnknown("__truediv__", (void*)intTruedivInt, (void*)intDivFloat, (void*)intTruediv);
     _addFuncIntFloatUnknown("__mul__", (void*)intMulInt, (void*)intMulFloat, (void*)intMul);
     _addFuncIntUnknown("__mod__", BOXED_INT, (void*)intModInt, (void*)intMod);
     _addFuncIntFloatUnknown("__pow__", (void*)intPowInt, (void*)intPowFloat, (void*)intPow);

src/runtime/types.h

@@ -20,6 +20,7 @@
 #include "Python.h"
 #include "structmember.h"
 
+#include "codegen/irgen/future.h"
 #include "core/threading.h"
 #include "core/types.h"
 #include "gc/gc_alloc.h"
@@ -307,6 +308,7 @@ class BoxedModule : public Box {
 public:
     HCAttrs attrs;
     std::string fn; // for traceback purposes; not the same as __file__
+    FutureFlags future_flags;
 
     BoxedModule(const std::string& name, const std::string& fn);
     std::string name();

test/tests/future_division.py

+# allow-warning
+
+# The __future__ module has an old-style class, so we allow warnings for now
+
+"docstring"
+
+from __future__ import division
+
+def test(a, b):
+    print a, '/', b, '=', a / b
+    t = a
+    t /= b
+    print a, '/', b, '=', t
+
+    print a, '//', b, '=', a // b
+    t = a
+    t //= b
+    print a, '//', b, '=', t
+
+test(3, 2)
+test(3, 2.0)
+test(3.0, 2)
+test(3.0, 2.0)

test/tests/future_non_existent.py

+from __future__ import rvalue_references # should cause syntax error

test/tests/future_not_at_beginning.py

+"docstring"
+
+"not a docstring"
+
+from __future__ import division # should cause syntax error

test/tests/future_not_at_beginning2.py

+"docstring"
+
+def f():
+    from __future__ import division # should cause syntax error

tools/tester.py

@@ -106,6 +106,7 @@ def run_test(fn, check_stats, run_memcheck):
     statchecks = []
     jit_args = ["-csrq"] + EXTRA_JIT_ARGS
     expected = "success"
+    allow_warning = False
     for l in open(fn):
         l = l.strip()
         if not l:
@@ -125,6 +126,8 @@ def run_test(fn, check_stats, run_memcheck):
             skip = eval(skip_if)
             if skip:
                 return r + "    (skipped due to 'skip-if: %s')" % skip_if[:30]
+        elif l.startswith("# allow-warning"):
+            allow_warning = True
 
     assert expected in ("success", "fail", "statfail"), expected
 
@@ -143,6 +146,9 @@ def run_test(fn, check_stats, run_memcheck):
     elapsed = time.time() - start
 
     stats = {}
+    if allow_warning:
+        out_lines = [l for l in out.split('\n') if not l.startswith("Warning: ")]
+        out = "\n".join(out_lines)
     if code == 0 and not TEST_PYPY:
         assert out.count("Stats:") == 1
         out, stats_str = out.split("Stats:")