fix a bunch of bugs which showed up in test_builtin.py

Unfortunately there are still a few hiding which means we can't yet run test_builtin.py right now

docs/INSTALLING.md

@@ -28,6 +28,12 @@ sudo apt-get install -yq automake git cmake ninja-build ccache libncurses5-dev l
 sudo yum install git make cmake clang gcc gcc-c++ ccache ninja-build xz-devel automake libtool gmp-devel mpfr-devel readline-devel openssl-devel sqlite-devel python-devel zlib-devel bzip2-devel ncurses-devel texlive-latex2man libffi-devel
 ```
 
+### Additional prerequisites for running the integration tests
+**Ubuntu**
+```
+sudo apt-get install libgeoip-dev
+```
+
 ### Building and testing
 ```
 git clone https://github.com/dropbox/pyston.git ~/pyston

src/capi/abstract.cpp

@@ -1947,15 +1947,15 @@ extern "C" PyObject* PyNumber_Positive(PyObject* o) noexcept {
 }
 
 extern "C" PyObject* PyNumber_Absolute(PyObject* o) noexcept {
-    if (o == Py_None)
-        return type_error("bad operand type for abs(): '%.200s'", o);
+    PyNumberMethods* m;
 
-    try {
-        return abs_(o);
-    } catch (ExcInfo e) {
-        fatalOrError(PyExc_NotImplementedError, "unimplemented");
-        return nullptr;
-    }
+    if (o == NULL)
+        return null_error();
+    m = o->cls->tp_as_number;
+    if (m && m->nb_absolute)
+        return m->nb_absolute(o);
+
+    return type_error("bad operand type for abs(): '%.200s'", o);
 }
 
 extern "C" PyObject* PyNumber_Invert(PyObject* o) noexcept {

src/runtime/builtin_modules/builtins.cpp

@@ -90,19 +90,10 @@ extern "C" Box* vars(Box* obj) {
 }
 
 extern "C" Box* abs_(Box* x) {
-    if (PyInt_Check(x)) {
-        i64 n = static_cast<BoxedInt*>(x)->n;
-        return boxInt(n >= 0 ? n : -n);
-    } else if (x->cls == float_cls) {
-        double d = static_cast<BoxedFloat*>(x)->d;
-        return boxFloat(std::abs(d));
-    } else if (x->cls == long_cls) {
-        return longAbs(static_cast<BoxedLong*>(x));
-    } else {
-        static BoxedString* abs_str = internStringImmortal("__abs__");
-        CallattrFlags callattr_flags{.cls_only = true, .null_on_nonexistent = false, .argspec = ArgPassSpec(0) };
-        return callattr(x, abs_str, callattr_flags, NULL, NULL, NULL, NULL, NULL);
-    }
+    Box* rtn = PyNumber_Absolute(x);
+    if (!rtn)
+        throwCAPIException();
+    return rtn;
 }
 
 extern "C" Box* binFunc(Box* x) {
@@ -326,11 +317,12 @@ extern "C" Box* chr(Box* arg) {
 }
 
 extern "C" Box* unichr(Box* arg) {
-    if (arg->cls != int_cls)
-        raiseExcHelper(TypeError, "an integer is required");
+    int n = -1;
+    if (!PyArg_ParseSingle(arg, 0, "unichr", "i", &n))
+        throwCAPIException();
 
-    i64 n = static_cast<BoxedInt*>(arg)->n;
     Box* rtn = PyUnicode_FromOrdinal(n);
+    if (!rtn)
         checkAndThrowCAPIException();
     return rtn;
 }
@@ -936,6 +928,111 @@ Fail_1:
     return NULL;
 }
 
+static PyObject* filterunicode(PyObject* func, PyObject* strobj) {
+    PyObject* result;
+    Py_ssize_t i, j;
+    Py_ssize_t len = PyUnicode_GetSize(strobj);
+    Py_ssize_t outlen = len;
+
+    if (func == Py_None) {
+        /* If it's a real string we can return the original,
+         * as no character is ever false and __getitem__
+         * does return this character. If it's a subclass
+         * we must go through the __getitem__ loop */
+        if (PyUnicode_CheckExact(strobj)) {
+            Py_INCREF(strobj);
+            return strobj;
+        }
+    }
+    if ((result = PyUnicode_FromUnicode(NULL, len)) == NULL)
+        return NULL;
+
+    for (i = j = 0; i < len; ++i) {
+        PyObject* item, *arg, *good;
+        int ok;
+
+        item = (*strobj->cls->tp_as_sequence->sq_item)(strobj, i);
+        if (item == NULL)
+            goto Fail_1;
+        if (func == Py_None) {
+            ok = 1;
+        } else {
+            arg = PyTuple_Pack(1, item);
+            if (arg == NULL) {
+                Py_DECREF(item);
+                goto Fail_1;
+            }
+            good = PyEval_CallObject(func, arg);
+            Py_DECREF(arg);
+            if (good == NULL) {
+                Py_DECREF(item);
+                goto Fail_1;
+            }
+            ok = PyObject_IsTrue(good);
+            Py_DECREF(good);
+        }
+        if (ok > 0) {
+            Py_ssize_t reslen;
+            if (!PyUnicode_Check(item)) {
+                PyErr_SetString(PyExc_TypeError, "can't filter unicode to unicode:"
+                                                 " __getitem__ returned different type");
+                Py_DECREF(item);
+                goto Fail_1;
+            }
+            reslen = PyUnicode_GET_SIZE(item);
+            if (reslen == 1)
+                PyUnicode_AS_UNICODE(result)[j++] = PyUnicode_AS_UNICODE(item)[0];
+            else {
+                /* do we need more space? */
+                Py_ssize_t need = j + reslen + len - i - 1;
+
+                /* check that didnt overflow */
+                if ((j > PY_SSIZE_T_MAX - reslen) || ((j + reslen) > PY_SSIZE_T_MAX - len) || ((j + reslen + len) < i)
+                    || ((j + reslen + len - i) <= 0)) {
+                    Py_DECREF(item);
+                    return NULL;
+                }
+
+                assert(need >= 0);
+                assert(outlen >= 0);
+
+                if (need > outlen) {
+                    /* overallocate,
+                       to avoid reallocations */
+                    if (need < 2 * outlen) {
+                        if (outlen > PY_SSIZE_T_MAX / 2) {
+                            Py_DECREF(item);
+                            return NULL;
+                        } else {
+                            need = 2 * outlen;
+                        }
+                    }
+
+                    if (PyUnicode_Resize(&result, need) < 0) {
+                        Py_DECREF(item);
+                        goto Fail_1;
+                    }
+                    outlen = need;
+                }
+                memcpy(PyUnicode_AS_UNICODE(result) + j, PyUnicode_AS_UNICODE(item), reslen * sizeof(Py_UNICODE));
+                j += reslen;
+            }
+        }
+        Py_DECREF(item);
+        if (ok < 0)
+            goto Fail_1;
+    }
+
+    if (j < outlen)
+        PyUnicode_Resize(&result, j);
+
+    return result;
+
+Fail_1:
+    Py_DECREF(result);
+    return NULL;
+}
+
 static PyObject* filtertuple(PyObject* func, PyObject* tuple) {
     PyObject* result;
     Py_ssize_t i, j;
@@ -1012,7 +1109,6 @@ Box* filter2(Box* f, Box* container) {
         f = bool_cls;
 
     // Special cases depending on the type of container influences the return type
-    // TODO There are other special cases like this
     if (PyTuple_Check(container)) {
         Box* rtn = filtertuple(f, static_cast<BoxedTuple*>(container));
         if (!rtn) {
@@ -1029,6 +1125,14 @@ Box* filter2(Box* f, Box* container) {
         return rtn;
     }
 
+    if (PyUnicode_Check(container)) {
+        Box* rtn = filterunicode(f, container);
+        if (!rtn) {
+            throwCAPIException();
+        }
+        return rtn;
+    }
+
     Box* rtn = new BoxedList();
     for (Box* e : container->pyElements()) {
         Box* r = runtimeCall(f, ArgPassSpec(1), e, NULL, NULL, NULL, NULL);
@@ -1511,7 +1615,7 @@ Box* input(Box* prompt) {
 Box* builtinRound(Box* _number, Box* _ndigits) {
     double x = PyFloat_AsDouble(_number);
     if (PyErr_Occurred())
-        raiseExcHelper(TypeError, "a float is required");
+        throwCAPIException();
 
     /* interpret 2nd argument as a Py_ssize_t; clip on overflow */
     Py_ssize_t ndigits = PyNumber_AsSsize_t(_ndigits, NULL);
@@ -2050,8 +2154,8 @@ void setupBuiltins() {
                                                  { NULL, NULL }, NULL, range_doc);
     builtins_module->giveAttr("range", range_obj);
 
-    auto* round_obj
-        = new BoxedBuiltinFunctionOrMethod(FunctionMetadata::create((void*)builtinRound, BOXED_FLOAT, 2, false, false),
+    auto* round_obj = new BoxedBuiltinFunctionOrMethod(
+        FunctionMetadata::create((void*)builtinRound, BOXED_FLOAT, 2, ParamNames({ "number", "ndigits" }, "", "")),
         "round", { boxInt(0) }, NULL, round_doc);
     builtins_module->giveAttr("round", round_obj);
 
@@ -2161,7 +2265,7 @@ void setupBuiltins() {
                                          FunctionMetadata::create((void*)builtinCmp, UNKNOWN, 2), "cmp", cmp_doc));
     builtins_module->giveAttr(
         "format", new BoxedBuiltinFunctionOrMethod(FunctionMetadata::create((void*)builtinFormat, UNKNOWN, 2), "format",
-                                                   format_doc));
+                                                   { NULL }, NULL, format_doc));
 
 
     static PyMethodDef builtin_methods[] = {

src/runtime/dict.cpp

@@ -14,6 +14,7 @@
 
 #include "runtime/dict.h"
 
+#include "capi/typeobject.h"
 #include "capi/types.h"
 #include "core/ast.h"
 #include "core/common.h"
@@ -855,6 +856,7 @@ void setupDict() {
         = dictiteritem_cls->instances_are_nonzero = true;
 
     dict_cls->tp_dealloc = &BoxedDict::dealloc;
+    dict_cls->tp_hash = PyObject_HashNotImplemented;
     dict_cls->has_safe_tp_dealloc = true;
 
     dict_cls->giveAttr("__len__", new BoxedFunction(FunctionMetadata::create((void*)dictLen, BOXED_INT, 1)));
@@ -913,6 +915,7 @@ void setupDict() {
 
     dict_cls->giveAttr("__nonzero__", new BoxedFunction(FunctionMetadata::create((void*)dictNonzero, BOXED_BOOL, 1)));
 
+    add_operators(dict_cls);
     dict_cls->freeze();
 
     // create the dictonary iterator types

src/runtime/int.cpp

@@ -365,16 +365,20 @@ extern "C" Box* div_i64_i64(i64 lhs, i64 rhs) {
     return boxInt(div_result);
 }
 
-extern "C" i64 mod_i64_i64(i64 lhs, i64 rhs) {
+extern "C" Box* mod_i64_i64(i64 lhs, i64 rhs) {
     if (rhs == 0) {
         raiseExcHelper(ZeroDivisionError, "integer division or modulo by zero");
     }
-    // I don't think this can overflow:
+
+    // this would overflow:
+    if (lhs == PYSTON_INT_MIN && rhs == -1)
+        return boxLong(0); // long because pypy and cpython both return a long
+
     if (lhs < 0 && rhs > 0)
-        return ((lhs + 1) % rhs) + (rhs - 1);
+        return boxInt(((lhs + 1) % rhs) + (rhs - 1));
     if (lhs > 0 && rhs < 0)
-        return ((lhs - 1) % rhs) + (rhs + 1);
-    return lhs % rhs;
+        return boxInt(((lhs - 1) % rhs) + (rhs + 1));
+    return boxInt(lhs % rhs);
 }
 
 extern "C" Box* pow_i64_i64(i64 lhs, i64 rhs, Box* mod) {
@@ -709,7 +713,7 @@ Box* intRLShift(BoxedInt* lhs, Box* rhs) {
 extern "C" Box* intModInt(BoxedInt* lhs, BoxedInt* rhs) {
     assert(PyInt_Check(lhs));
     assert(PyInt_Check(rhs));
-    return boxInt(mod_i64_i64(lhs->n, rhs->n));
+    return mod_i64_i64(lhs->n, rhs->n);
 }
 
 extern "C" Box* intMod(BoxedInt* lhs, Box* rhs) {
@@ -720,7 +724,7 @@ extern "C" Box* intMod(BoxedInt* lhs, Box* rhs) {
         return NotImplemented;
     }
     BoxedInt* rhs_int = static_cast<BoxedInt*>(rhs);
-    return boxInt(mod_i64_i64(lhs->n, rhs_int->n));
+    return mod_i64_i64(lhs->n, rhs_int->n);
 }
 
 Box* intRMod(BoxedInt* lhs, Box* rhs) {
@@ -732,7 +736,7 @@ Box* intRMod(BoxedInt* lhs, Box* rhs) {
         return NotImplemented;
     }
     BoxedInt* rhs_int = static_cast<BoxedInt*>(rhs);
-    return boxInt(mod_i64_i64(rhs_int->n, lhs->n));
+    return mod_i64_i64(rhs_int->n, lhs->n);
 }
 
 extern "C" Box* intDivmod(BoxedInt* lhs, Box* rhs) {
@@ -1010,6 +1014,9 @@ extern "C" Box* intHash(BoxedInt* self) {
         raiseExcHelper(TypeError, "descriptor '__hash__' requires a 'int' object but received a '%s'",
                        getTypeName(self));
 
+    if (self->n == -1)
+        return boxInt(-2);
+
     if (self->cls == int_cls)
         return self;
     return boxInt(self->n);
@@ -1355,6 +1362,30 @@ static PyObject* int_getnewargs(BoxedInt* v) noexcept {
     return Py_BuildValue("(l)", v->n);
 }
 
+static Box* intFormat(PyObject* self, Box* format_spec) {
+    if (PyBytes_Check(format_spec)) {
+        Box* rtn = _PyInt_FormatAdvanced(self, PyBytes_AS_STRING(format_spec), PyBytes_GET_SIZE(format_spec));
+        if (!rtn)
+            throwCAPIException();
+        return rtn;
+    }
+    if (PyUnicode_Check(format_spec)) {
+        /* Convert format_spec to a str */
+        PyObject* result;
+        PyObject* str_spec = PyObject_Str(format_spec);
+
+        if (str_spec == NULL)
+            throwCAPIException();
+
+        result = _PyInt_FormatAdvanced(self, PyBytes_AS_STRING(str_spec), PyBytes_GET_SIZE(str_spec));
+        Py_DECREF(str_spec);
+        if (!result)
+            throwCAPIException();
+        return result;
+    }
+    raiseExcHelper(TypeError, "__format__ requires str or unicode");
+}
+
 void setupInt() {
     static PyNumberMethods int_as_number;
     int_cls->tp_as_number = &int_as_number;
@@ -1376,7 +1407,7 @@ void setupInt() {
     _addFuncIntFloatUnknown("__floordiv__", (void*)intFloordivInt, (void*)intFloordivFloat, (void*)intFloordiv);
     _addFuncIntFloatUnknown("__truediv__", (void*)intTruedivInt, (void*)intTruedivFloat, (void*)intTruediv);
     _addFuncIntFloatUnknown("__mul__", (void*)intMulInt, (void*)intMulFloat, (void*)intMul);
-    _addFuncIntUnknown("__mod__", BOXED_INT, (void*)intModInt, (void*)intMod);
+    _addFuncIntUnknown("__mod__", UNKNOWN, (void*)intModInt, (void*)intMod);
     _addFuncPow("__pow__", BOXED_INT, (void*)intPowFloat, (void*)intPow);
 
     int_cls->giveAttr("__radd__", new BoxedFunction(FunctionMetadata::create((void*)intRAdd, UNKNOWN, 2)));
@@ -1422,6 +1453,8 @@ void setupInt() {
     int_cls->giveAttr("__float__", new BoxedFunction(FunctionMetadata::create((void*)intFloat, BOXED_FLOAT, 1)));
     int_cls->giveAttr("__long__", new BoxedFunction(FunctionMetadata::create((void*)intLong, LONG, 1)));
 
+    int_cls->giveAttr("__format__", new BoxedFunction(FunctionMetadata::create((void*)intFormat, STR, 2)));
+
     int_cls->giveAttr("__doc__",
                       boxString("int(x=0) -> int or long\n"
                                 "int(x, base=10) -> int or long\n"

src/runtime/int.h

@@ -28,7 +28,7 @@ static_assert(sizeof(int64_t) == sizeof(long), "");
 #define PYSTON_INT_MAX LONG_MAX
 
 extern "C" Box* div_i64_i64(i64 lhs, i64 rhs);
-extern "C" i64 mod_i64_i64(i64 lhs, i64 rhs);
+extern "C" Box* mod_i64_i64(i64 lhs, i64 rhs);
 
 extern "C" Box* add_i64_i64(i64 lhs, i64 rhs);
 extern "C" Box* sub_i64_i64(i64 lhs, i64 rhs);

src/runtime/long.cpp

@@ -1531,8 +1531,12 @@ Box* longHash(BoxedLong* self) {
                        getTypeName(self));
 
     // If the long fits into an int we have to return the same hash in order that we can find the value in a dict.
-    if (mpz_fits_slong_p(self->n))
-        return boxInt(mpz_get_si(self->n));
+    if (mpz_fits_slong_p(self->n)) {
+        auto v = mpz_get_si(self->n);
+        if (v == -1)
+            v = -2;
+        return boxInt(v);
+    }
 
     // CPython use the absolute value of self mod ULONG_MAX.
     unsigned long remainder = mpz_tdiv_ui(self->n, ULONG_MAX);
@@ -1547,6 +1551,14 @@ Box* longHash(BoxedLong* self) {
     return boxInt(remainder);
 }
 
+long long_hash(PyObject* self) noexcept {
+    try {
+        return unboxInt(longHash((BoxedLong*)self));
+    } catch (ExcInfo e) {
+        RELEASE_ASSERT(0, "");
+    }
+}
+
 extern "C" Box* longTrunc(BoxedLong* self) {
     if (!PyLong_Check(self))
         raiseExcHelper(TypeError, "descriptor '__trunc__' requires a 'long' object but received a '%s'",
@@ -1734,5 +1746,6 @@ void setupLong() {
     long_cls->freeze();
 
     long_cls->tp_as_number->nb_power = long_pow;
+    long_cls->tp_hash = long_hash;
 }
 }

test/tests/builtins.py

+import sys
+
 __builtins__.aoeu = 1
 print aoeu
 
@@ -41,6 +43,8 @@ print list(enumerate(start=-42, sequence=xrange(5, 10)))
 # If the first argument is None, filter calls checks for truthiness (ie is equivalent to passing 'bool')
 print filter(None, xrange(-5, 5))
 
+print filter(None, unicode("12"))
+
 print isinstance(1, int)
 print isinstance(1, (float, int))
 print isinstance(1, (float, (), (int, 3), 4))
@@ -79,6 +83,7 @@ print hex(12345)
 print oct(234)
 print hex(0)
 print oct(0) # This should not add an additional leading 0, ie should return "0" not "00"
+print abs((-sys.maxint)-1)
 
 try:
     print hex([])
@@ -129,9 +134,24 @@ print apply(sorted, [l], { "reverse" : True })
 print format(5.0, '+')
 print format(5.011111111111, '+.6')
 print format("abc", '')
+print format(0, str(10))
 
 print '{n}'.format(n=None)
 
+print hash(1L)
+def C(long):
+    def __hash__(self):
+        return self
+print hash(2L)
+try:
+    print hash({})
+except TypeError as e:
+    print e
+try:
+    print hash(set())
+except TypeError as e:
+    print e
+
 # Thankfully, setting __builtins__ has no effect:
 __builtins__ = {'zzz': 2}
 try: