stdlib updates

- Add some more time functions
- Workaround for logging for now, to not try to show the caller frame
- Copy over everything in LIBPYSTON in the cmake build
  (useful for django to copy over things like templates)
- Allow str/unicode subclasses in more places
- Add a BoxedFile gc handler
- PyLong_FromString
- set.difference
- iter(iter(""))

from_cpython/Include/pyconfig.h

@@ -56,6 +56,8 @@
 #define HAVE_GETPEERNAME 1
 #define HAVE_STRFTIME 1
 #define HAVE_TIMES 1
+#define HAVE_STRUCT_TM_TM_ZONE 1
+#define HAVE_MKTIME 1
 
 #define PY_FORMAT_LONG_LONG "ll"
 #define PY_FORMAT_SIZE_T "z"

from_cpython/Lib/logging/__init__.py

@@ -1228,6 +1228,10 @@ class Logger(Filterer):
         Find the stack frame of the caller so that we can note the source
         file name, line number and function name.
         """
+
+        # Pyston change:
+        return "(unknown file)", 0, "(unknown function)"
+
         f = currentframe()
         #On some versions of IronPython, currentframe() returns None if
         #IronPython isn't run with -X:Frames.

lib_pyston/CMakeLists.txt

 # Copy any changed lib_pyston sources:
-file(GLOB_RECURSE LIBPYSTON_SRCS . "*.py")
+file(GLOB_RECURSE LIBPYSTON_SRCS . "*")
 set(LIBPYSTON_TARGETS "")
 foreach(STDLIB_FILE ${LIBPYSTON_SRCS})
     file(RELATIVE_PATH FN_REL ${CMAKE_SOURCE_DIR} ${STDLIB_FILE})

src/runtime/builtin_modules/builtins.cpp

@@ -516,7 +516,7 @@ Box* delattrFunc(Box* obj, Box* _str) {
 Box* getattrFunc(Box* obj, Box* _str, Box* default_value) {
     _str = coerceUnicodeToStr(_str);
 
-    if (_str->cls != str_cls) {
+    if (!isSubclass(_str->cls, str_cls)) {
         raiseExcHelper(TypeError, "getattr(): attribute name must be string");
     }
 

src/runtime/file.cpp

@@ -1358,6 +1358,18 @@ void fileDestructor(Box* b) {
     self->f_fp = NULL;
 }
 
+void BoxedFile::gcHandler(GCVisitor* v, Box* b) {
+    boxGCHandler(v, b);
+
+    assert(isSubclass(b->cls, file_cls));
+    BoxedFile* f = static_cast<BoxedFile*>(b);
+
+    v->visit(f->f_name);
+    v->visit(f->f_mode);
+    v->visit(f->f_encoding);
+    v->visit(f->f_errors);
+}
+
 void setupFile() {
     file_cls->simple_destructor = fileDestructor;
 

src/runtime/file.h

@@ -22,7 +22,8 @@ namespace pyston {
 
 class BoxedFile : public Box {
 public:
-    PyObject_HEAD FILE* f_fp;
+    PyObject_HEAD;
+    FILE* f_fp;
     PyObject* f_name;
     PyObject* f_mode;
     int (*f_close)(FILE*);
@@ -48,6 +49,8 @@ public:
         __attribute__((visibility("default")));
 
     DEFAULT_CLASS(file_cls);
+
+    static void gcHandler(GCVisitor* v, Box* b);
 };
 }
 

src/runtime/int.cpp

@@ -931,7 +931,7 @@ static Box* _intNew(Box* val, Box* base) {
         if (val->cls == int_cls)
             return n;
         return new BoxedInt(n->n);
-    } else if (val->cls == str_cls) {
+    } else if (isSubclass(val->cls, str_cls)) {
         int base_n;
         if (!base)
             base_n = 10;
@@ -947,7 +947,7 @@ static Box* _intNew(Box* val, Box* base) {
         if (!r)
             throwCAPIException();
         return r;
-    } else if (val->cls == unicode_cls) {
+    } else if (isSubclass(val->cls, unicode_cls)) {
         int base_n;
         if (!base)
             base_n = 10;

src/runtime/long.cpp

@@ -102,7 +102,15 @@ extern "C" PY_LONG_LONG PyLong_AsLongLongAndOverflow(PyObject* obj, int* overflo
 }
 
 extern "C" PyObject* PyLong_FromString(const char* str, char** pend, int base) noexcept {
-    Py_FatalError("unimplemented");
+    RELEASE_ASSERT(pend == NULL, "unsupported");
+
+    // See comment in _longNew
+    RELEASE_ASSERT(base >= 0, "unsupported");
+
+    BoxedLong* rtn = new BoxedLong();
+    int r = mpz_init_set_str(rtn->n, str, base);
+    RELEASE_ASSERT(r == 0, "");
+    return rtn;
 }
 
 static int64_t asSignedLong(BoxedLong* self) {

src/runtime/objmodel.cpp

@@ -1989,7 +1989,7 @@ extern "C" bool nonzero(Box* obj) {
     if (func == NULL) {
         ASSERT(isUserDefined(obj->cls) || obj->cls == classobj_cls || obj->cls == type_cls
                    || isSubclass(obj->cls, Exception) || obj->cls == file_cls || obj->cls == traceback_cls
-                   || obj->cls == instancemethod_cls || obj->cls == module_cls,
+                   || obj->cls == instancemethod_cls || obj->cls == module_cls || obj->cls == capifunc_cls,
                "%s.__nonzero__", getTypeName(obj)); // TODO
 
         // TODO should rewrite these?
@@ -2109,7 +2109,7 @@ extern "C" BoxedInt* hash(Box* obj) {
 
     if (hash == NULL) {
         ASSERT(isUserDefined(obj->cls) || obj->cls == function_cls || obj->cls == object_cls || obj->cls == classobj_cls
-                   || obj->cls == module_cls,
+                   || obj->cls == module_cls || obj->cls == capifunc_cls || obj->cls == instancemethod_cls,
                "%s.__hash__", getTypeName(obj));
         // TODO not the best way to handle this...
         return static_cast<BoxedInt*>(boxInt((i64)obj));

src/runtime/set.cpp

@@ -262,6 +262,22 @@ Box* setUnion(BoxedSet* self, BoxedTuple* args) {
     return rtn;
 }
 
+Box* setDifference(BoxedSet* self, BoxedTuple* args) {
+    if (!isSubclass(self->cls, set_cls) && !isSubclass(self->cls, frozenset_cls))
+        raiseExcHelper(TypeError, "descriptor 'difference' requires a 'set' object but received a '%s'",
+                       getTypeName(self));
+
+    BoxedSet* rtn = (BoxedSet*)setNew(self->cls, self);
+
+    for (auto container : args->pyElements()) {
+        for (auto elt : container->pyElements()) {
+            rtn->s.erase(elt);
+        }
+    }
+
+    return rtn;
+}
+
 static BoxedSet* setIntersection2(BoxedSet* self, Box* container) {
     assert(self->cls == set_cls);
 
@@ -458,6 +474,8 @@ void setupSet() {
     set_cls->giveAttr("union", new BoxedFunction(boxRTFunction((void*)setUnion, UNKNOWN, 1, 0, true, false)));
     set_cls->giveAttr("intersection",
                       new BoxedFunction(boxRTFunction((void*)setIntersection, UNKNOWN, 1, 0, true, false)));
+    set_cls->giveAttr("difference", new BoxedFunction(boxRTFunction((void*)setDifference, UNKNOWN, 1, 0, true, false)));
+    frozenset_cls->giveAttr("difference", set_cls->getattr("difference"));
     set_cls->giveAttr("issubset", new BoxedFunction(boxRTFunction((void*)setIssubset, UNKNOWN, 2)));
     set_cls->giveAttr("issuperset", new BoxedFunction(boxRTFunction((void*)setIssuperset, UNKNOWN, 2)));
 

src/runtime/str.cpp

@@ -304,13 +304,16 @@ extern "C" PyObject* PyString_FromFormat(const char* format, ...) noexcept {
 extern "C" Box* strAdd(BoxedString* lhs, Box* _rhs) {
     assert(isSubclass(lhs->cls, str_cls));
 
-    if (_rhs->cls == unicode_cls) {
+    if (isSubclass(_rhs->cls, unicode_cls)) {
         Box* rtn = PyUnicode_Concat(lhs, _rhs);
         checkAndThrowCAPIException();
         return rtn;
     }
 
-    if (_rhs->cls != str_cls) {
+    if (!isSubclass(_rhs->cls, str_cls)) {
+        // Note: this is deliberately not returning NotImplemented, even though
+        // that would be more usual.  I assume this behavior of CPython's is
+        // for backwards compatibility.
         raiseExcHelper(TypeError, "cannot concatenate 'str' and '%s' objects", getTypeName(_rhs));
     }
 
@@ -1094,7 +1097,7 @@ extern "C" Box* strMul(BoxedString* lhs, Box* rhs) {
 extern "C" Box* strLt(BoxedString* lhs, Box* rhs) {
     assert(isSubclass(lhs->cls, str_cls));
 
-    if (rhs->cls != str_cls)
+    if (!isSubclass(rhs->cls, str_cls))
         return NotImplemented;
 
     BoxedString* srhs = static_cast<BoxedString*>(rhs);
@@ -1104,7 +1107,7 @@ extern "C" Box* strLt(BoxedString* lhs, Box* rhs) {
 extern "C" Box* strLe(BoxedString* lhs, Box* rhs) {
     assert(isSubclass(lhs->cls, str_cls));
 
-    if (rhs->cls != str_cls)
+    if (!isSubclass(rhs->cls, str_cls))
         return NotImplemented;
 
     BoxedString* srhs = static_cast<BoxedString*>(rhs);
@@ -1114,7 +1117,7 @@ extern "C" Box* strLe(BoxedString* lhs, Box* rhs) {
 extern "C" Box* strGt(BoxedString* lhs, Box* rhs) {
     assert(isSubclass(lhs->cls, str_cls));
 
-    if (rhs->cls != str_cls)
+    if (!isSubclass(rhs->cls, str_cls))
         return NotImplemented;
 
     BoxedString* srhs = static_cast<BoxedString*>(rhs);
@@ -1124,7 +1127,7 @@ extern "C" Box* strGt(BoxedString* lhs, Box* rhs) {
 extern "C" Box* strGe(BoxedString* lhs, Box* rhs) {
     assert(isSubclass(lhs->cls, str_cls));
 
-    if (rhs->cls != str_cls)
+    if (!isSubclass(rhs->cls, str_cls))
         return NotImplemented;
 
     BoxedString* srhs = static_cast<BoxedString*>(rhs);
@@ -1134,7 +1137,7 @@ extern "C" Box* strGe(BoxedString* lhs, Box* rhs) {
 extern "C" Box* strEq(BoxedString* lhs, Box* rhs) {
     assert(isSubclass(lhs->cls, str_cls));
 
-    if (rhs->cls != str_cls)
+    if (!isSubclass(rhs->cls, str_cls))
         return NotImplemented;
 
     BoxedString* srhs = static_cast<BoxedString*>(rhs);
@@ -1144,7 +1147,7 @@ extern "C" Box* strEq(BoxedString* lhs, Box* rhs) {
 extern "C" Box* strNe(BoxedString* lhs, Box* rhs) {
     assert(isSubclass(lhs->cls, str_cls));
 
-    if (rhs->cls != str_cls)
+    if (!isSubclass(rhs->cls, str_cls))
         return NotImplemented;
 
     BoxedString* srhs = static_cast<BoxedString*>(rhs);
@@ -1689,7 +1692,7 @@ extern "C" PyObject* _PyString_Join(PyObject* sep, PyObject* x) noexcept {
 }
 
 Box* strReplace(Box* _self, Box* _old, Box* _new, Box** _args) {
-    if (_self->cls != str_cls)
+    if (!isSubclass(_self->cls, str_cls))
         raiseExcHelper(TypeError, "descriptor 'replace' requires a 'str' object but received a '%s'",
                        getTypeName(_self));
     BoxedString* self = static_cast<BoxedString*>(_self);
@@ -1699,11 +1702,11 @@ Box* strReplace(Box* _self, Box* _old, Box* _new, Box** _args) {
         return PyUnicode_Replace((PyObject*)self, _old, _new, -1 /*count*/);
 #endif
 
-    if (_old->cls != str_cls)
+    if (!isSubclass(_old->cls, str_cls))
         raiseExcHelper(TypeError, "expected a character buffer object");
     BoxedString* old = static_cast<BoxedString*>(_old);
 
-    if (_new->cls != str_cls)
+    if (!isSubclass(_new->cls, str_cls))
         raiseExcHelper(TypeError, "expected a character buffer object");
     BoxedString* new_ = static_cast<BoxedString*>(_new);
 
@@ -1843,20 +1846,20 @@ Box* strTitle(BoxedString* self) {
 }
 
 Box* strTranslate(BoxedString* self, BoxedString* table, BoxedString* delete_chars) {
-    if (self->cls != str_cls)
+    if (!isSubclass(self->cls, str_cls))
         raiseExcHelper(TypeError, "descriptor 'translate' requires a 'str' object but received a '%s'",
                        getTypeName(self));
 
     std::unordered_set<char> delete_set;
     if (delete_chars) {
-        if (delete_chars->cls != str_cls)
+        if (!isSubclass(delete_chars->cls, str_cls))
             raiseExcHelper(TypeError, "expected a character buffer object");
         delete_set.insert(delete_chars->s.begin(), delete_chars->s.end());
     }
 
     bool have_table = table != None;
     if (have_table) {
-        if (table->cls != str_cls)
+        if (!isSubclass(table->cls, str_cls))
             raiseExcHelper(TypeError, "expected a character buffer object");
         if (table->s.size() != 256)
             raiseExcHelper(ValueError, "translation table must be 256 characters long");
@@ -1903,7 +1906,7 @@ Box* strContains(BoxedString* self, Box* elt) {
         return boxBool(r);
     }
 
-    if (elt->cls != str_cls)
+    if (!isSubclass(elt->cls, str_cls))
         raiseExcHelper(TypeError, "'in <string>' requires string as left operand, not %s", getTypeName(elt));
 
     BoxedString* sub = static_cast<BoxedString*>(elt);
@@ -1917,7 +1920,7 @@ Box* strContains(BoxedString* self, Box* elt) {
 Box* strStartswith(BoxedString* self, Box* elt, Box* start, Box** _args) {
     Box* end = _args[0];
 
-    if (self->cls != str_cls)
+    if (!isSubclass(self->cls, str_cls))
         raiseExcHelper(TypeError, "descriptor 'startswith' requires a 'str' object but received a '%s'",
                        getTypeName(self));
 
@@ -1952,7 +1955,7 @@ Box* strStartswith(BoxedString* self, Box* elt, Box* start, Box** _args) {
         return boxBool(r);
     }
 
-    if (elt->cls != str_cls)
+    if (!isSubclass(elt->cls, str_cls))
         raiseExcHelper(TypeError, "expected a character buffer object");
 
     BoxedString* sub = static_cast<BoxedString*>(elt);
@@ -1980,7 +1983,7 @@ Box* strStartswith(BoxedString* self, Box* elt, Box* start, Box** _args) {
 Box* strEndswith(BoxedString* self, Box* elt, Box* start, Box** _args) {
     Box* end = _args[0];
 
-    if (self->cls != str_cls)
+    if (!isSubclass(self->cls, str_cls))
         raiseExcHelper(TypeError, "descriptor 'endswith' requires a 'str' object but received a '%s'",
                        getTypeName(self));
 
@@ -2015,7 +2018,7 @@ Box* strEndswith(BoxedString* self, Box* elt, Box* start, Box** _args) {
         return False;
     }
 
-    if (elt->cls != str_cls)
+    if (!isSubclass(elt->cls, str_cls))
         raiseExcHelper(TypeError, "expected a character buffer object");
 
     BoxedString* sub = static_cast<BoxedString*>(elt);
@@ -2043,7 +2046,7 @@ Box* strEndswith(BoxedString* self, Box* elt, Box* start, Box** _args) {
 }
 
 Box* strDecode(BoxedString* self, Box* encoding, Box* error) {
-    if (self->cls != str_cls)
+    if (!isSubclass(self->cls, str_cls))
         raiseExcHelper(TypeError, "descriptor 'decode' requires a 'str' object but received a '%s'", getTypeName(self));
 
     BoxedString* encoding_str = (BoxedString*)encoding;
@@ -2052,13 +2055,13 @@ Box* strDecode(BoxedString* self, Box* encoding, Box* error) {
     if (encoding_str && encoding_str->cls == unicode_cls)
         encoding_str = (BoxedString*)_PyUnicode_AsDefaultEncodedString(encoding_str, NULL);
 
-    if (encoding_str && encoding_str->cls != str_cls)
+    if (encoding_str && !isSubclass(encoding_str->cls, str_cls))
         raiseExcHelper(TypeError, "decode() argument 1 must be string, not '%s'", getTypeName(encoding_str));
 
     if (error_str && error_str->cls == unicode_cls)
         error_str = (BoxedString*)_PyUnicode_AsDefaultEncodedString(error_str, NULL);
 
-    if (error_str && error_str->cls != str_cls)
+    if (error_str && !isSubclass(error_str->cls, str_cls))
         raiseExcHelper(TypeError, "decode() argument 2 must be string, not '%s'", getTypeName(error_str));
 
     Box* result
@@ -2068,7 +2071,7 @@ Box* strDecode(BoxedString* self, Box* encoding, Box* error) {
 }
 
 Box* strEncode(BoxedString* self, Box* encoding, Box* error) {
-    if (self->cls != str_cls)
+    if (!isSubclass(self->cls, str_cls))
         raiseExcHelper(TypeError, "descriptor 'encode' requires a 'str' object but received a '%s'", getTypeName(self));
 
     BoxedString* encoding_str = (BoxedString*)encoding;
@@ -2077,13 +2080,13 @@ Box* strEncode(BoxedString* self, Box* encoding, Box* error) {
     if (encoding_str && encoding_str->cls == unicode_cls)
         encoding_str = (BoxedString*)_PyUnicode_AsDefaultEncodedString(encoding_str, NULL);
 
-    if (encoding_str && encoding_str->cls != str_cls)
+    if (encoding_str && !isSubclass(encoding_str->cls, str_cls))
         raiseExcHelper(TypeError, "encode() argument 1 must be string, not '%s'", getTypeName(encoding_str));
 
     if (error_str && error_str->cls == unicode_cls)
         error_str = (BoxedString*)_PyUnicode_AsDefaultEncodedString(error_str, NULL);
 
-    if (error_str && error_str->cls != str_cls)
+    if (error_str && !isSubclass(error_str->cls, str_cls))
         raiseExcHelper(TypeError, "encode() argument 2 must be string, not '%s'", getTypeName(error_str));
 
     Box* result = PyCodec_Encode(self, encoding_str ? encoding_str->s.c_str() : PyUnicode_GetDefaultEncoding(),
@@ -2145,6 +2148,11 @@ public:
         return boxBool(self->it != self->end);
     }
 
+    static Box* iter(BoxedStringIterator* self) {
+        assert(self->cls == str_iterator_cls);
+        return self;
+    }
+
     static Box* next(BoxedStringIterator* self) {
         assert(self->cls == str_iterator_cls);
         assert(hasnextUnboxed(self));
@@ -2454,6 +2462,8 @@ void setupStr() {
                                               sizeof(BoxedStringIterator), false, "striterator");
     str_iterator_cls->giveAttr("__hasnext__",
                                new BoxedFunction(boxRTFunction((void*)BoxedStringIterator::hasnext, BOXED_BOOL, 1)));
+    str_iterator_cls->giveAttr("__iter__",
+                               new BoxedFunction(boxRTFunction((void*)BoxedStringIterator::iter, UNKNOWN, 1)));
     str_iterator_cls->giveAttr("next", new BoxedFunction(boxRTFunction((void*)BoxedStringIterator::next, STR, 1)));
     str_iterator_cls->freeze();
     str_iterator_cls->tpp_hasnext = (BoxedClass::pyston_inquiry)BoxedStringIterator::hasnextUnboxed;

src/runtime/types.cpp

@@ -1842,8 +1842,8 @@ void setupRuntime() {
     attrwrapper_cls = new BoxedHeapClass(object_cls, &AttrWrapper::gcHandler, 0, 0, sizeof(AttrWrapper), false,
                                          new BoxedString("attrwrapper"));
     dict_cls = new BoxedHeapClass(object_cls, &dictGCHandler, 0, 0, sizeof(BoxedDict), false, new BoxedString("dict"));
-    file_cls = new BoxedHeapClass(object_cls, NULL, 0, offsetof(BoxedFile, weakreflist), sizeof(BoxedFile), false,
-                                  new BoxedString("file"));
+    file_cls = new BoxedHeapClass(object_cls, &BoxedFile::gcHandler, 0, offsetof(BoxedFile, weakreflist),
+                                  sizeof(BoxedFile), false, new BoxedString("file"));
     int_cls = new BoxedHeapClass(object_cls, NULL, 0, 0, sizeof(BoxedInt), false, new BoxedString("int"));
     bool_cls = new BoxedHeapClass(int_cls, NULL, 0, 0, sizeof(BoxedBool), false, new BoxedString("bool"));
     complex_cls = new BoxedHeapClass(object_cls, NULL, 0, 0, sizeof(BoxedComplex), false, new BoxedString("complex"));

test/tests/long.py

@@ -98,3 +98,5 @@ x = I(D(C()))
 print type(x)
 
 print type(long(C()))
+
+print repr(int("123456789123456789123456789", 16))

test/tests/set.py

@@ -86,6 +86,11 @@ print sorted(s1.union(s2)), sorted(s1.intersection(s2))
 print sorted(s1.union(range(5, 7))), sorted(s1.intersection(range(5, 7)))
 print sorted(s2.union([], [], [], [])), sorted(s2.intersection())
 
+s = frozenset([1, 5])
+d = s.difference([1], [1], [2])
+print d, len(s)
+print
+
 l = []
 s = set(range(5))
 while s:
@@ -102,4 +107,4 @@ print s
 s = set(range(5))
 for i in xrange(10):
     s2 = set(range(i))
-    print s.issubset(s2), s.issuperset(s2), s == s2, s != s2
+    print s.issubset(s2), s.issuperset(s2), s == s2, s != s2, s.difference(s2)

test/tests/str_functions.py

@@ -159,3 +159,6 @@ print 'ab c\n\nde fg\rkl\r\n'.splitlines()
 print 'ab c\n\nde fg\rkl\r\n'.splitlines(True)
 
 print "1".zfill(3), "+1".zfill(3), "-1".zfill(3), "0".zfill(3)
+
+it = iter("hello world")
+print list(it)

test/tests/str_subclassing.py

@@ -6,3 +6,7 @@ print repr(s)
 
 import sys
 sys.stdout.write(s)
+print
+
+print repr("hello" + MyStr("world"))
+print int(MyStr("2"))

test/tests/time_test.py

@@ -4,3 +4,7 @@ print type(time)
 time.sleep(0)
 time.sleep(False)
 time.clock()
+
+print time.timezone
+
+print long(time.mktime(time.localtime(1020)))

test/tests/unicode_test.py

@@ -141,3 +141,5 @@ class C(object):
     def __repr__(self):
         return u"hello world"
 print [C()], set([C()]), {1:C()}
+
+print "hello" + u"world"