Merge #381 from toshok:iobench

from_cpython/Include/pyconfig.h

@@ -55,6 +55,7 @@
 #define HAVE_SOCKETPAIR 1
 #define HAVE_GETPEERNAME 1
 #define HAVE_STRFTIME 1
+#define HAVE_TIMES 1
 
 #define PY_FORMAT_LONG_LONG "ll"
 #define PY_FORMAT_SIZE_T "z"

minibenchmarks/iobench.py

+# -*- coding: utf-8 -*-
+# This file should be kept compatible with both Python 2.6 and Python >= 3.0.
+
+import time
+import os
+import re
+import sys
+import hashlib
+import functools
+import itertools
+from optparse import OptionParser
+
+out = sys.stdout
+
+TEXT_ENCODING = 'utf8'
+NEWLINES = 'lf'
+
+# Compatibility
+try:
+    xrange
+except NameError:
+    xrange = range
+
+def text_open(fn, mode, encoding=None):
+    try:
+        return open(fn, mode, encoding=encoding or TEXT_ENCODING)
+    except TypeError:
+        return open(fn, mode)
+
+def get_file_sizes():
+    for s in ['20 KB', '400 KB', '10 MB']:
+        size, unit = s.split()
+        size = int(size) * {'KB': 1024, 'MB': 1024 ** 2}[unit]
+        yield s.replace(' ', ''), size
+
+def get_binary_files():
+    return ((name + ".bin", size) for name, size in get_file_sizes())
+
+def get_text_files():
+    return (("%s-%s-%s.txt" % (name, TEXT_ENCODING, NEWLINES), size)
+        for name, size in get_file_sizes())
+
+def with_open_mode(mode):
+    def decorate(f):
+        f.file_open_mode = mode
+        return f
+    return decorate
+
+def with_sizes(*sizes):
+    def decorate(f):
+        f.file_sizes = sizes
+        return f
+    return decorate
+
+
+# Here begin the tests
+
+@with_open_mode("r")
+@with_sizes("medium")
+def read_bytewise(f):
+    """ read one unit at a time """
+    f.seek(0)
+    while f.read(1):
+        pass
+
+@with_open_mode("r")
+@with_sizes("medium")
+def read_small_chunks(f):
+    """ read 20 units at a time """
+    f.seek(0)
+    while f.read(20):
+        pass
+
+@with_open_mode("r")
+@with_sizes("medium")
+def read_big_chunks(f):
+    """ read 4096 units at a time """
+    f.seek(0)
+    while f.read(4096):
+        pass
+
+@with_open_mode("r")
+@with_sizes("small", "medium", "large")
+def read_whole_file(f):
+    """ read whole contents at once """
+    f.seek(0)
+    while f.read():
+        pass
+
+@with_open_mode("rt")
+@with_sizes("medium")
+def read_lines(f):
+    """ read one line at a time """
+    f.seek(0)
+    for line in f:
+        pass
+
+@with_open_mode("r")
+@with_sizes("medium")
+def seek_forward_bytewise(f):
+    """ seek forward one unit at a time """
+    f.seek(0, 2)
+    size = f.tell()
+    f.seek(0, 0)
+    for i in xrange(0, size - 1):
+        f.seek(i, 0)
+
+@with_open_mode("r")
+@with_sizes("medium")
+def seek_forward_blockwise(f):
+    """ seek forward 1000 units at a time """
+    f.seek(0, 2)
+    size = f.tell()
+    f.seek(0, 0)
+    for i in xrange(0, size - 1, 1000):
+        f.seek(i, 0)
+
+@with_open_mode("rb")
+@with_sizes("medium")
+def read_seek_bytewise(f):
+    """ alternate read & seek one unit """
+    f.seek(0)
+    while f.read(1):
+        f.seek(1, 1)
+
+@with_open_mode("rb")
+@with_sizes("medium")
+def read_seek_blockwise(f):
+    """ alternate read & seek 1000 units """
+    f.seek(0)
+    while f.read(1000):
+        f.seek(1000, 1)
+
+
+@with_open_mode("w")
+@with_sizes("small")
+def write_bytewise(f, source):
+    """ write one unit at a time """
+    for i in xrange(0, len(source)):
+        f.write(source[i:i+1])
+
+@with_open_mode("w")
+@with_sizes("medium")
+def write_small_chunks(f, source):
+    """ write 20 units at a time """
+    for i in xrange(0, len(source), 20):
+        f.write(source[i:i+20])
+
+@with_open_mode("w")
+@with_sizes("medium")
+def write_medium_chunks(f, source):
+    """ write 4096 units at a time """
+    for i in xrange(0, len(source), 4096):
+        f.write(source[i:i+4096])
+
+@with_open_mode("w")
+@with_sizes("large")
+def write_large_chunks(f, source):
+    """ write 1e6 units at a time """
+    for i in xrange(0, len(source), 1000000):
+        f.write(source[i:i+1000000])
+
+
+@with_open_mode("w+")
+@with_sizes("small")
+def modify_bytewise(f, source):
+    """ modify one unit at a time """
+    f.seek(0)
+    for i in xrange(0, len(source)):
+        f.write(source[i:i+1])
+
+@with_open_mode("w+")
+@with_sizes("medium")
+def modify_small_chunks(f, source):
+    """ modify 20 units at a time """
+    f.seek(0)
+    for i in xrange(0, len(source), 20):
+        f.write(source[i:i+20])
+
+@with_open_mode("w+")
+@with_sizes("medium")
+def modify_medium_chunks(f, source):
+    """ modify 4096 units at a time """
+    f.seek(0)
+    for i in xrange(0, len(source), 4096):
+        f.write(source[i:i+4096])
+
+@with_open_mode("wb+")
+@with_sizes("medium")
+def modify_seek_forward_bytewise(f, source):
+    """ alternate write & seek one unit """
+    f.seek(0)
+    for i in xrange(0, len(source), 2):
+        f.write(source[i:i+1])
+        f.seek(i+2)
+
+@with_open_mode("wb+")
+@with_sizes("medium")
+def modify_seek_forward_blockwise(f, source):
+    """ alternate write & seek 1000 units """
+    f.seek(0)
+    for i in xrange(0, len(source), 2000):
+        f.write(source[i:i+1000])
+        f.seek(i+2000)
+
+# XXX the 2 following tests don't work with py3k's text IO
+@with_open_mode("wb+")
+@with_sizes("medium")
+def read_modify_bytewise(f, source):
+    """ alternate read & write one unit """
+    f.seek(0)
+    for i in xrange(0, len(source), 2):
+        f.read(1)
+        f.write(source[i+1:i+2])
+
+@with_open_mode("wb+")
+@with_sizes("medium")
+def read_modify_blockwise(f, source):
+    """ alternate read & write 1000 units """
+    f.seek(0)
+    for i in xrange(0, len(source), 2000):
+        f.read(1000)
+        f.write(source[i+1000:i+2000])
+
+
+read_tests = [
+    read_bytewise, read_small_chunks, read_lines, read_big_chunks,
+    None, read_whole_file, None,
+    seek_forward_bytewise, seek_forward_blockwise,
+    read_seek_bytewise, read_seek_blockwise,
+]
+
+write_tests = [
+    write_bytewise, write_small_chunks, write_medium_chunks, write_large_chunks,
+]
+
+modify_tests = [
+    modify_bytewise, modify_small_chunks, modify_medium_chunks,
+    None,
+    modify_seek_forward_bytewise, modify_seek_forward_blockwise,
+    read_modify_bytewise, read_modify_blockwise,
+]
+
+def run_during(duration, func):
+    _t = time.time
+    n = 0
+    start = os.times()
+    start_timestamp = _t()
+    real_start = start[4] or start_timestamp
+    while True:
+        func()
+        n += 1
+        if _t() - start_timestamp > duration:
+            break
+    end = os.times()
+    real = (end[4] if start[4] else time.time()) - real_start
+    return n, real, sum(end[0:2]) - sum(start[0:2])
+
+def warm_cache(filename):
+    with open(filename, "rb") as f:
+        f.read()
+
+
+def run_all_tests(options):
+    def print_label(filename, func):
+        name = re.split(r'[-.]', filename)[0]
+        out.write(
+            ("[%s] %s... "
+                % (name.center(7), func.__doc__.strip())
+            ).ljust(52))
+        out.flush()
+
+    def print_results(size, n, real, cpu):
+        bw = n * float(size) / 1024 ** 2 / real
+        bw = ("%4d MB/s" if bw > 100 else "%.3g MB/s") % bw
+        out.write(bw.rjust(12) + "\n")
+        if cpu < 0.90 * real:
+            out.write("   warning: test above used only %d%% CPU, "
+                "result may be flawed!\n" % (100.0 * cpu / real))
+
+    def run_one_test(name, size, open_func, test_func, *args):
+        mode = test_func.file_open_mode
+        print_label(name, test_func)
+        if "w" not in mode or "+" in mode:
+            warm_cache(name)
+        with open_func(name) as f:
+            n, real, cpu = run_during(1.5, lambda: test_func(f, *args))
+        print_results(size, n, real, cpu)
+
+    def run_test_family(tests, mode_filter, files, open_func, *make_args):
+        for test_func in tests:
+            if test_func is None:
+                out.write("\n")
+                continue
+            if mode_filter in test_func.file_open_mode:
+                continue
+            for s in test_func.file_sizes:
+                name, size = files[size_names[s]]
+                #name += file_ext
+                args = tuple(f(name, size) for f in make_args)
+                run_one_test(name, size,
+                    open_func, test_func, *args)
+
+    size_names = {
+        "small": 0,
+        "medium": 1,
+        "large": 2,
+    }
+
+    binary_files = list(get_binary_files())
+    text_files = list(get_text_files())
+    if "b" in options:
+        print("Binary unit = one byte")
+    if "t" in options:
+        print("Text unit = one character (%s-decoded)" % TEXT_ENCODING)
+
+    # Binary reads
+    if "b" in options and "r" in options:
+        print("\n** Binary input **\n")
+        run_test_family(read_tests, "t", binary_files, lambda fn: open(fn, "rb"))
+
+    # Text reads
+    if "t" in options and "r" in options:
+        print("\n** Text input **\n")
+        run_test_family(read_tests, "b", text_files, lambda fn: text_open(fn, "r"))
+
+    # Binary writes
+    if "b" in options and "w" in options:
+        print("\n** Binary append **\n")
+        def make_test_source(name, size):
+            with open(name, "rb") as f:
+                return f.read()
+        run_test_family(write_tests, "t", binary_files,
+            lambda fn: open(os.devnull, "wb"), make_test_source)
+
+    # Text writes
+    if "t" in options and "w" in options:
+        print("\n** Text append **\n")
+        def make_test_source(name, size):
+            with text_open(name, "r") as f:
+                return f.read()
+        run_test_family(write_tests, "b", text_files,
+            lambda fn: text_open(os.devnull, "w"), make_test_source)
+
+    # Binary overwrites
+    if "b" in options and "w" in options:
+        print("\n** Binary overwrite **\n")
+        def make_test_source(name, size):
+            with open(name, "rb") as f:
+                return f.read()
+        run_test_family(modify_tests, "t", binary_files,
+            lambda fn: open(fn, "r+b"), make_test_source)
+
+    # Text overwrites
+    if "t" in options and "w" in options:
+        print("\n** Text overwrite **\n")
+        def make_test_source(name, size):
+            with text_open(name, "r") as f:
+                return f.read()
+        run_test_family(modify_tests, "b", text_files,
+            lambda fn: text_open(fn, "r+"), make_test_source)
+
+
+def prepare_files():
+    print("Preparing files...")
+    # Binary files
+    for name, size in get_binary_files():
+        if os.path.isfile(name) and os.path.getsize(name) == size:
+            continue
+        with open(name, "wb") as f:
+            f.write(os.urandom(size))
+    # Text files
+    chunk = []
+    with text_open(__file__, "rU", encoding='utf8') as f:
+        for line in f:
+            if line.startswith("# <iobench text chunk marker>"):
+                break
+        else:
+            raise RuntimeError(
+                "Couldn't find chunk marker in %s !" % __file__)
+        if NEWLINES == "all":
+            it = itertools.cycle(["\n", "\r", "\r\n"])
+        else:
+            it = itertools.repeat(
+                {"cr": "\r", "lf": "\n", "crlf": "\r\n"}[NEWLINES])
+        chunk = "".join(line.replace("\n", next(it)) for line in f)
+        if isinstance(chunk, bytes):
+            chunk = chunk.decode('utf8')
+        chunk = chunk.encode(TEXT_ENCODING)
+    for name, size in get_text_files():
+        if os.path.isfile(name) and os.path.getsize(name) == size:
+            continue
+        head = chunk * (size // len(chunk))
+        tail = chunk[:size % len(chunk)]
+        # Adjust tail to end on a character boundary
+        while True:
+            try:
+                tail.decode(TEXT_ENCODING)
+                break
+            except UnicodeDecodeError:
+                tail = tail[:-1]
+        with open(name, "wb") as f:
+            f.write(head)
+            f.write(tail)
+
+def main():
+    global TEXT_ENCODING, NEWLINES
+
+    usage = "usage: %prog [-h|--help] [options]"
+    parser = OptionParser(usage=usage)
+    parser.add_option("-b", "--binary",
+                      action="store_true", dest="binary", default=False,
+                      help="run binary I/O tests")
+    parser.add_option("-t", "--text",
+                      action="store_true", dest="text", default=False,
+                      help="run text I/O tests")
+    parser.add_option("-r", "--read",
+                      action="store_true", dest="read", default=False,
+                      help="run read tests")
+    parser.add_option("-w", "--write",
+                      action="store_true", dest="write", default=False,
+                      help="run write & modify tests")
+    parser.add_option("-E", "--encoding",
+                      action="store", dest="encoding", default=None,
+                      help="encoding for text tests (default: %s)" % TEXT_ENCODING)
+    parser.add_option("-N", "--newlines",
+                      action="store", dest="newlines", default='lf',
+                      help="line endings for text tests "
+                           "(one of: {lf (default), cr, crlf, all})")
+    options, args = parser.parse_args()
+    if args:
+        parser.error("unexpected arguments")
+    NEWLINES = options.newlines.lower()
+    if NEWLINES not in ('lf', 'cr', 'crlf', 'all'):
+        parser.error("invalid 'newlines' option: %r" % NEWLINES)
+
+    test_options = ""
+    if options.read:
+        test_options += "r"
+    if options.write:
+        test_options += "w"
+    elif not options.read:
+        test_options += "rw"
+    if options.text:
+        test_options += "t"
+    if options.binary:
+        test_options += "b"
+    elif not options.text:
+        test_options += "tb"
+
+    if options.encoding:
+        TEXT_ENCODING = options.encoding
+
+    prepare_files()
+    run_all_tests(test_options)
+
+if __name__ == "__main__":
+    main()
+
+
+# -- This part to exercise text reading. Don't change anything! --
+# <iobench text chunk marker>
+
+"""
+1.
+Gáttir allar,
+áðr gangi fram,
+um skoðask skyli,
+um skyggnast skyli,
+því at óvíst er at vita,
+hvar óvinir
+sitja á fleti fyrir.
+
+2.
+Gefendr heilir!
+Gestr er inn kominn,
+hvar skal sitja sjá?
+Mjök er bráðr,
+sá er á bröndum skal
+síns of freista frama.
+
+3.
+Elds er þörf,
+þeims inn er kominn
+ok á kné kalinn;
+matar ok váða
+er manni þörf,
+þeim er hefr um fjall farit.
+
+4.
+Vatns er þörf,
+þeim er til verðar kemr,
+þerru ok þjóðlaðar,
+góðs of æðis,
+ef sér geta mætti,
+orðs ok endrþögu.
+
+5.
+Vits er þörf,
+þeim er víða ratar;
+dælt er heima hvat;
+at augabragði verðr,
+sá er ekki kann
+ok með snotrum sitr.
+
+6.
+At hyggjandi sinni
+skyli-t maðr hræsinn vera,
+heldr gætinn at geði;
+þá er horskr ok þögull
+kemr heimisgarða til,
+sjaldan verðr víti vörum,
+því at óbrigðra vin
+fær maðr aldregi
+en mannvit mikit.
+
+7.
+Inn vari gestr,
+er til verðar kemr,
+þunnu hljóði þegir,
+eyrum hlýðir,
+en augum skoðar;
+svá nýsisk fróðra hverr fyrir.
+
+8.
+Hinn er sæll,
+er sér of getr
+lof ok líknstafi;
+ódælla er við þat,
+er maðr eiga skal
+annars brjóstum í.
+"""
+
+"""
+C'est revenir tard, je le sens, sur un sujet trop rebattu et déjà presque oublié. Mon état, qui ne me permet plus aucun travail suivi, mon aversion pour le genre polémique, ont causé ma lenteur à écrire et ma répugnance à publier. J'aurais même tout à fait supprimé ces Lettres, ou plutôt je lie les aurais point écrites, s'il n'eût été question que de moi : Mais ma patrie ne m'est pas tellement devenue étrangère que je puisse voir tranquillement opprimer ses citoyens, surtout lorsqu'ils n'ont compromis leurs droits qu'en défendant ma cause. Je serais le dernier des hommes si dans une telle occasion j'écoutais un sentiment qui n'est plus ni douceur ni patience, mais faiblesse et lâcheté, dans celui qu'il empêche de remplir son devoir.
+Rien de moins important pour le public, j'en conviens, que la matière de ces lettres. La constitution d'une petite République, le sort d'un petit particulier, l'exposé de quelques injustices, la réfutation de quelques sophismes ; tout cela n'a rien en soi d'assez considérable pour mériter beaucoup de lecteurs : mais si mes sujets sont petits mes objets sont grands, et dignes de l'attention de tout honnête homme. Laissons Genève à sa place, et Rousseau dans sa dépression ; mais la religion, mais la liberté, la justice ! voilà, qui que vous soyez, ce qui n'est pas au-dessous de vous.
+Qu'on ne cherche pas même ici dans le style le dédommagement de l'aridité de la matière. Ceux que quelques traits heureux de ma plume ont si fort irrités trouveront de quoi s'apaiser dans ces lettres, L'honneur de défendre un opprimé eût enflammé mon coeur si j'avais parlé pour un autre. Réduit au triste emploi de me défendre moi-même, j'ai dû me borner à raisonner ; m'échauffer eût été m'avilir. J'aurai donc trouvé grâce en ce point devant ceux qui s'imaginent qu'il est essentiel à la vérité d'être dite froidement ; opinion que pourtant j'ai peine à comprendre. Lorsqu'une vive persuasion nous anime, le moyen d'employer un langage glacé ? Quand Archimède tout transporté courait nu dans les rues de Syracuse, en avait-il moins trouvé la vérité parce qu'il se passionnait pour elle ? Tout au contraire, celui qui la sent ne peut s'abstenir de l'adorer ; celui qui demeure froid ne l'a pas vue.
+Quoi qu'il en soit, je prie les lecteurs de vouloir bien mettre à part mon beau style, et d'examiner seulement si je raisonne bien ou mal ; car enfin, de cela seul qu'un auteur s'exprime en bons termes, je ne vois pas comment il peut s'ensuivre que cet auteur ne sait ce qu'il dit.
+"""

src/capi/abstract.cpp

@@ -1341,6 +1341,47 @@ extern "C" int PyNumber_CoerceEx(PyObject**, PyObject**) noexcept {
     Py_FatalError("unimplemented");
 }
 
+extern "C" PyObject* _PyNumber_ConvertIntegralToInt(PyObject* integral, const char* error_format) noexcept {
+    const char* type_name;
+    static PyObject* int_name = NULL;
+    if (int_name == NULL) {
+        int_name = PyString_InternFromString("__int__");
+        if (int_name == NULL)
+            return NULL;
+    }
+
+    if (integral && (!PyInt_Check(integral) && !PyLong_Check(integral))) {
+        /* Don't go through tp_as_number->nb_int to avoid
+           hitting the classic class fallback to __trunc__. */
+        PyObject* int_func = PyObject_GetAttr(integral, int_name);
+        if (int_func == NULL) {
+            PyErr_Clear(); /* Raise a different error. */
+            goto non_integral_error;
+        }
+        Py_DECREF(integral);
+        integral = PyEval_CallObject(int_func, NULL);
+        Py_DECREF(int_func);
+        if (integral && (!PyInt_Check(integral) && !PyLong_Check(integral))) {
+            goto non_integral_error;
+        }
+    }
+    return integral;
+
+non_integral_error:
+    if (PyInstance_Check(integral)) {
+        Py_FatalError("unimplemented");
+        /* cpython has this:
+        type_name = PyString_AS_STRING(((PyInstanceObject *)integral)
+                                       ->in_class->cl_name);
+        */
+    } else {
+        type_name = integral->cls->tp_name;
+    }
+    PyErr_Format(PyExc_TypeError, error_format, type_name);
+    Py_DECREF(integral);
+    return NULL;
+}
+
 extern "C" PyObject* PyNumber_Int(PyObject* o) noexcept {
     PyNumberMethods* m;
     const char* buffer;
@@ -1370,21 +1411,20 @@ extern "C" PyObject* PyNumber_Int(PyObject* o) noexcept {
         return PyInt_FromLong(io->n);
     }
 
-    Py_FatalError("unimplemented __trunc__ and string -> int conversion");
-// the remainder of PyNumber_Int deals with __trunc__ usage, and converting from unicode/string to int
-#if 0
     PyObject* trunc_func = getattr(o, "__trunc__");
     if (trunc_func) {
-      PyObject *truncated = PyEval_CallObject(trunc_func, NULL);
-      Py_DECREF(trunc_func);
-      /* __trunc__ is specified to return an Integral type, but
-     int() needs to return an int. */
-      return _PyNumber_ConvertIntegralToInt(
-                      truncated,
-                      "__trunc__ returned non-Integral (type %.200s)");
+        PyObject* truncated = PyEval_CallObject(trunc_func, NULL);
+        Py_DECREF(trunc_func);
+        /* __trunc__ is specified to return an Integral type, but
+       int() needs to return an int. */
+
+        return _PyNumber_ConvertIntegralToInt(truncated, "__trunc__ returned non-Integral (type %.200s)");
     }
-    PyErr_Clear();  /* It's not an error if  o.__trunc__ doesn't exist. */
+    PyErr_Clear(); /* It's not an error if  o.__trunc__ doesn't exist. */
 
+    // the remainder of PyNumber_Int deals with converting from unicode/string to int
+    Py_FatalError("unimplemented string -> int conversion");
+#if 0
     if (PyString_Check(o))
       return int_from_string(PyString_AS_STRING(o),
                  PyString_GET_SIZE(o));

src/capi/modsupport.cpp

@@ -124,6 +124,9 @@ static PyObject* do_mkvalue(const char** p_format, va_list* p_va, int flags) noe
             case 'l':
                 return PyInt_FromLong(va_arg(*p_va, long));
 
+            case 'd':
+                return PyFloat_FromDouble(va_arg(*p_va, double));
+
             case 'N':
             case 'S':
             case 'O':
@@ -380,7 +383,7 @@ extern "C" PyObject* Py_BuildValue(const char* fmt, ...) noexcept {
 
 extern "C" PyObject* Py_InitModule4(const char* name, PyMethodDef* methods, const char* doc, PyObject* self,
                                     int apiver) noexcept {
-    BoxedModule* module = createModule(name, "__builtin__");
+    BoxedModule* module = createModule(name, "__builtin__", doc);
 
     // Pass self as is, even if NULL we are not allowed to change it to None
     Box* passthrough = static_cast<Box*>(self);
@@ -394,10 +397,6 @@ extern "C" PyObject* Py_InitModule4(const char* name, PyMethodDef* methods, cons
         methods++;
     }
 
-    if (doc) {
-        module->setattr("__doc__", boxStrConstant(doc), NULL);
-    }
-
     return module;
 }
 

src/codegen/ast_interpreter.cpp

@@ -675,7 +675,6 @@ Box* ASTInterpreter::createFunction(AST* node, AST_arguments* args, const std::v
         }
         assert(closure);
     }
-
     return boxCLFunction(cl, closure, is_generator, u.il);
 }
 

src/codegen/irgen.cpp

@@ -413,6 +413,8 @@ static void emitBBs(IRGenState* irstate, TypeAnalysis* types, const OSREntryDesc
                     ptr = entry_emitter->getBuilder()->CreateBitCast(ptr, g.double_->getPointerTo());
                 } else if (p.second == GENERATOR) {
                     ptr = entry_emitter->getBuilder()->CreateBitCast(ptr, g.llvm_generator_type_ptr->getPointerTo());
+                } else if (p.second == CLOSURE) {
+                    ptr = entry_emitter->getBuilder()->CreateBitCast(ptr, g.llvm_closure_type_ptr->getPointerTo());
                 } else {
                     assert(p.second->llvmType() == g.llvm_value_type_ptr);
                 }

src/codegen/irgen/hooks.cpp

@@ -101,6 +101,17 @@ const std::string SourceInfo::getName() {
     }
 }
 
+Box* SourceInfo::getDocString() {
+    AST_Str* first_str = NULL;
+
+    if (body.size() > 0 && body[0]->type == AST_TYPE::Expr
+        && static_cast<AST_Expr*>(body[0])->value->type == AST_TYPE::Str) {
+        return boxString(static_cast<AST_Str*>(static_cast<AST_Expr*>(body[0])->value)->str_data);
+    }
+
+    return None;
+}
+
 ScopeInfo* SourceInfo::getScopeInfo() {
     return scoping->getScopeInfoForNode(ast);
 }
@@ -298,6 +309,8 @@ void compileAndRunModule(AST_Module* m, BoxedModule* bm) {
         SourceInfo* si = new SourceInfo(bm, scoping, m, m->body);
         CLFunction* cl_f = new CLFunction(0, 0, false, false, si);
 
+        bm->setattr("__doc__", si->getDocString(), NULL);
+
         EffortLevel effort = initialEffort();
 
         cf = compileFunction(cl_f, new FunctionSpecialization(VOID), effort, NULL);

src/core/types.h

@@ -259,6 +259,8 @@ public:
     const std::string getName();
     InternedString mangleName(InternedString id);
 
+    Box* getDocString();
+
     SourceInfo(BoxedModule* m, ScopingAnalysis* scoping, AST* ast, const std::vector<AST_stmt*>& body);
 };
 
@@ -514,7 +516,7 @@ class BoxedClass;
 void setupRuntime();
 void teardownRuntime();
 BoxedModule* createAndRunModule(const std::string& name, const std::string& fn);
-BoxedModule* createModule(const std::string& name, const std::string& fn);
+BoxedModule* createModule(const std::string& name, const std::string& fn, const char* doc = NULL);
 
 // TODO where to put this
 void appendToSysPath(const std::string& path);

src/runtime/builtin_modules/builtins.cpp

@@ -1019,7 +1019,9 @@ Box* builtinCmp(Box* lhs, Box* rhs) {
 }
 
 void setupBuiltins() {
-    builtins_module = createModule("__builtin__", "__builtin__");
+    builtins_module = createModule("__builtin__", "__builtin__",
+                                   "Built-in functions, exceptions, and other objects.\n\nNoteworthy: None is "
+                                   "the `nil' object; Ellipsis represents `...' in slices.");
 
     BoxedHeapClass* ellipsis_cls
         = BoxedHeapClass::create(type_cls, object_cls, NULL, 0, 0, sizeof(Box), false, "ellipsis");

src/runtime/float.cpp

@@ -652,6 +652,36 @@ Box* floatRepr(BoxedFloat* self) {
     return boxString(floatFmt(self->d, 16, 'g'));
 }
 
+Box* floatTrunc(BoxedFloat* self) {
+    if (!isSubclass(self->cls, float_cls))
+        raiseExcHelper(TypeError, "descriptor '__trunc__' requires a 'float' object but received a '%s'",
+                       getTypeName(self));
+
+    double wholepart; /* integral portion of x, rounded toward 0 */
+
+    (void)modf(self->d, &wholepart);
+    /* Try to get out cheap if this fits in a Python int.  The attempt
+     * to cast to long must be protected, as C doesn't define what
+     * happens if the double is too big to fit in a long.  Some rare
+     * systems raise an exception then (RISCOS was mentioned as one,
+     * and someone using a non-default option on Sun also bumped into
+     * that).  Note that checking for <= LONG_MAX is unsafe: if a long
+     * has more bits of precision than a double, casting LONG_MAX to
+     * double may yield an approximation, and if that's rounded up,
+     * then, e.g., wholepart=LONG_MAX+1 would yield true from the C
+     * expression wholepart<=LONG_MAX, despite that wholepart is
+     * actually greater than LONG_MAX.  However, assuming a two's complement
+     * machine with no trap representation, LONG_MIN will be a power of 2 (and
+     * hence exactly representable as a double), and LONG_MAX = -1-LONG_MIN, so
+     * the comparisons with (double)LONG_MIN below should be safe.
+     */
+    if ((double)LONG_MIN <= wholepart && wholepart < -(double)LONG_MIN) {
+        const long aslong = (long)wholepart;
+        return PyInt_FromLong(aslong);
+    }
+    return PyLong_FromDouble(wholepart);
+}
+
 extern "C" void printFloat(double d) {
     std::string s = floatFmt(d, 12, 'g');
     printf("%s", s.c_str());
@@ -711,6 +741,9 @@ void setupFloat() {
     // float_cls->giveAttr("__nonzero__", new BoxedFunction(boxRTFunction((void*)floatNonzero, NULL, 1)));
     float_cls->giveAttr("__str__", new BoxedFunction(boxRTFunction((void*)floatStr, STR, 1)));
     float_cls->giveAttr("__repr__", new BoxedFunction(boxRTFunction((void*)floatRepr, STR, 1)));
+
+    float_cls->giveAttr("__trunc__", new BoxedFunction(boxRTFunction((void*)floatTrunc, BOXED_INT, 1)));
+
     float_cls->freeze();
 }
 

src/runtime/import.cpp

@@ -685,7 +685,9 @@ Box* impLoadModule(Box* _name, Box* _file, Box* _pathname, Box** args) {
 }
 
 void setupImport() {
-    BoxedModule* imp_module = createModule("imp", "__builtin__");
+    BoxedModule* imp_module
+        = createModule("imp", "__builtin__", "'This module provides the components needed to build your own\n"
+                                             "__import__ function.  Undocumented functions are obsolete.'");
 
     imp_module->giveAttr("PY_SOURCE", boxInt(SearchResult::PY_SOURCE));
     imp_module->giveAttr("PY_COMPILED", boxInt(SearchResult::PY_COMPILED));

src/runtime/int.cpp

@@ -916,6 +916,14 @@ extern "C" Box* intOct(BoxedInt* self) {
     return new BoxedString(std::string(buf, len));
 }
 
+extern "C" Box* intTrunc(BoxedInt* self) {
+    if (!isSubclass(self->cls, int_cls))
+        raiseExcHelper(TypeError, "descriptor '__trunc__' requires a 'int' object but received a '%s'",
+                       getTypeName(self));
+
+    return self;
+}
+
 static Box* _intNew(Box* val, Box* base) {
     if (isSubclass(val->cls, int_cls)) {
         RELEASE_ASSERT(!base, "");
@@ -1072,6 +1080,8 @@ void setupInt() {
     int_cls->giveAttr("__hex__", new BoxedFunction(boxRTFunction((void*)intHex, STR, 1)));
     int_cls->giveAttr("__oct__", new BoxedFunction(boxRTFunction((void*)intOct, STR, 1)));
 
+    int_cls->giveAttr("__trunc__", new BoxedFunction(boxRTFunction((void*)intTrunc, BOXED_INT, 1)));
+
     int_cls->giveAttr(
         "__new__", new BoxedFunction(boxRTFunction((void*)intNew, UNKNOWN, 3, 2, false, false), { boxInt(0), NULL }));
 

src/runtime/long.cpp

@@ -1035,6 +1035,14 @@ Box* longHash(BoxedLong* self) {
     return boxInt(n);
 }
 
+extern "C" Box* longTrunc(BoxedLong* self) {
+    if (!isSubclass(self->cls, long_cls))
+        raiseExcHelper(TypeError, "descriptor '__trunc__' requires a 'long' object but received a '%s'",
+                       getTypeName(self));
+
+    return self;
+}
+
 void* customised_allocation(size_t alloc_size) {
     return gc::gc_alloc(alloc_size, gc::GCKind::CONSERVATIVE);
 }
@@ -1097,6 +1105,8 @@ void setupLong() {
     long_cls->giveAttr("__nonzero__", new BoxedFunction(boxRTFunction((void*)longNonzero, BOXED_BOOL, 1)));
     long_cls->giveAttr("__hash__", new BoxedFunction(boxRTFunction((void*)longHash, BOXED_INT, 1)));
 
+    long_cls->giveAttr("__trunc__", new BoxedFunction(boxRTFunction((void*)longTrunc, UNKNOWN, 1)));
+
     long_cls->freeze();
 }
 }

src/runtime/types.cpp

@@ -263,22 +263,22 @@ std::string builtinStr("__builtin__");
 
 extern "C" BoxedFunctionBase::BoxedFunctionBase(CLFunction* f)
     : in_weakreflist(NULL), f(f), closure(NULL), isGenerator(false), ndefaults(0), defaults(NULL), modname(NULL),
-      name(NULL) {
+      name(NULL), doc(NULL) {
     if (f->source) {
         this->modname = f->source->parent_module->getattr("__name__", NULL);
+        this->doc = f->source->getDocString();
     } else {
         this->modname = boxStringPtr(&builtinStr);
+        this->doc = None;
     }
 
-    this->giveAttr("__doc__", None);
-
     assert(f->num_defaults == ndefaults);
 }
 
 extern "C" BoxedFunctionBase::BoxedFunctionBase(CLFunction* f, std::initializer_list<Box*> defaults,
                                                 BoxedClosure* closure, bool isGenerator)
     : in_weakreflist(NULL), f(f), closure(closure), isGenerator(isGenerator), ndefaults(0), defaults(NULL),
-      modname(NULL), name(NULL) {
+      modname(NULL), name(NULL), doc(NULL) {
     if (defaults.size()) {
         // make sure to initialize defaults first, since the GC behavior is triggered by ndefaults,
         // and a GC can happen within this constructor:
@@ -289,8 +289,10 @@ extern "C" BoxedFunctionBase::BoxedFunctionBase(CLFunction* f, std::initializer_
 
     if (f->source) {
         this->modname = f->source->parent_module->getattr("__name__", NULL);
+        this->doc = f->source->getDocString();
     } else {
         this->modname = boxStringPtr(&builtinStr);
+        this->doc = None;
     }
 
     assert(f->num_defaults == ndefaults);
@@ -309,21 +311,22 @@ BoxedFunction::BoxedFunction(CLFunction* f, std::initializer_list<Box*> defaults
     if (f->source) {
         this->name = static_cast<BoxedString*>(boxString(f->source->getName()));
     }
-
-    this->giveAttr("__doc__", None);
 }
 
-BoxedBuiltinFunctionOrMethod::BoxedBuiltinFunctionOrMethod(CLFunction* f, const char* name)
+BoxedBuiltinFunctionOrMethod::BoxedBuiltinFunctionOrMethod(CLFunction* f, const char* name, const char* doc)
     : BoxedBuiltinFunctionOrMethod(f, name, {}) {
+
+    this->doc = doc ? boxStrConstant(doc) : None;
 }
 
 BoxedBuiltinFunctionOrMethod::BoxedBuiltinFunctionOrMethod(CLFunction* f, const char* name,
                                                            std::initializer_list<Box*> defaults, BoxedClosure* closure,
-                                                           bool isGenerator)
+                                                           bool isGenerator, const char* doc)
     : BoxedFunctionBase(f, defaults, closure, isGenerator) {
 
     assert(name);
     this->name = static_cast<BoxedString*>(boxString(name));
+    this->doc = doc ? boxStrConstant(doc) : None;
 }
 
 extern "C" void functionGCHandler(GCVisitor* v, Box* b) {
@@ -338,6 +341,9 @@ extern "C" void functionGCHandler(GCVisitor* v, Box* b) {
     if (f->modname)
         v->visit(f->modname);
 
+    if (f->doc)
+        v->visit(f->doc);
+
     if (f->closure)
         v->visit(f->closure);
 
@@ -360,9 +366,10 @@ static void functionDtor(Box* b) {
     self->dependent_ics.~ICInvalidator();
 }
 
-BoxedModule::BoxedModule(const std::string& name, const std::string& fn) : fn(fn) {
+BoxedModule::BoxedModule(const std::string& name, const std::string& fn, const char* doc) : fn(fn) {
     this->giveAttr("__name__", boxString(name));
     this->giveAttr("__file__", boxString(fn));
+    this->giveAttr("__doc__", doc ? boxStrConstant(doc) : None);
 }
 
 std::string BoxedModule::name() {
@@ -929,6 +936,7 @@ static void typeSetModule(Box* _type, PyObject* value, void* context) {
     type->setattr("__module__", value, NULL);
 }
 
+
 Box* typeHash(BoxedClass* self) {
     assert(isSubclass(self->cls, type_cls));
 
@@ -1893,6 +1901,8 @@ void setupRuntime() {
     function_cls->giveAttr("__repr__", new BoxedFunction(boxRTFunction((void*)functionRepr, STR, 1)));
     function_cls->giveAttr("__module__", new BoxedMemberDescriptor(BoxedMemberDescriptor::OBJECT,
                                                                    offsetof(BoxedFunction, modname), false));
+    function_cls->giveAttr(
+        "__doc__", new BoxedMemberDescriptor(BoxedMemberDescriptor::OBJECT, offsetof(BoxedFunction, doc), false));
     function_cls->giveAttr("__get__", new BoxedFunction(boxRTFunction((void*)functionGet, UNKNOWN, 3)));
     function_cls->giveAttr("__call__",
                            new BoxedFunction(boxRTFunction((void*)functionCall, UNKNOWN, 1, 0, true, true)));
@@ -1906,6 +1916,9 @@ void setupRuntime() {
         "__repr__", new BoxedFunction(boxRTFunction((void*)builtinFunctionOrMethodRepr, STR, 1)));
     builtin_function_or_method_cls->giveAttr(
         "__name__", new (pyston_getset_cls) BoxedGetsetDescriptor(builtinFunctionOrMethodName, NULL, NULL));
+    builtin_function_or_method_cls->giveAttr(
+        "__doc__",
+        new BoxedMemberDescriptor(BoxedMemberDescriptor::OBJECT, offsetof(BoxedBuiltinFunctionOrMethod, doc), false));
     builtin_function_or_method_cls->freeze();
 
     instancemethod_cls->giveAttr(
@@ -2036,16 +2049,15 @@ void setupRuntime() {
     TRACK_ALLOCATIONS = true;
 }
 
-BoxedModule* createModule(const std::string& name, const std::string& fn) {
+BoxedModule* createModule(const std::string& name, const std::string& fn, const char* doc) {
     assert(fn.size() && "probably wanted to set the fn to <stdin>?");
-    BoxedModule* module = new BoxedModule(name, fn);
+    BoxedModule* module = new BoxedModule(name, fn, doc);
 
     BoxedDict* d = getSysModulesDict();
     Box* b_name = boxStringPtr(&name);
     ASSERT(d->d.count(b_name) == 0, "%s", name.c_str());
     d->d[b_name] = module;
 
-    module->giveAttr("__doc__", None);
     return module;
 }
 

src/runtime/types.h

@@ -447,6 +447,7 @@ public:
     // Accessed via member descriptor
     Box* modname;      // __module__
     BoxedString* name; // __name__ (should be here or in one of the derived classes?)
+    Box* doc;          // __doc__
 
     BoxedFunctionBase(CLFunction* f);
     BoxedFunctionBase(CLFunction* f, std::initializer_list<Box*> defaults, BoxedClosure* closure = NULL,
@@ -466,9 +467,9 @@ public:
 
 class BoxedBuiltinFunctionOrMethod : public BoxedFunctionBase {
 public:
-    BoxedBuiltinFunctionOrMethod(CLFunction* f, const char* name);
+    BoxedBuiltinFunctionOrMethod(CLFunction* f, const char* name, const char* doc = NULL);
     BoxedBuiltinFunctionOrMethod(CLFunction* f, const char* name, std::initializer_list<Box*> defaults,
-                                 BoxedClosure* closure = NULL, bool isGenerator = false);
+                                 BoxedClosure* closure = NULL, bool isGenerator = false, const char* doc = NULL);
 
     DEFAULT_CLASS(builtin_function_or_method_cls);
 };
@@ -481,7 +482,7 @@ public:
     std::string fn;
     FutureFlags future_flags;
 
-    BoxedModule(const std::string& name, const std::string& fn);
+    BoxedModule(const std::string& name, const std::string& fn, const char* doc = NULL);
     std::string name();
 
     DEFAULT_CLASS(module_cls);

test/tests/docstrings.py

@@ -2,6 +2,19 @@ print __doc__
 __doc__ = "module_doc"
 print __doc__
 
+import test_package
+print test_package.__doc__
+test_package.__doc__ = "changeable module docs"
+print test_package.__doc__
+
+def foo():
+    """ foo docs go here """
+    pass
+
+print foo.__doc__
+foo.__doc__ = "no they don't"
+print foo.__doc__
+
 class C1(object):
     print 1, __doc__
     "hello world"

test/tests/test_package/__init__.py

+"""
+  module docs go here
+"""

test/tests/trunc_test.py

+import math
+
+def type_trunc(type, arg):
+    try:
+        print type.__trunc__(arg)
+    except TypeError as e:
+        print e
+
+type_trunc(float, 5.25)
+type_trunc(float, 5)
+type_trunc(float, 5L)
+type_trunc(float, "5")
+
+type_trunc(int, 5.25)
+type_trunc(int, 5)
+type_trunc(int, 5L)
+type_trunc(int, "5")
+
+type_trunc(long, 5.25)
+type_trunc(long, 5)
+type_trunc(long, 5L)
+type_trunc(long, "5")
+
+class Test:
+    def __trunc__(self):
+        print "made it"
+        return 5
+
+t = Test()
+print math.trunc(t)
+
+class TruncReturnsNonInt:
+    def __trunc__(self):
+        print "made it"
+        return "hi"
+
+t2 = TruncReturnsNonInt()
+print math.trunc(t2)