Merge pull request #1146 from kmod/refcounting

fix bjit crash with large dicts

from_cpython/Lib/test/test_SimpleHTTPServer.py

-# expected: reffail
 """
 These tests only check url parsing for now.
 We don't want to require the 'network' resource.

from_cpython/Lib/test/test_array.py

 # expected: reffail
-# - fails in interpreter due to keeping refcounts a bit too long
+# - fails in interpreter due to keeping refcounts a bit too long (does manual sys.getrefcount checking)
 """Test the arraymodule.
    Roger E. Masse
 """

from_cpython/Lib/test/test_docxmlrpc.py

 # expected: reffail
+# - negative ref
 from DocXMLRPCServer import DocXMLRPCServer
 import httplib
 import sys

from_cpython/Lib/test/test_file2k.py

 # expected: reffail
+# - negative ref
 import sys
 import os
 import unittest

from_cpython/Lib/test/test_file_eintr.py

-# expected: reffail
 # Written to test interrupted system calls interfering with our many buffered
 # IO implementations.  http://bugs.python.org/issue12268
 #

from_cpython/Lib/test/test_fileinput.py

-# expected: reffail
 '''
 Tests for fileinput module.
 Nick Mathewson

from_cpython/Lib/test/test_fileio.py

-# expected: reffail
 # Adapted from test_file.py by Daniel Stutzbach
 
 from __future__ import unicode_literals

from_cpython/Lib/test/test_ftplib.py

-# expected: reffail
 """Test script for ftplib module."""
 
 # Modified by Giampaolo Rodola' to test FTP class, IPv6 and TLS

from_cpython/Lib/test/test_gzip.py

-# expected: reffail
 """Test script for the gzip module.
 """
 

from_cpython/Lib/test/test_index.py

 # expected: reffail
+# - leaked refs
 import unittest
 from test import test_support
 import operator

from_cpython/Lib/test/test_logging.py

 # expected: reffail
+# - negative ref
 # Copyright 2001-2013 by Vinay Sajip. All Rights Reserved.
 #
 # Permission to use, copy, modify, and distribute this software and its

from_cpython/Lib/test/test_memoryio.py

 # expected: reffail
+# - Some sort of segfault
 """Unit tests for memory-based file-like objects.
 StringIO -- for unicode strings
 BytesIO -- for bytes

from_cpython/Lib/test/test_mimetypes.py

-# expected: reffail
 import mimetypes
 import StringIO
 import unittest

from_cpython/Lib/test/test_minidom.py

 # expected: reffail
+# - "pyexpat.xmlparser needs a tp_traverse"
 # test for xml.dom.minidom
 
 import pickle

from_cpython/Lib/test/test_parser.py

 # expected: reffail
+# - leaked refs
 import parser
 import unittest
 import sys

from_cpython/Lib/test/test_plistlib.py

 # expected: reffail
+# - negative ref
 # Copyright (C) 2003 Python Software Foundation
 
 import unittest

from_cpython/Lib/test/test_posix.py

 # expected: reffail
+# - leaked refs
 "Test posix functions"
 
 from test import test_support

from_cpython/Lib/test/test_sax.py

 # expected: reffail
+# - "pyexpat.xmlparser needs a tp_traverse"
 # regression test for SAX 2.0            -*- coding: utf-8 -*-
 # $Id$
 

from_cpython/Lib/test/test_shutil.py

 # expected: reffail
+# - leaked refs, some test failures?
 # Copyright (C) 2003 Python Software Foundation
 
 import unittest

from_cpython/Lib/test/test_signal.py

-# expected: reffail
 import unittest
 from test import test_support
 from contextlib import closing

from_cpython/Lib/test/test_ssl.py

-# expected: reffail
 # Test the support for SSL and sockets
 
 import sys

from_cpython/Lib/test/test_string.py

 # expected: reffail
+# - unknown segfault
 import unittest, string
 from test import test_support, string_tests
 from UserList import UserList

from_cpython/Lib/test/test_structseq.py

 # expected: reffail
+# - unknown segfault
 import unittest
 from test import test_support
 

from_cpython/Lib/test/test_telnetlib.py

 # expected: reffail
+# - unknown unwinding issue
 import socket
 import telnetlib
 import time

from_cpython/Lib/test/test_threaded_import.py

-# expected: fail
 # This is a variant of the very old (early 90's) file
 # Demo/threads/bug.py.  It simply provokes a number of threads into
 # trying to import the same module "at the same time".

from_cpython/Lib/test/test_tuple.py

 # expected: reffail
+# - unknown unwinding issue
 from test import test_support, seq_tests
 
 import gc

from_cpython/Lib/test/test_univnewlines.py

-# expected: fail
 # Tests universal newline support for both reading and parsing files.
 
 # NOTE: this file tests the new `io` library backported from Python 3.x.

from_cpython/Lib/test/test_unpack.py

 # expected: reffail
+# - unknown segfault
 doctests = """
 
 Unpack tuple

from_cpython/Lib/test/test_urllib2_localnet.py

-# expected: reffail
 import urlparse
 import urllib2
 import BaseHTTPServer

src/codegen/ast_interpreter.cpp

@@ -1583,25 +1583,19 @@ Value ASTInterpreter::visit_lambda(AST_Lambda* node) {
 Value ASTInterpreter::visit_dict(AST_Dict* node) {
     RELEASE_ASSERT(node->keys.size() == node->values.size(), "not implemented");
 
-    llvm::SmallVector<RewriterVar*, 8> keys;
-    llvm::SmallVector<RewriterVar*, 8> values;
-
     BoxedDict* dict = new BoxedDict();
+    RewriterVar* r_dict = jit ? jit->emitCreateDict() : NULL;
     for (size_t i = 0; i < node->keys.size(); ++i) {
         Value v = visit_expr(node->values[i]);
-        AUTO_DECREF(v.o);
         Value k = visit_expr(node->keys[i]);
-        AUTO_DECREF(k.o);
-
-        int ret = PyDict_SetItem(dict, k.o, v.o);
-        if (ret == -1)
-            throwCAPIException();
 
-        values.push_back(v);
-        keys.push_back(k);
+        dictSetInternal(dict, k.o, v.o);
+        if (jit) {
+            jit->emitDictSet(r_dict, k, v);
+        }
     }
 
-    return Value(dict, jit ? jit->emitCreateDict(keys, values) : NULL);
+    return Value(dict, r_dict);
 }
 
 Value ASTInterpreter::visit_set(AST_Set* node) {

src/codegen/baseline_jit.cpp

@@ -249,25 +249,14 @@ RewriterVar* JitFragmentWriter::emitCompare(AST_expr* node, RewriterVar* lhs, Re
     return emitPPCall((void*)compare, { lhs, rhs, imm(op_type) }, 2, 240, node).first->setType(RefType::OWNED);
 }
 
-RewriterVar* JitFragmentWriter::emitCreateDict(const llvm::ArrayRef<RewriterVar*> keys,
-                                               const llvm::ArrayRef<RewriterVar*> values) {
-    assert(keys.size() == values.size());
-    if (keys.empty())
-        return call(false, (void*)createDict)->setType(RefType::OWNED);
-    RewriterVar::SmallVector additional_uses;
-    additional_uses.insert(additional_uses.end(), keys.begin(), keys.end());
-    additional_uses.insert(additional_uses.end(), values.begin(), values.end());
-    auto rtn = emitCallWithAllocatedArgs((void*)createDictHelper,
-                                         { imm(keys.size()), allocArgs(keys, RewriterVar::SetattrType::REFUSED),
-                                           allocArgs(values, RewriterVar::SetattrType::REFUSED) },
-                                         additional_uses)->setType(RefType::OWNED);
-    for (RewriterVar* k : keys) {
-        k->refConsumed();
-    }
-    for (RewriterVar* v : values) {
-        v->refConsumed();
-    }
-    return rtn;
+RewriterVar* JitFragmentWriter::emitCreateDict() {
+    return call(false, (void*)createDict)->setType(RefType::OWNED);
+}
+
+void JitFragmentWriter::emitDictSet(RewriterVar* dict, RewriterVar* k, RewriterVar* v) {
+    call(true, (void*)dictSetInternal, dict, k, v);
+    k->refConsumed();
+    v->refConsumed();
 }
 
 // TODO: merge this function's functionality with refUsed

src/codegen/baseline_jit.h

@@ -220,7 +220,8 @@ public:
     RewriterVar* emitCallattr(AST_expr* node, RewriterVar* obj, BoxedString* attr, CallattrFlags flags,
                               const llvm::ArrayRef<RewriterVar*> args, std::vector<BoxedString*>* keyword_names);
     RewriterVar* emitCompare(AST_expr* node, RewriterVar* lhs, RewriterVar* rhs, int op_type);
-    RewriterVar* emitCreateDict(const llvm::ArrayRef<RewriterVar*> keys, const llvm::ArrayRef<RewriterVar*> values);
+    RewriterVar* emitCreateDict();
+    void emitDictSet(RewriterVar* dict, RewriterVar* k, RewriterVar* v);
     RewriterVar* emitCreateList(const llvm::ArrayRef<STOLEN(RewriterVar*)> values);
     RewriterVar* emitCreateSet(const llvm::ArrayRef<RewriterVar*> values);
     RewriterVar* emitCreateSlice(RewriterVar* start, RewriterVar* stop, RewriterVar* step);

src/codegen/runtime_hooks.h

@@ -43,7 +43,7 @@ struct GlobalFuncs {
     llvm::Value* unpackIntoArray, *raiseAttributeError, *raiseAttributeErrorStr, *raiseAttributeErrorCapi,
         *raiseAttributeErrorStrCapi, *raiseNotIterableError, *raiseIndexErrorStr, *raiseIndexErrorStrCapi,
         *assertNameDefined, *assertFail, *assertFailDerefNameDefined, *printExprHelper, *printHelper;
-    llvm::Value* listAppendInternal;
+    llvm::Value* listAppendInternal, *dictSetInternal;
     ExceptionSwitchable<llvm::Value*> runtimeCall0, runtimeCall1, runtimeCall2, runtimeCall3, runtimeCall, runtimeCallN;
     ExceptionSwitchable<llvm::Value*> callattr0, callattr1, callattr2, callattr3, callattr, callattrN;
     llvm::Value* reoptCompiledFunc, *compilePartialFunc;

src/runtime/dict.cpp

@@ -52,6 +52,14 @@ static void _dictSet(BoxedDict* self, BoxAndHash k, Box* v) {
     _dictSetStolen(self, k, incref(v));
 }
 
+extern "C" void dictSetInternal(Box* self, STOLEN(Box*) k, STOLEN(Box*) v) {
+    assert(self->cls == dict_cls);
+    AUTO_DECREF(v);
+    AUTO_DECREF(k);
+
+    _dictSet(static_cast<BoxedDict*>(self), k, v);
+}
+
 Box* dictRepr(BoxedDict* self) {
     std::vector<char> chars;
     int status = Py_ReprEnter((PyObject*)self);

src/runtime/inline/link_forcer.cpp

@@ -121,6 +121,7 @@ void force() {
     FORCE(printHelper);
 
     FORCE(listAppendInternal);
+    FORCE(dictSetInternal);
 
     FORCE(runtimeCall);
     FORCE(runtimeCallCapi);

src/runtime/objmodel.cpp

@@ -1861,7 +1861,7 @@ Box* nondataDescriptorInstanceSpecialCases(GetattrRewriteArgs* rewrite_args, Box
                 if (rewrite_args) {
                     r_descr->addAttrGuard(offsetof(BoxedInstanceMethod, obj), 0, /* negate */ false);
                     r_im_self = rewrite_args->obj;
-                    r_im_func = r_descr->getAttr(offsetof(BoxedInstanceMethod, func));
+                    r_im_func = r_descr->getAttr(offsetof(BoxedInstanceMethod, func))->setType(RefType::BORROWED);
                 }
             }
         } else {

src/runtime/objmodel.h

@@ -94,6 +94,7 @@ extern "C" Box** unpackIntoArray(Box* obj, int64_t expected_size, Box** out_keep
 extern "C" void assertNameDefined(bool b, const char* name, BoxedClass* exc_cls, bool local_var_msg);
 extern "C" void assertFailDerefNameDefined(const char* name);
 extern "C" void assertFail(Box* assertion_type, Box* msg);
+extern "C" void dictSetInternal(Box* d, STOLEN(Box*) k, STOLEN(Box* v));
 
 inline bool isSubclass(BoxedClass* child, BoxedClass* parent) {
     return child == parent || PyType_IsSubtype(child, parent);

test/cpython/test_pyexpat.py

 # expected: reffail
+# - "pyexpat.xmlparser needs a tp_traverse"
 # XXX TypeErrors on calling handlers, or on bad return values from a
 # handler, are obscure and unhelpful.
 

test/tests/large_dict.py

+# Test to make sure we can handle large dict literals.  We used to have issues since we would try to allocate
+# stack space per item, and would eventually run out.
+
+# from pydoc.py:
+
+def f():
+    topics = {
+        'TYPES': ('types', 'STRINGS UNICODE NUMBERS SEQUENCES MAPPINGS '
+                  'FUNCTIONS CLASSES MODULES FILES inspect'),
+        'STRINGS': ('strings', 'str UNICODE SEQUENCES STRINGMETHODS FORMATTING '
+                    'TYPES'),
+        'STRINGMETHODS': ('string-methods', 'STRINGS FORMATTING'),
+        'FORMATTING': ('formatstrings', 'OPERATORS'),
+        'UNICODE': ('strings', 'encodings unicode SEQUENCES STRINGMETHODS '
+                    'FORMATTING TYPES'),
+        'NUMBERS': ('numbers', 'INTEGER FLOAT COMPLEX TYPES'),
+        'INTEGER': ('integers', 'int range'),
+        'FLOAT': ('floating', 'float math'),
+        'COMPLEX': ('imaginary', 'complex cmath'),
+        'SEQUENCES': ('typesseq', 'STRINGMETHODS FORMATTING xrange LISTS'),
+        'MAPPINGS': 'DICTIONARIES',
+        'FUNCTIONS': ('typesfunctions', 'def TYPES'),
+        'METHODS': ('typesmethods', 'class def CLASSES TYPES'),
+        'CODEOBJECTS': ('bltin-code-objects', 'compile FUNCTIONS TYPES'),
+        'TYPEOBJECTS': ('bltin-type-objects', 'types TYPES'),
+        'FRAMEOBJECTS': 'TYPES',
+        'TRACEBACKS': 'TYPES',
+        'NONE': ('bltin-null-object', ''),
+        'ELLIPSIS': ('bltin-ellipsis-object', 'SLICINGS'),
+        'FILES': ('bltin-file-objects', ''),
+        'SPECIALATTRIBUTES': ('specialattrs', ''),
+        'CLASSES': ('types', 'class SPECIALMETHODS PRIVATENAMES'),
+        'MODULES': ('typesmodules', 'import'),
+        'PACKAGES': 'import',
+        'EXPRESSIONS': ('operator-summary', 'lambda or and not in is BOOLEAN '
+                        'COMPARISON BITWISE SHIFTING BINARY FORMATTING POWER '
+                        'UNARY ATTRIBUTES SUBSCRIPTS SLICINGS CALLS TUPLES '
+                        'LISTS DICTIONARIES BACKQUOTES'),
+        'OPERATORS': 'EXPRESSIONS',
+        'PRECEDENCE': 'EXPRESSIONS',
+        'OBJECTS': ('objects', 'TYPES'),
+        'SPECIALMETHODS': ('specialnames', 'BASICMETHODS ATTRIBUTEMETHODS '
+                           'CALLABLEMETHODS SEQUENCEMETHODS1 MAPPINGMETHODS '
+                           'SEQUENCEMETHODS2 NUMBERMETHODS CLASSES'),
+        'BASICMETHODS': ('customization', 'cmp hash repr str SPECIALMETHODS'),
+        'ATTRIBUTEMETHODS': ('attribute-access', 'ATTRIBUTES SPECIALMETHODS'),
+        'CALLABLEMETHODS': ('callable-types', 'CALLS SPECIALMETHODS'),
+        'SEQUENCEMETHODS1': ('sequence-types', 'SEQUENCES SEQUENCEMETHODS2 '
+                             'SPECIALMETHODS'),
+        'SEQUENCEMETHODS2': ('sequence-methods', 'SEQUENCES SEQUENCEMETHODS1 '
+                             'SPECIALMETHODS'),
+        'MAPPINGMETHODS': ('sequence-types', 'MAPPINGS SPECIALMETHODS'),
+        'NUMBERMETHODS': ('numeric-types', 'NUMBERS AUGMENTEDASSIGNMENT '
+                          'SPECIALMETHODS'),
+        'EXECUTION': ('execmodel', 'NAMESPACES DYNAMICFEATURES EXCEPTIONS'),
+        'NAMESPACES': ('naming', 'global ASSIGNMENT DELETION DYNAMICFEATURES'),
+        'DYNAMICFEATURES': ('dynamic-features', ''),
+        'SCOPING': 'NAMESPACES',
+        'FRAMES': 'NAMESPACES',
+        'EXCEPTIONS': ('exceptions', 'try except finally raise'),
+        'COERCIONS': ('coercion-rules','CONVERSIONS'),
+        'CONVERSIONS': ('conversions', 'COERCIONS'),
+        'IDENTIFIERS': ('identifiers', 'keywords SPECIALIDENTIFIERS'),
+        'SPECIALIDENTIFIERS': ('id-classes', ''),
+        'PRIVATENAMES': ('atom-identifiers', ''),
+        'LITERALS': ('atom-literals', 'STRINGS BACKQUOTES NUMBERS '
+                     'TUPLELITERALS LISTLITERALS DICTIONARYLITERALS'),
+        'TUPLES': 'SEQUENCES',
+        'TUPLELITERALS': ('exprlists', 'TUPLES LITERALS'),
+        'LISTS': ('typesseq-mutable', 'LISTLITERALS'),
+        'LISTLITERALS': ('lists', 'LISTS LITERALS'),
+        'DICTIONARIES': ('typesmapping', 'DICTIONARYLITERALS'),
+        'DICTIONARYLITERALS': ('dict', 'DICTIONARIES LITERALS'),
+        'BACKQUOTES': ('string-conversions', 'repr str STRINGS LITERALS'),
+        'ATTRIBUTES': ('attribute-references', 'getattr hasattr setattr '
+                       'ATTRIBUTEMETHODS'),
+        'SUBSCRIPTS': ('subscriptions', 'SEQUENCEMETHODS1'),
+        'SLICINGS': ('slicings', 'SEQUENCEMETHODS2'),
+        'CALLS': ('calls', 'EXPRESSIONS'),
+        'POWER': ('power', 'EXPRESSIONS'),
+        'UNARY': ('unary', 'EXPRESSIONS'),
+        'BINARY': ('binary', 'EXPRESSIONS'),
+        'SHIFTING': ('shifting', 'EXPRESSIONS'),
+        'BITWISE': ('bitwise', 'EXPRESSIONS'),
+        'COMPARISON': ('comparisons', 'EXPRESSIONS BASICMETHODS'),
+        'BOOLEAN': ('booleans', 'EXPRESSIONS TRUTHVALUE'),
+        'ASSERTION': 'assert',
+        'ASSIGNMENT': ('assignment', 'AUGMENTEDASSIGNMENT'),
+        'AUGMENTEDASSIGNMENT': ('augassign', 'NUMBERMETHODS'),
+        'DELETION': 'del',
+        'PRINTING': 'print',
+        'RETURNING': 'return',
+        'IMPORTING': 'import',
+        'CONDITIONAL': 'if',
+        'LOOPING': ('compound', 'for while break continue'),
+        'TRUTHVALUE': ('truth', 'if while and or not BASICMETHODS'),
+        'DEBUGGING': ('debugger', 'pdb'),
+        'CONTEXTMANAGERS': ('context-managers', 'with'),
+    }
+
+for i in xrange(100):
+    f()

test/tests/literals_order.py

+# expected: fail
+# - the CFG expands to the wrong code here
+
+class H(object):
+    def __init__(self, n):
+        self.n = n
+        print "Creating", repr(n)
+
+    def __hash__(self):
+        print "hashing", repr(self.n)
+        return self.n
+
+    def __eq__(self, rhs):
+        print "eq"
+        return self is rhs
+
+print "testing sets"
+# Set literals evaluate all subexpressions, then do hash operatons:
+def f():
+    {H(1), H(2), H(1L)}
+for i in xrange(100):
+    f()
+
+
+print "testing dicts"
+# Dict literals evaluate subexpressions one by one, then do hash operations
+def f2():
+    {H(1): H(4), H(2): H(5), H(1L): H(6)}
+for i in xrange(100):
+    f2()