socket: workaround socket close problem

The socket implementation used to only call close on a socket when all references where gone.
This works fine for cpython which uses reference counting but for pyston (and pypy) this is a problem.
This works around the problem in a similar way as pypy does.
It also uses the same method names so that 3th party libs which already support pypy will automatically work with pyston too.
(e.g. urllib3)

from_cpython/Lib/socket.py

@@ -168,6 +168,12 @@ class _closedsocket(object):
     __slots__ = []
     def _dummy(*args):
         raise error(EBADF, 'Bad file descriptor')
+
+    # Pyston change: socket close: add refcounting similar to pypy approach
+    def _reuse(self):
+        pass
+    def _drop(self):
+        pass
     # All _delegate_methods must also be initialized here.
     send = recv = recv_into = sendto = recvfrom = recvfrom_into = _dummy
     __getattr__ = _dummy
@@ -176,6 +182,9 @@ class _closedsocket(object):
 # a platform-independent dup() functionality. The
 # implementation currently relies on reference counting
 # to close the underlying socket object.
+# Pyston change: socket close: we workaround the socket closing problem similar to pypy
+#                by manually keeping track of the ref count.
+#                When we switch to ref counting we should remove this changes!
 class _socketobject(object):
 
     __doc__ = _realsocket.__doc__
@@ -185,6 +194,11 @@ class _socketobject(object):
     def __init__(self, family=AF_INET, type=SOCK_STREAM, proto=0, _sock=None):
         if _sock is None:
             _sock = _realsocket(family, type, proto)
+
+        # Pyston change: socket close: add refcounting similar to pypys approach
+        else:
+            _sock._reuse()
+
         self._sock = _sock
         for method in _delegate_methods:
             setattr(self, method, getattr(_sock, method))
@@ -192,6 +206,10 @@ class _socketobject(object):
     def close(self, _closedsocket=_closedsocket,
               _delegate_methods=_delegate_methods, setattr=setattr):
         # This function should not reference any globals. See issue #808164.
+
+        # Pyston change: socket close: add refcounting similar to pypys approach
+        self._sock._drop()
+
         self._sock = _closedsocket()
         dummy = self._sock._dummy
         for method in _delegate_methods:
@@ -200,7 +218,13 @@ class _socketobject(object):
 
     def accept(self):
         sock, addr = self._sock.accept()
-        return _socketobject(_sock=sock), addr
+
+        # Pyston change: socket close: add refcounting similar to pypys approach
+        # return _socketobject(_sock=sock), addr
+        sockcopy = _socketobject(_sock=sock)
+        sock._drop()
+        return sockcopy, addr
+
     accept.__doc__ = _realsocket.accept.__doc__
 
     def dup(self):
@@ -245,6 +269,10 @@ class _fileobject(object):
 
     def __init__(self, sock, mode='rb', bufsize=-1, close=False):
         self._sock = sock
+
+        # Pyston change: socket close: add refcounting similar to pypys approach
+        sock._reuse()
+
         self.mode = mode # Not actually used in this version
         if bufsize < 0:
             bufsize = self.default_bufsize
@@ -280,6 +308,11 @@ class _fileobject(object):
         finally:
             if self._close:
                 self._sock.close()
+
+            # Pyston change: socket close: add refcounting similar to pypys approach
+            else:
+                self._sock._drop()
+
             self._sock = None
 
     def __del__(self):

from_cpython/Lib/ssl.py

@@ -156,6 +156,9 @@ class SSLSocket(socket):
         self.suppress_ragged_eofs = suppress_ragged_eofs
         self._makefile_refs = 0
 
+        # Pyston change: socket close: we have to decrease the socket refcount by calling close (pypy does the same)
+        sock.close()
+
     def read(self, len=1024):
 
         """Read up to LEN bytes and return them.
@@ -371,11 +374,21 @@ class SSLSocket(socket):
         works with the SSL connection.  Just use the code
         from the socket module."""
 
-        self._makefile_refs += 1
+        # Pyston change: socket close: we increase the refcount inside _fileobject.__init__
+        # self._makefile_refs += 1
+
         # close=True so as to decrement the reference count when done with
         # the file-like object.
         return _fileobject(self, mode, bufsize, close=True)
 
+    # Pyston change: socket close: add refcounting similar to pypys approach
+    def _reuse(self):
+        self._makefile_refs += 1
+    def _drop(self):
+        if self._makefile_refs < 1:
+            self.close()
+        else:
+            self._makefile_refs -= 1
 
 
 def wrap_socket(sock, keyfile=None, certfile=None,

from_cpython/Lib/test/test_asynchat.py

-# expected: fail
 # test asynchat
 
 import asyncore, asynchat, socket, time

from_cpython/Lib/test/test_asyncore.py

-# expected: fail
 import asyncore
 import unittest
 import select

from_cpython/Lib/test/test_ftplib.py

-# expected: fail
 """Test script for ftplib module."""
 
 # Modified by Giampaolo Rodola' to test FTP class, IPv6 and TLS

from_cpython/Lib/test/test_httplib.py

-# expected: fail
 import httplib
 import array
 import httplib

from_cpython/Lib/test/test_poplib.py

-# expected: fail
 """Test script for poplib module."""
 
 # Modified by Giampaolo Rodola' to give poplib.POP3 and poplib.POP3_SSL

from_cpython/Lib/test/test_ssl.py

-# expected: fail
 # Test the support for SSL and sockets
 
 import sys

from_cpython/Lib/test/test_telnetlib.py

-# expected: fail
 import socket
 import telnetlib
 import time

from_cpython/Lib/test/test_urllib2net.py

-# expected: fail
 import unittest
 from test import test_support
 from test.test_urllib2 import sanepathname2url

from_cpython/Lib/test/test_urllibnet.py

-# expected: fail
 import unittest
 from test import test_support
 

from_cpython/Lib/urllib2.py

@@ -1200,6 +1200,11 @@ class AbstractHTTPHandler(BaseHandler):
         # out of socket._fileobject() and into a base class.
 
         r.recv = r.read
+
+        # Pyston change: socket close: add refcounting similar to pypys approach
+        r._reuse = lambda: None
+        r._drop = lambda: None
+
         fp = socket._fileobject(r, close=True)
 
         resp = addinfourl(fp, r.msg, req.get_full_url())

from_cpython/Modules/socketmodule.c

@@ -789,6 +789,9 @@ init_sockobject(PySocketSockObject *s,
 
     s->errorhandler = &set_error;
 
+    // Pyston change: socket close: add refcounting similar to pypys approach
+    s->close_ref_count = 1;
+
     if (defaulttimeout >= 0.0)
         internal_setblocking(s, 0);
 
@@ -2983,6 +2986,21 @@ sock_shutdown(PySocketSockObject *s, PyObject *arg)
     return Py_None;
 }
 
+// Pyston change: socket close: add refcounting similar to pypys approach
+static PyObject *
+sock_reuse(PySocketSockObject *s) {
+    assert(s->close_ref_count > 0);
+    ++s->close_ref_count;
+    return Py_None;
+}
+static PyObject *
+sock_drop(PySocketSockObject *s) {
+    --s->close_ref_count;
+    if (s->close_ref_count <= 0)
+        sock_close(s);
+    return Py_None;
+}
+
 PyDoc_STRVAR(shutdown_doc,
 "shutdown(flag)\n\
 \n\
@@ -3099,6 +3117,11 @@ static PyMethodDef sock_methods[] = {
     {"sleeptaskw",        (PyCFunction)sock_sleeptaskw, METH_O,
                       sleeptaskw_doc},
 #endif
+
+    // Pyston change: socket close: add refcounting similar to pypys approach
+    {"_reuse",          (PyCFunction)sock_reuse, METH_NOARGS, NULL},
+    {"_drop",           (PyCFunction)sock_drop, METH_NOARGS, NULL},
+
     {NULL,                      NULL}           /* sentinel */
 };
 

from_cpython/Modules/socketmodule.h

@@ -132,6 +132,9 @@ typedef struct {
                                         sets a Python exception */
     double sock_timeout;                 /* Operation timeout in seconds;
                                         0.0 means non-blocking */
+
+    // Pyston change: socket close: add refcounting similar to pypys approach
+    int close_ref_count;
 } PySocketSockObject;
 
 /* --- C API ----------------------------------------------------*/

test/CPYTHON_TEST_NOTES.md

@@ -24,8 +24,6 @@ test_al                 No module named al
 test_applesingle        Not really a failure, but it tries to skip itself and we don't support that
 test_argparse           [unknown]
 test_ascii_formatd      segfault in ctypes (but only on CI)
-test_asynchat           [unknown]
-test_asyncore           [unknown]
 test_atexit             [unknown]
 test_audioop            [unknown]
 test_bigmem             [unknown]
@@ -96,7 +94,6 @@ test_file_eintr         not sure
 test_fileio             [unknown]
 test_fork1              [unknown]
 test_frozen             [unknown]
-test_ftplib             [unknown]
 test_funcattrs          we don't allow changing numing of function defaults
 test_functools          unknown errors
 test_future5            [unknown]
@@ -112,7 +109,6 @@ test_gl                 [unknown]
 test_grammar            bug in our tokenizer
 test_heapq              [unknown]
 test_hotshot            [unknown]
-test_httplib            [unknown]
 test_httpservers        [unknown]
 test_idle               [unknown]
 test_imageop            [unknown]
@@ -155,7 +151,6 @@ test_pep352             various unique bugs
 test_pickletools        [unknown]
 test_pickle             unknown
 test_pkg                unknown bug
-test_poplib             SSLError (but only on CI)
 test_pprint             [unknown]
 test_profile            [unknown]
 test_py3kwarn           [unknown]
@@ -173,11 +168,10 @@ test_scope              eval of code object from existing function (not currentl
 test_scriptpackages     [unknown]
 test_shelve             [unknown]
 test_site               [unknown]
-test_socketserver       [unknown]
+test_socketserver       missing imp.lock_held, otherwise works
 test_socket             [unknown]
 test_sort               argument specification issue in listSort?
 test_sqlite             [unknown]
-test_ssl                [unknown]
 test_startfile          [unknown]
 test_str                memory leak?
 test_structmembers      [unknown]
@@ -194,7 +188,6 @@ test_sys_settrace       [unknown]
 test_sys                [unknown]
 test_tarfile            [unknown]
 test_tcl                [unknown]
-test_telnetlib          [unknown]
 test_tempfile           [unknown]
 test_threaded_import    [unknown]
 test_threading_local    [unknown]
@@ -217,8 +210,6 @@ test_unicode_file       exit code 139, no error message
 test_unittest           serialize_ast assert
 test_univnewlines2k     [unknown]
 test_univnewlines       [unknown]
-test_urllib2net         [unknown]
-test_urllibnet          [unknown]
 test_userdict           segfault: repr of recursive dict?
 test_userlist           slice(1L, 1L)
 test_userstring         float(1L); hangs in test_replace

test/cpython/badcert.pem

+../../from_cpython/Lib/test/badcert.pem
\ No newline at end of file

test/cpython/badkey.pem

+../../from_cpython/Lib/test/badkey.pem
\ No newline at end of file

test/cpython/https_svn_python_org_root.pem

+../../from_cpython/Lib/test/https_svn_python_org_root.pem
\ No newline at end of file

test/cpython/keycert.pem

+../../from_cpython/Lib/test/keycert.pem
\ No newline at end of file

test/cpython/nokia.pem

+../../from_cpython/Lib/test/nokia.pem
\ No newline at end of file

test/cpython/nullbytecert.pem

+../../from_cpython/Lib/test/nullbytecert.pem
\ No newline at end of file

test/cpython/nullcert.pem

+../../from_cpython/Lib/test/nullcert.pem
\ No newline at end of file

test/cpython/sha256.pem

+../../from_cpython/Lib/test/sha256.pem
\ No newline at end of file

test/cpython/wrongcert.pem

+../../from_cpython/Lib/test/wrongcert.pem
\ No newline at end of file

test/extra/M2Crypto_patch.patch

@@ -20,4 +20,18 @@ diff -ur M2Crypto-0.21.1.orig/SWIG/_lib.i ./SWIG/_lib.i
              new_style_callback = 1;
          }    
      } else {
+diff -ur M2Crypto-0.21.1/M2Crypto/m2urllib2.py ./M2Crypto/m2urllib2.py
+--- M2Crypto-0.21.1/M2Crypto/m2urllib2.py	2011-01-15 19:10:05.000000000 +0000
++++ ./M2Crypto/m2urllib2.py	2016-02-10 14:36:12.091812337 +0000
+@@ -97,6 +97,11 @@
+         # out of socket._fileobject() and into a base class.
  
+         r.recv = r.read
++
++        # Pyston change: our socket implementation needs this functions
++        r._reuse = lambda: None
++        r._drop = lambda: None
++
+         fp = _closing_fileobject(r)
+ 
+         resp = addinfourl(fp, r.msg, req.get_full_url())