merge

README.rst

@@ -22,7 +22,7 @@ Features
  * with_timeout(): wrap any funcall with a timeout.
  * Wait on kqueue events like file/directory changes, signals, processes, etc... [kqueue only]
  * DNS stub resolver (full-fledged resolver may be forthcoming)
- * HTTP server and client
+ * HTTP server and client (plus WebSocket, RFC6455 & hixie-76)
  * Support for TLS via tlslite (openssl interface may be forthcoming)
  * other protocols/codecs: ldap, asn1, ftp, mysql, postgres, AMQP_.
  * `MIT License`_.

coro/asn1/__init__.py

+# -*- Mode: Python -*-
+# even empty, this file is needed so cython will see the .pxd

coro/asn1/ber.pxd

+# -*- Mode: Cython -*-
+
+# flags for BER tags
+cdef enum FLAGS:
+    FLAGS_UNIVERSAL       = 0x00
+    FLAGS_STRUCTURED      = 0x20
+    FLAGS_APPLICATION     = 0x40
+    FLAGS_CONTEXT         = 0x80
+
+# NULL is a pyrex keyword
+# universal BER tags
+cdef enum TAGS:
+    TAGS_BOOLEAN          = 0x01
+    TAGS_INTEGER          = 0x02
+    TAGS_BITSTRING        = 0x03
+    TAGS_OCTET_STRING     = 0x04
+    TAGS_NULL             = 0x05
+    TAGS_OBJID            = 0x06
+    TAGS_OBJDESCRIPTOR    = 0x07
+    TAGS_EXTERNAL         = 0x08
+    TAGS_REAL             = 0x09
+    TAGS_ENUMERATED       = 0x0a
+    TAGS_EMBEDDED_PDV     = 0x0b
+    TAGS_UTF8STRING       = 0x0c
+    TAGS_SEQUENCE         = 0x10 | 0x20 # Equivalent to FLAGS_STRUCTURED
+    TAGS_SET              = 0x11 | 0x20 # Equivalent to FLAGS_STRUCTURED
+
+cdef int length_of_length (int n)
+cdef void encode_length (int l, int n, char * buffer)
+cdef object _encode_integer (int n)
+cdef object _encode_long_integer (n)
+cdef object _TLV1 (int tag, bytes data)
+cdef object _TLV (int tag, object data)
+cdef object _CHOICE (int n, bint structured)
+cdef object _APPLICATION (int n)
+cdef object _ENUMERATED (int n)
+cdef object _INTEGER (int n)
+cdef object _BOOLEAN (int n)
+cdef object _SEQUENCE (object elems)
+cdef object _SET (object elems)
+cdef object _OCTET_STRING (bytes s)
+cdef object _OBJID (list l)
+cdef object decode_string (unsigned char * s, int * pos, int length)
+cdef object decode_raw (unsigned char * s, int * pos, int length)
+cdef object decode_bitstring (unsigned char * s, int * pos, int length)
+cdef object decode_integer (unsigned char * s, int * pos, int length)
+cdef object decode_long_integer (unsigned char * s, int * pos, int length)
+cdef object decode_structured (unsigned char * s, int * pos, int length)
+cdef object decode_objid (unsigned char * s, int * pos, int length)
+cdef object decode_boolean (unsigned char * s, int * pos, int length)
+cdef int _decode_length (unsigned char * s, int * pos, int lol)
+cdef object _decode (unsigned char * s, int * pos, int eos, bint just_tlv)

coro/asn1/test/t0.py

+# -*- Mode: Python -*-
+
+from coro.asn1.ber import *
+import unittest
+
+# These are mostly positive test cases, need some negative ones as well.
+# Though - this code *has* been through the protos c06-ldapv3-enc-r1 test suite,
+#   but it's a rather large suite (89MB).  Consider automating a download of
+#   the suite here?
+
+class ber_test_case (unittest.TestCase):
+    pass
+
+class simple_test (ber_test_case):
+    def runTest (self):
+        x = SEQUENCE (
+            SET (INTEGER(34), INTEGER(19), OCTET_STRING('fishing line')),
+            OBJID ([2,3,4,5,6,88]),
+            OCTET_STRING ("spaghetti"),
+            )
+        self.assertEqual (x, '0(1\x14\x02\x01"\x02\x01\x13\x04\x0cfishing line\x06\x05S\x04\x05\x06X\x04\tspaghetti')
+        self.assertEqual (decode (x), ([[34, 19, 'fishing line'], ('oid', [2, 3, 4, 5, 6, 88]), 'spaghetti'], 42))
+
+# www.google.com cert
+google_cert = """-----BEGIN CERTIFICATE-----
+MIIDITCCAoqgAwIBAgIQT52W2WawmStUwpV8tBV9TTANBgkqhkiG9w0BAQUFADBM
+MQswCQYDVQQGEwJaQTElMCMGA1UEChMcVGhhd3RlIENvbnN1bHRpbmcgKFB0eSkg
+THRkLjEWMBQGA1UEAxMNVGhhd3RlIFNHQyBDQTAeFw0xMTEwMjYwMDAwMDBaFw0x
+MzA5MzAyMzU5NTlaMGgxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpDYWxpZm9ybmlh
+MRYwFAYDVQQHFA1Nb3VudGFpbiBWaWV3MRMwEQYDVQQKFApHb29nbGUgSW5jMRcw
+FQYDVQQDFA53d3cuZ29vZ2xlLmNvbTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkC
+gYEA3rcmQ6aZhc04pxUJuc8PycNVjIjujI0oJyRLKl6g2Bb6YRhLz21ggNM1QDJy
+wI8S2OVOj7my9tkVXlqGMaO6hqpryNlxjMzNJxMenUJdOPanrO/6YvMYgdQkRn8B
+d3zGKokUmbuYOR2oGfs5AER9G5RqeC1prcB6LPrQ2iASmNMCAwEAAaOB5zCB5DAM
+BgNVHRMBAf8EAjAAMDYGA1UdHwQvMC0wK6ApoCeGJWh0dHA6Ly9jcmwudGhhd3Rl
+LmNvbS9UaGF3dGVTR0NDQS5jcmwwKAYDVR0lBCEwHwYIKwYBBQUHAwEGCCsGAQUF
+BwMCBglghkgBhvhCBAEwcgYIKwYBBQUHAQEEZjBkMCIGCCsGAQUFBzABhhZodHRw
+Oi8vb2NzcC50aGF3dGUuY29tMD4GCCsGAQUFBzAChjJodHRwOi8vd3d3LnRoYXd0
+ZS5jb20vcmVwb3NpdG9yeS9UaGF3dGVfU0dDX0NBLmNydDANBgkqhkiG9w0BAQUF
+AAOBgQAhrNWuyjSJWsKrUtKyNGadeqvu5nzVfsJcKLt0AMkQH0IT/GmKHiSgAgDp
+ulvKGQSy068Bsn5fFNum21K5mvMSf3yinDtvmX3qUA12IxL/92ZzKbeVCq3Yi7Le
+IOkKcGQRCMha8X2e7GmlpdWC1ycenlbN0nbVeSv3JUMcafC4+Q==
+-----END CERTIFICATE-----"""
+
+class x509_test (ber_test_case):
+
+    def runTest (self):
+        import base64
+        lines = google_cert.split ('\n')
+        enc = base64.decodestring (''.join (lines[1:-1]))
+        self.assertEqual (
+            decode (enc),
+            ([[('context', 0, [2]),
+               105827261859531100510423749949966875981L,
+               [('oid', [1, 2, 840, 113549, 1, 1, 5]), None],
+               [[[('oid', [2, 5, 4, 6]), ('PRINTABLE_STRING', 19, 'ZA')]],
+                [[('oid', [2, 5, 4, 10]),
+                  ('PRINTABLE_STRING', 19, 'Thawte Consulting (Pty) Ltd.')]],
+                  [[('oid', [2, 5, 4, 3]), ('PRINTABLE_STRING', 19, 'Thawte SGC CA')]]],
+                  [('UTC_TIME', 23, '111026000000Z'), ('UTC_TIME', 23, '130930235959Z')],
+                  [[[('oid', [2, 5, 4, 6]), ('PRINTABLE_STRING', 19, 'US')]],
+                   [[('oid', [2, 5, 4, 8]), ('PRINTABLE_STRING', 19, 'California')]],
+                   [[('oid', [2, 5, 4, 7]), ('T61_STRING', 20, 'Mountain View')]],
+                   [[('oid', [2, 5, 4, 10]), ('T61_STRING', 20, 'Google Inc')]],
+                   [[('oid', [2, 5, 4, 3]), ('T61_STRING', 20, 'www.google.com')]]],
+                   [[('oid', [1, 2, 840, 113549, 1, 1, 1]), None],
+                    ('bitstring',
+                     (0,
+                      "0\x81\x89\x02\x81\x81\x00\xde\xb7&C\xa6\x99\x85\xcd8\xa7\x15\t\xb9\xcf\x0f"
+                      "\xc9\xc3U\x8c\x88\xee\x8c\x8d('$K*^\xa0\xd8\x16\xfaa\x18K\xcfm`\x80\xd35@2r"
+                      "\xc0\x8f\x12\xd8\xe5N\x8f\xb9\xb2\xf6\xd9\x15^Z\x861\xa3\xba\x86\xaak\xc8\xd9"
+                      "q\x8c\xcc\xcd'\x13\x1e\x9dB]8\xf6\xa7\xac\xef\xfab\xf3\x18\x81\xd4$F\x7f\x01w|"
+                      "\xc6*\x89\x14\x99\xbb\x989\x1d\xa8\x19\xfb9\x00D}\x1b\x94jx-i\xad\xc0z,\xfa\xd0"
+                      "\xda \x12\x98\xd3\x02\x03\x01\x00\x01"))],
+                      ('context',
+                       3,
+                       [[[('oid', [2, 5, 29, 19]), True, '0\x00'],
+                         [('oid', [2, 5, 29, 31]),
+                          "0-0+\xa0)\xa0'\x86%http://crl.thawte.com/ThawteSGCCA.crl"],
+                          [('oid', [2, 5, 29, 37]),
+                           '0\x1f\x06\x08+\x06\x01\x05\x05\x07\x03\x01\x06\x08+\x06\x01\x05\x05\x07\x03'
+                           '\x02\x06\t`\x86H\x01\x86\xf8B\x04\x01'],
+                           [('oid', [1, 3, 6, 1, 5, 5, 7, 1, 1]),
+                            '0d0"\x06\x08+\x06\x01\x05\x05\x070\x01\x86\x16http://ocsp.thawte.com0>\x06'
+                            '\x08+\x06\x01\x05\x05\x070\x02\x862http://www.thawte.com/repository/Thawte_SGC_CA.crt']]])],
+                            [('oid', [1, 2, 840, 113549, 1, 1, 5]), None],
+                            ('bitstring',
+                             (0,
+                              "!\xac\xd5\xae\xca4\x89Z\xc2\xabR\xd2\xb24f\x9dz\xab\xee\xe6|\xd5~\xc2\\("
+                              "\xbbt\x00\xc9\x10\x1fB\x13\xfci\x8a\x1e$\xa0\x02\x00\xe9\xba[\xca\x19\x04"
+                              "\xb2\xd3\xaf\x01\xb2~_\x14\xdb\xa6\xdbR\xb9\x9a\xf3\x12\x7f|\xa2\x9c;o\x99"
+                              "}\xeaP\rv#\x12\xff\xf7fs)\xb7\x95\n\xad\xd8\x8b\xb2\xde \xe9\npd\x11\x08"
+                              "\xc8Z\xf1}\x9e\xeci\xa5\xa5\xd5\x82\xd7'\x1e\x9eV\xcd\xd2v\xd5y+\xf7%C\x1c"
+                              "i\xf0\xb8\xf9"))],
+                805)
+            )
+        dec, length = decode (enc)
+        public_key = dec[0][6][1][1][1]
+        self.assertEqual (
+            decode (public_key),
+            ([156396091895984667473837837332877995558144703880815901117439532534031286131520903863087599986938779606924811933611903716377206837300122262900786662124968110191717844999183338594373129421417536020806373385428322642107305024162536996222164292639147591878860587271770855626780464602884552232097424473091745159379L, 65537], 140)
+            )
+
+class bignum_test (ber_test_case):
+
+    def runTest (self):
+        self.assertEquals (
+            decode ('\x02\x82\x04\xe3\x01' + '\x00' * 1250),
+            (1<<10000, 1255)
+            )
+        self.assertEquals (
+            INTEGER (1<<10000),
+            '\x02\x82\x04\xe3\x01' + '\x00' * 1250,
+            )
+
+class bignum_test_2 (ber_test_case):
+
+    def runTest (self):
+        for i in range (5):
+            n = 1 << (10 ** i)
+            self.assertEquals (
+                decode (INTEGER (n))[0],
+                n
+                )
+
+class bignum_test_3 (ber_test_case):
+
+    def runTest (self):
+        import random
+        n = 1
+        for x in range (10000):
+            n = n * 10 + random.randint (0, 10)
+        print n
+        self.assertEquals (decode (INTEGER (n))[0],  n)
+
+def suite():
+    suite = unittest.TestSuite()
+    suite.addTest (simple_test())
+    suite.addTest (x509_test())
+    suite.addTest (bignum_test())
+    suite.addTest (bignum_test_2())
+    suite.addTest (bignum_test_3())
+    return suite
+
+if __name__ == '__main__':
+    unittest.main (defaultTest='suite')

coro/backdoor.py

@@ -108,9 +108,13 @@ class backdoor:
                 lines.append('')
             self.send (self.line_separator.join (lines))
 
+    def login (self):
+        "override to provide authentication"
+        pass
+
     def read_eval_print_loop (self):
+        self.login()
         self.send_welcome_message()
-
         if self.global_dict is None:
             # this does the equivalent of 'from __main__ import *'
             env = sys.modules['__main__'].__dict__.copy()
@@ -138,7 +142,7 @@ class backdoor:
         "override to process the result (e.g., pprint)"
         print result
 
-    def parse(self, line, env):
+    def parse (self, line, env):
         save = sys.stdout, sys.stderr
         output = cStringIO.StringIO()
         try:

coro/emulation/socket.py

@@ -52,9 +52,11 @@ for name in _socketmodule.__all__:
         g[name] = value
 del g, name, value
 
-BDADDR_ANY = _socketmodule.BDADDR_ANY
-BDADDR_LOCAL = _socketmodule.BDADDR_LOCAL
-
+try:
+    BDADDR_ANY = _socketmodule.BDADDR_ANY
+    BDADDR_LOCAL = _socketmodule.BDADDR_LOCAL
+except AttributeError:
+    pass
 
 ##############################################################################
 # Timeout.

coro/emulation/threading.py

@@ -178,7 +178,7 @@ class Thread(object):
         if _co is None:
             self.__co = coro.new(self.__bootstrap)
         else:
-            self.__co = co
+            self.__co = _co
         self.ident = self.__co.id
         if name:
             self.__co.set_name(name)

coro/http/demo/websocket/chat_server.py

+# -*- Mode: Python -*-
+
+# http://martinsikora.com/nodejs-and-websocket-simple-chat-tutorial
+
+import json
+from coro.http.websocket import handler, websocket
+
+import coro
+W = coro.write_stderr
+
+class server:
+
+    colors = ['red', 'green', 'blue', 'magenta', 'purple', 'plum', 'orange']
+
+    def __init__ (self):
+        self.clients = set()
+        self.color_index = 0
+
+    def next_color (self):
+        r = self.colors[self.color_index % len (self.colors)]
+        self.color_index += 1
+        return r
+
+    def new_session (self, proto, http_request, conn):
+        self.clients.add (connection (self, proto, http_request))
+
+    def broadcast (self, name, color, payload):
+        to_remove = set()
+        for client in self.clients:
+            try:
+                client.send_message (name, color, payload)
+            except:
+                to_remove.add (client)
+        self.clients.difference_update (to_remove)
+
+class connection (websocket):
+
+    def __init__ (self, server, proto, req):
+        self.server = server
+        self.color = server.next_color()
+        self.name = None
+        websocket.__init__ (self, proto, req)
+
+    def handle_packet (self, p):
+        payload = p.unpack()
+        if p.opcode == 0x01:
+            if self.name is None:
+                reply = json.dumps ({'type':'color', 'data':self.color})
+                self.send_text (reply)
+                self.name = payload
+            else:
+                self.server.broadcast (self.name, self.color, payload)
+        return False
+
+    def send_message (self, name, color, message):
+        #W ('send_message %r %r %r\n' % (name, color, message))
+        self.send_text (
+            json.dumps ({
+                'type':'message',
+                'data': {
+                    'time' : int (coro.now_usec / 1000000),
+                    'text' : message,
+                    'author' : name,
+                    'color' : color
+                    }
+                })
+            )
+
+if __name__ == '__main__':
+    import coro.http
+    import coro.backdoor
+    import os
+    cwd = os.getcwd()
+
+    chat_server = server()
+
+    ih = coro.http.handlers.favicon_handler()
+    sh = coro.http.handlers.coro_status_handler()
+    fh = coro.http.handlers.file_handler (cwd)
+    wh = handler ('/chat', chat_server.new_session)
+    handlers = [ih, sh, fh, wh]
+    #http_server = coro.http.server (('0.0.0.0', 9001))
+    http_server = coro.http.server ()
+    for h in handlers:
+        http_server.push_handler (h)
+    coro.spawn (http_server.start, ('0.0.0.0', 9001))
+    coro.spawn (coro.backdoor.serve, unix_path='/tmp/ws_chat.bd')
+    coro.event_loop (30.0)

coro/http/demo/websocket/echo_server.py

+# -*- Mode: Python -*-
+
+from coro.http.websocket import handler, websocket
+
+import coro
+W = coro.write_stderr
+
+class echo_server (websocket):
+
+    def __init__ (self, *args, **kwargs):
+        self.pending = []
+        websocket.__init__ (self, *args, **kwargs)
+
+    def handle_packet (self, p):
+        #W ('packet=%r\n' % (p,))
+        self.pending.append (p.unpack())
+        if p.fin:
+            data, self.pending = self.pending, []
+            self.send_text (''.join (data))
+        return False
+
+if __name__ == '__main__':
+    import coro.http
+    import coro.backdoor
+    fh = coro.http.handlers.favicon_handler()
+    sh = coro.http.handlers.coro_status_handler()
+    wh = handler ('/echo', echo_server)
+    handlers = [fh, sh, wh]
+    #server = coro.http.server (('0.0.0.0', 9001))
+    server = coro.http.server ()
+    for h in handlers:
+        server.push_handler (h)
+    #coro.spawn (server.start)
+    coro.spawn (server.start, ('0.0.0.0', 9001))
+    coro.spawn (coro.backdoor.serve, unix_path='/tmp/ws.bd')
+    coro.event_loop (30.0)

coro/http/demo/websocket/field/README.rst

+MMO-ish demo
+============
+
+This demo creates a large field of objects (5000 rectangles), randomly distributed.
+It then spawns 1000 threads, each representing a 'non-player character', a small colored circle, that wanders drunkenly around the space.
+
+Each browser has its own independent viewport to the field, 1024x1024 pixels.
+
+Quadtrees are used to manage the display of the background objects (rectangles), the moving objects (the circles), and the browser viewports.
+
+The display is done via an html5 2d canvas support, and has been tested on Chrome, Firefox, Safari.  It also works on iOS.  [I've tested it on my iPhone over 3G as well].
+
+Changes to each 'viewport' are sent via websocket.
+
+Quadtree
+========
+
+The original quadtree implementation (some really old code of mine) was able to handle the 1000 objects, but the CPU load was rather high (80% @ 2.3GHz), so I've pushed that code into Cython.
+
+So in order to run this you'll need to build the Cython extension thus::
+
+ $ python setup.py build_ext --inplace
+
+

coro/http/demo/websocket/field/field.html

+<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML//EN">
+<html> <head>
+<title></title>
+</head>
+<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.7.2/jquery.min.js"></script>
+<body>
+<canvas id="canvas" width="1024" height="1024"></canvas>
+<div id="msgs"></div>
+<script src="field.js"></script>
+</body>
+</html>

coro/http/demo/websocket/field/field.js

+
+var connection;
+var msgs_div = document.getElementById ('msgs')
+var draw_cmds = [];
+
+function message (msg) {
+    msgs_div.innerHTML = msg;
+}
+
+if (window["WebSocket"]) {
+    connection = new WebSocket("ws://127.0.0.1:9001/field")
+
+    connection.onopen = function () {
+	message ('connected')
+	animate();
+    };
+
+    connection.onclose = function (event) {
+	message ('disconnected')
+    }
+    
+    connection.onmessage = function (event) {
+	//message ('websocket event: ' + event.data)
+	switch (event.data[0]) {
+	case 'F':
+	    draw_cmds = event.data.split ('|');
+	    break;
+	case 'M':
+	    message (event.data);
+	    break;
+	}
+    }
+}
+
+window.requestAnimFrame = (function(callback){
+    return window.requestAnimationFrame ||
+    window.webkitRequestAnimationFrame ||
+    window.mozRequestAnimationFrame ||
+    window.oRequestAnimationFrame ||
+    window.msRequestAnimationFrame ||
+    function(callback){
+        window.setTimeout(callback, 1000 / 60);
+    };
+})();
+
+function animate() {
+    context.clearRect (0, 0, 1024, 1024);
+    context.strokeStyle = 'rgb(0,0,0,0)';
+    for (i=0; i < draw_cmds.length; i++) {
+	var cmd = draw_cmds[i];
+	var p = cmd.split (',')
+	switch (p[0]) {
+	case 'B':
+	    context.fillStyle = p[1];
+	    context.fillRect (parseInt (p[2]), parseInt (p[3]), parseInt (p[4]), parseInt (p[5]));
+	    context.strokeRect (parseInt (p[2]), parseInt (p[3]), parseInt (p[4]), parseInt (p[5]));
+	    break;
+	case 'C':
+	    context.fillStyle = p[1];
+	    draw_circle (parseInt (p[2]), parseInt (p[3]), parseInt (p[4]));
+	    break;
+	}
+    }
+    requestAnimFrame (animate);
+}
+
+
+function draw_circle (x0, y0, r) {
+    // context.fillRect (x0-r, y0-r, r*2, r*2);
+    context.beginPath();
+    context.arc (x0, y0, r, 0, 2*Math.PI);
+    context.closePath();
+    context.fill();
+    context.stroke();
+}
+
+var canvas = document.getElementById ('canvas')
+var context = canvas.getContext ('2d')
+
+document.addEventListener ('mousedown', on_mouse_down, false)
+document.addEventListener ('mouseup', on_mouse_up, false)
+document.addEventListener ('mousemove', on_mouse_move, false)
+document.addEventListener ('keydown', on_key_down, false)
+document.addEventListener ('keyup', on_key_up, false)
+
+document.addEventListener ('touchstart', on_touch_start, false)
+document.addEventListener ('touchmove', on_touch_move, false)
+document.addEventListener ('touchend', on_touch_end, false)
+
+function on_mouse_down (event) {
+    connection.send ('MD,' + event.clientX + ',' + event.clientY)
+}
+function on_mouse_up (event) {
+    connection.send ('MU,' + event.clientX + ',' + event.clientY)
+}
+function on_mouse_move (event) {
+    connection.send ('MM,' + event.clientX + ',' + event.clientY)
+}
+function make_touch_list (tl) {
+    var result = new Array (tl.length);
+    for (i=0; i < tl.length; i++) {
+	result[i] = tl[i].pageX + '.' + tl[i].pageY;
+    }
+    return result.join (',');
+}
+function on_touch_start (event) {
+    event.preventDefault();
+    connection.send ('TS,' + make_touch_list (event.touches));
+}
+function on_touch_end (event) {
+    // no touch list on this one...
+    connection.send ('TE');
+}
+function on_touch_move (event) {
+    connection.send ('TM,' + make_touch_list (event.touches));
+}
+function on_key_down (event) {
+    connection.send ('KD,' + event.keyCode)
+}
+function on_key_up (event) {
+    connection.send ('KU,' + event.keyCode)
+}

coro/http/demo/websocket/field/field.py

+# -*- Mode: Python -*-
+
+from coro.http.websocket import handler, websocket
+import math
+import pickle
+import random
+import re
+
+import region
+import quadtree
+
+import coro
+W = coro.write_stderr
+
+# need a way to declare a dirty region, and send a single
+#   redraw command for all the dirtied objects.  So we need to separate
+#   object movement from redrawing... probably with a timer of some kind,
+#   accumulate dirty rects, then redraw in one swell foop
+# another thing to consider: sending deltas rather than the entire list.
+#  for example, if the viewport hasn't moved, then the list of rectangles
+#  won't be changing.  Can we send a diff?  [this might just be easier with layers]
+
+# for layers see: http://stackoverflow.com/questions/3008635/html5-canvas-element-multiple-layers
+
+colors = ['red', 'green', 'blue', 'magenta', 'purple', 'plum', 'orange']
+
+# sample 'box' object.
+class box (quadtree.ob):
+    def __init__ (self, color, (l,t,r,b)):
+        self.color = color
+        self.set_rect (l,t,r,b)
+    def move (self, dx, dy):
+        x0, y0, x1, y1 = self.get_rect()
+        self.set_rect (
+            int(x0 + dx), int(y0 + dy), int(x1 + dx), int(y0 + dy)
+            )
+    def cmd (self, xoff, yoff):
+        # command to draw me relative to <xoff,yoff>?
+        x0, y0, x1, y1 = self.get_rect()
+        return 'B,%s,%d,%d,%d,%d' % (self.color, x0-xoff, y0-yoff, x1-x0, y1-y0)
+    def __repr__ (self):
+        return '<box (%d,%d,%d,%d)>' % self.get_rect()
+
+class circle (box):
+    def __init__ (self, color, center, radius):
+        x, y = center
+        r = radius
+        self.color = color
+        self.center = center
+        self.radius = radius
+        self.set_rect (*self.get_rect())
+    def get_rect (self):
+        x, y = self.center
+        r = self.radius
+        return x-r, y-r, x+r, y+r
+    def move (self, dx, dy):
+        x, y = self.center
+        self.center = int(x + dx), int(y + dy)
+        self.set_rect (*self.get_rect())
+    def cmd (self, xoff, yoff):
+        # command to draw me relative to <xoff,yoff>?
+        x, y = self.center
+        return 'C,%s,%d,%d,%d' % (self.color, x-xoff, y-yoff, self.radius)
+    def __repr__ (self):
+        return '<circle (%d,%d) radius=%d)>' % (self.center[0], self.center[1], self.radius)
+
+# should have multiple qt's:
+# 1) for the background, immutable
+# 2) for the client viewports
+# 3) for moving objects
+
+class field:
+    def __init__ (self, w=1024*20, h=1024*20):
+        self.w = w
+        self.h = h
+        self.Q_views = quadtree.quadtree()
+        self.Q_back = quadtree.quadtree()
+        self.Q_obs = quadtree.quadtree()
+        self.generate_random_field()
+
+    def generate_random_field (self):
+        for i in range (5000):
+            c = random.choice (colors)
+            x = random.randint (0, self.w - 100)
+            y = random.randint (0, self.h - 100)
+            w = random.randint (50, 300)
+            h = random.randint (50, 300)
+            b = box (c, (x, y, x+w, y+h))
+            self.Q_back.insert (b)
+        for i in range (1000):
+            coro.spawn (self.wanderer)
+
+    def new_conn (self, *args, **kwargs):
+        c = field_conn (self, *args, **kwargs)
+        self.Q_views.insert (c)
+
+    def new_ob (self, ob):
+        self.Q_obs.insert (ob)
+        #W ('new ob %r\n' % (self.Q_obs,))
+        #self.Q_obs.dump()
+
+    def move_ob (self, ob, dx, dy):
+        r0 = ob.get_rect()
+        self.Q_obs.delete (ob)
+        ob.move (dx, dy)
+        r1 = ob.get_rect()
+        self.Q_obs.insert (ob)
+        #self.Q_obs.dump()
+        # notify any viewers
+        r2 = region.union (r0, r1)
+        for client in self.Q_views.search (r2):
+            client.draw_window()
+
+    sleep = 0.1
+    def wanderer (self):
+        # spawn me!
+        x = random.randint (100, self.w - 100)
+        y = random.randint (100, self.h - 100)
+        c = random.choice (colors)
+        ob = circle (c, (x, y), 25)
+        self.new_ob (ob)
+        while 1:
+            # pick a random direction
+            heading = random.randint (0, 360) * (math.pi / 180.0)
+            speed = (random.random() * 10) + 3
+            dx = math.cos (heading) * speed
+            dy = math.sin (heading) * speed
+            # go that way for 20 steps
+            for i in range (20):
+                self.move_ob (ob, dx, dy)
+                # not working yet...
+                ## if not ob.range_check (0, 0, self.w, self.h):
+                ##     W ('%r hit an edge!\n' % (ob,))
+                ##     dx = - (dx * 5)
+                ##     dy = - (dy * 5)
+                ##     self.move_ob (ob, dx, dy)
+                coro.sleep_relative (self.sleep)
+
+class field_conn (websocket, quadtree.ob):
+
+    def __init__ (self, field, *args, **kwargs):
+        websocket.__init__ (self, *args, **kwargs)
+        self.send_mutex = coro.mutex()
+        self.field = field
+        rx = random.randint (0, self.field.w-1024)
+        ry = random.randint (0, self.field.h-1024)
+        self.set_rect (rx, ry, rx+1024, ry+1024)
+        self.draw_window()
+        self.mouse_down = None, None
+
+    def move_window (self, mx, my):
+        self.field.Q_views.delete (self)
+        x0, y0, x1, y1 = self.get_rect()
+        x0 = x0 + mx; y0 = y0 + my
+        x1 = x1 + mx; y1 = y1 + my
+        if x0 < 0:
+            x0 = 0; x1 = 1024
+        if y0 < 0:
+            y0 = 0; y1 = 1024
+        if x1 > self.field.w-1024:
+            x1 = self.field.w-1024; x0 = x1 - 1024
+        if y1 > self.field.h-1024:
+            y1 = self.field.h-1024; y0 = y1 - 1024
+        self.set_rect (x0, y0, x1, y1)
+        self.field.Q_views.insert (self)
+        self.draw_window()
+        self.send_text ('M pos=%d,%d' % self.get_rect()[:2])
+
+    def draw_qt (self, r, Q):
+        px, py = self.get_rect()[:2]
+        for ob in Q.search (self.rect):
+            r.append (ob.cmd (px, py))
+
+    def draw_window (self):
+        r = ['F']
+        self.draw_qt (r, self.field.Q_back)
+        self.draw_qt (r, self.field.Q_obs)
+        self.send_text ('|'.join (r))
+
+    def send_text (self, payload):
+        with self.send_mutex:
+            websocket.send_text (self, payload)
+
+    def handle_packet (self, p):
+        data = p.unpack()
+        event = p.unpack().split (',')
+        #W ('packet = %r event=%r\n' % (p, event))
+        if event[0] == 'KD':
+            ascii = int (event[1])
+            if ascii == 87: # W
+                self.move_window (0, -10)
+            elif ascii == 65: # A
+                self.move_window (-10, 0)
+            elif ascii == 83: # S
+                self.move_window (0, 10)
+            elif ascii == 68: # D
+                self.move_window (10, 0)
+            elif ascii == 82: # R
+                x0, y0 = self.get_rect()[:2]
+                self.move_window (-x0, -y0)
+        elif event[0] == 'MD':
+            self.on_mouse_down (int (event[1]), int (event[2]))
+        elif event[0] == 'MU':
+            self.on_mouse_up (int (event[1]), int (event[2]))
+        elif event[0] == 'MM':
+            self.on_mouse_move (int (event[1]), int (event[2]))
+        elif event[0] == 'TS':
+            tl = self.unpack_touch_list (event[1:])
+            self.last_touch_move = tl[0][0], tl[0][1]
+            self.on_mouse_down (tl[0][0], tl[0][1])
+        elif event[0] == 'TM':
+            tl = self.unpack_touch_list (event[1:])
+            self.last_touch_move = tl[0][0], tl[0][1]
+            self.on_mouse_move (tl[0][0], tl[0][1])
+        elif event[0] == 'TE':
+            # emulate mouse up by with saved last touch_move
+            x0, y0 = self.last_touch_move
+            if x0 is not None:
+                self.on_mouse_up (x0, y0)
+            self.last_touch_move = None, None
+        else:
+            W ('unknown event: %r\n' % (event,))
+        return False
+
+    def unpack_touch_list (self, tl):
+        return [ [int(y) for y in x.split('.')] for x in tl ]
+
+    def on_mouse_down (self, x, y):
+        self.mouse_down = x, y
+        
+    def on_mouse_up (self, x1, y1):
+        x0, y0 = self.mouse_down
+        self.mouse_down = None, None
+        if x0 is not None:
+            self.move_window (x0-x1, y0-y1)
+            self.draw_window()
+
+    def on_mouse_move (self, x1, y1):
+        x0, y0 = self.mouse_down
+        if x0:
+            if abs(x1-x0) > 10 or abs(y1-y0) > 10:
+                # moved enough to redraw
+                self.mouse_down = x1, y1
+                self.move_window (x0-x1, y0-y1)
+
+if __name__ == '__main__':
+    import coro.http
+    import coro.backdoor
+    import os
+    cwd = os.getcwd()
+    f = field()
+    ih = coro.http.handlers.favicon_handler()
+    sh = coro.http.handlers.coro_status_handler()
+    th = handler ('/field', f.new_conn)
+    fh = coro.http.handlers.file_handler (cwd)
+    # so you can browse the source
+    import mimetypes
+    mimetypes.init()
+    mimetypes.add_type ('text/plain', '.py')
+    handlers = [th, ih, sh, fh]
+    #server = coro.http.server (('0.0.0.0', 9001))
+    server = coro.http.server()
+    for h in handlers:
+        server.push_handler (h)
+    #coro.spawn (server.start)
+    coro.spawn (server.start, ('0.0.0.0', 9001))
+    coro.spawn (coro.backdoor.serve, unix_path='/tmp/ws.bd')
+    coro.event_loop (30.0)

coro/http/demo/websocket/field/quadtree.pyx

+# -*- Mode: Python; tab-width: 4 -*-
+
+# SMR 2012: originally from dynwin, circa 1995-96.
+#   modernized, replace some funs with generators
+#
+# Cython version.
+#
+
+#
+# Quad-Tree.  A 2D spatial data structure.
+#
+# Used to quickly locate 'objects' within a particular region.  Each
+# node in the tree represents a rectangular region, while its children
+# represent that region split into four quadrants.  An 'object' is
+# stored at a particular level if it is contained within that region,
+# but will not fit into any of the individual quadrants inside it.
+#
+# If an object is inserted into a quadtree that 'overflows' the current
+# boundaries, the tree is re-rooted in a larger space.
+
+# --------------------------------------------------------------------------------
+# regions
+#
+# do two rectangular regions intersect?
+#
+# +--->
+# |           lb,tb-------+
+# |           |           |
+# |    la,ta--+-----+     |
+# |   |       |     |     |
+# |   |       +-----+-rb,bb
+# V   |             |
+#     +--------ra,ba
+
+# a rect is (left,top,right,bottom)
+# top is < bottom, left is < right
+
+# proof: imagine all possible cases in 1 dimension,
+# (there are six) and then generalize.  simplify the
+# expression and you get this.  (trust me 8^)
+
+cimport cython
+
+from libc.stdint cimport int32_t, uint32_t
+
+ctypedef struct rect:
+    int32_t l,t,r,b
+
+cdef bint intersects (rect * a, rect * b):
+    return a.r >= b.l and b.r >= a.l and a.b >= b.t and b.b >= a.t
+
+# does region <a> fully contain region <b>?
+cdef bint contains (rect * a, rect * b):
+    return a.l <= b.l and a.r >= b.r and a.t <= b.t and a.b >= b.b
+
+cdef void union (rect * a, rect * b, rect * c):
+    c.l = min (a.l, b.l)
+    c.t = min (a.l, b.l)
+    c.r = max (a.r, b.r)
+    c.b = max (a.b, b.b)
+
+# split a rect into four quadrants
+@cython.cdivision (True)
+cdef split (rect * x, rect * p):
+    cdef int32_t w2 = ((x.r - x.l) // 2) + x.l
+    cdef int32_t h2 = ((x.b - x.t) // 2) + x.t
+    p[0].l = x.l; p[0].t = x.t; p[0].r = w2;  p[0].b = h2
+    p[1].l = w2;  p[1].t = x.t; p[1].r = x.r; p[1].b = h2
+    p[2].l = x.l; p[2].t = h2;  p[2].r = w2;  p[2].b = x.b
+    p[3].l = w2;  p[3].t = h2;  p[3].r = x.r; p[3].b = x.b
+
+cdef inline set_rect (rect * x, int32_t l, int32_t t, int32_t r, int32_t b):
+    x.l = l; x.t = t; x.r = r; x.b = b
+
+import sys
+W = sys.stderr.write
+
+cdef rect_repr (rect * r):
+    return '(%d,%d,%d,%d)' % (r.l, r.t, r.r, r.b)
+
+class QuadtreeError (Exception):
+    pass
+
+cdef class node:
+    cdef list quads
+    cdef set obs
+
+    def __init__ (self, quads=None):
+        if quads is None:
+            self.quads = [None, None, None, None]
+        else:
+            self.quads = quads
+        self.obs = set()
+
+    cdef insert (self, rect * r, ob ob):
+        cdef rect parts[4]
+        cdef node q
+        if r.r - r.l <= 16:
+            # degenerate rectangle, store here
+            self.obs.add (ob)
+        else:
+            split (r, parts)
+            for i in range (4):
+                q = self.quads[i]
+                if contains (&parts[i], &ob.rect):
+                    if q is None:
+                        self.quads[i] = q = node()
+                    q.insert (&parts[i], ob)
+                    return
+        self.obs.add (ob)
+
+    # delete a particular object from the tree.
+    cdef delete (self, rect * r, ob ob):
+        cdef rect parts[4]
+        cdef node q
+        if self.obs:
+            try:
+                self.obs.remove (ob)
+                return self.obs or any (self.quads)
+            except KeyError:
+                # object not stored here
+                pass
+        split (r, parts)
+        for i in range (4):
+            q = self.quads[i]
+            if q is not None and intersects (&parts[i], &ob.rect):
+                if not q.delete (&parts[i], ob):
+                    self.quads[i] = None
+                    return self.obs or any (self.quads)
+        return self.obs or any (self.quads)
+
+    def gen_all (self):
+        cdef node x
+        if self.obs is not None:
+            for ob in self.obs:
+                yield ob
+        for i in range (4):
+            if self.quads[i] is not None:
+                x = self.quads[i]
+                for y in x.gen_all():
+                    yield y
+
+    cdef search (self, rect * tree_rect, rect * search_rect, list result):
+        cdef rect parts[4], x
+        cdef node q
+        cdef ob ob
+        split (tree_rect, parts)
+        # copy the set to avoid 'set changed size during iteration'
+        for ob in list (self.obs):
+            if intersects (search_rect, &ob.rect):
+                result.append (ob)
+        for i in range (4):
+            if self.quads[i] is not None and intersects (&parts[i], search_rect):
+                q = self.quads[i]
+                q.search (&parts[i], search_rect, result)
+
+cdef class ob:
+    "base class for objects to be stored in the quad tree"
+    cdef rect rect
+    def set_rect (self, int32_t l, int32_t t, int32_t r, int32_t b):
+        "set this object's bounding rectangle"
+        set_rect (&self.rect, l, t, r, b)
+    def get_rect (self):
+        return (self.rect.l, self.rect.t, self.rect.r, self.rect.b)
+    property rect:
+        def __get__ (self):
+            return (self.rect.l, self.rect.t, self.rect.r, self.rect.b)
+        def __set__ (self, value):
+            l,t,r,b = value
+            set_rect (&self.rect, l, t, r, b)
+    property upper_left:
+        def __get__ (self):
+            return self.rect.l, self.rect.t
+    property width:
+        def __get__ (self):
+            return self.rect.r - self.rect.l
+    property height:
+        def __get__ (self):
+            return self.rect.b - self.rect.t
+    def range_check (self, int32_t l, int32_t t, int32_t r, int32_t b):
+        return self.rect.l >= l and self.rect.r <= r and self.rect.t >= t and self.rect.b <= b
+
+cdef class quadtree:
+
+    cdef node tree
+    cdef rect rect
+    cdef uint32_t num_obs
+
+    def __init__ (self, rect=(0,0,16,16)):
+        set_rect (&self.rect, rect[0], rect[1], rect[2], rect[3])
+        self.tree = node()
+        self.num_obs = 0
+
+    def __repr__ (self):
+        return '<quad tree (objects:%d)  rect:(%d,%d,%d,%d)>' % (
+            self.num_obs,
+            self.rect.l,
+            self.rect.t,
+            self.rect.r,
+            self.rect.b,
+            )
+
+    def insert (self, ob ob):
+        cdef int32_t w, h
+        cdef node new_root
+        while not contains (&self.rect, &ob.rect):
+            w = self.rect.r - self.rect.l
+            h = self.rect.b - self.rect.t
+            # favor growing right and down
+            if (ob.rect.r > self.rect.r) or (ob.rect.b > self.rect.b):
+                # resize, placing original in the upper left
+                self.rect.r += w
+                self.rect.b += h
+                self.tree = node ([self.tree, None, None, None])
+            elif (ob.rect.l < self.rect.l) or (ob.rect.t < self.rect.t):
+                # resize, placing original in the lower right
+                self.rect.l -= w
+                self.rect.t -= h
+                self.tree = node ([None, None, None, self.tree])
+        # we know the target rect fits in our space
+        self.tree.insert (&self.rect, ob)
+        self.num_obs += 1
+
+    def gen_all (self):
+        for ob in self.tree.gen_all():
+            yield ob
+
+    def search (self, search_rect):
+        cdef list result
+        cdef rect r
+        r.l, r.t, r.r, r.b = search_rect
+        result = []
+        self.tree.search (&self.rect, &r, result)
+        return result
+
+    def delete (self, ob ob):
+        cdef rect r
+        self.tree.delete (&self.rect, ob)
+        # XXX this is bad, it assumes the object was deleted
+        self.num_obs -= 1

coro/http/demo/websocket/field/quadtree_py.py

+# -*- Mode: Python; tab-width: 4 -*-
+
+# SMR 2012: originally from dynwin, circa 1995-96.
+#   modernized, replace some funs with generators
+
+#
+# Quad-Tree.  A 2D spatial data structure.
+#
+# Used to quickly locate 'objects' within a particular region.  Each
+# node in the tree represents a rectangular region, while its children
+# represent that region split into four quadrants.  An 'object' is
+# stored at a particular level if it is contained within that region,
+# but will not fit into any of the individual quadrants inside it.
+#
+# If an object is inserted into a quadtree that 'overflows' the current
+# boundaries, the tree is re-rooted in a larger space.
+
+import region
+
+contains = region.region_contains_region_p
+intersects = region.region_intersect_p
+
+# split a rect into four quadrants
+
+def split (rect):
+    l,t,r,b = rect
+    w2 = ((r-l)/2)+l
+    h2 = ((b-t)/2)+t
+    return (
+        (l,t,w2,h2),
+        (w2,t,r,h2),
+        (l,h2,w2,b),
+        (w2,h2,r,b)
+        )
+
+# insert an object into the tree.  The object must have a
+# 'get_rect()' method in order to support searching.
+
+def insert (tree, tree_rect, ob, ob_rect):
+    quads = split(tree_rect)
+    # If tree_rect is in quads, then we've shrunk down to a
+    # degenerate rectangle, and we will store the object at
+    # this level without splitting further.
+    if tree_rect not in quads:
+        for i in range(4):
+            if contains (quads[i], ob_rect):
+                if not tree[i]:
+                    tree[i] = [None,None,None,None,set()]
+                insert (tree[i], quads[i], ob, ob_rect)
+                return
+    tree[4].add (ob)
+
+# generate all the objects intersecting with <search_rect>
+def search_gen (tree, tree_rect, search_rect):
+    quads = split (tree_rect)
+    # copy the set to avoid 'set changed size during iteration'
+    for ob in list (tree[4]):
+        if intersects (ob.get_rect(), search_rect):
+            yield ob
+    for i in range(4):
+        if tree[i] and intersects (quads[i], search_rect):
+            for ob in search_gen (tree[i], quads[i], search_rect):
+                yield ob
+
+# delete a particular object from the tree.
+
+def delete (tree, tree_rect, ob, ob_rect):
+    if tree[4]:
+        try:
+            tree[4].remove (ob)
+            return any (tree)
+        except KeyError:
+            # object not stored here
+            pass
+    quads = split (tree_rect)
+    for i in range(4):
+        if tree[i] and intersects (quads[i], ob_rect):
+            if not delete (tree[i], quads[i], ob, ob_rect):
+                tree[i] = None
+                # equivalent to "tree != [None,None,None,None,[]]"
+                return any (tree)
+    return any (tree)
+
+def gen_all (tree):
+    if tree[4]:
+        for ob in tree[4]:
+            yield ob
+    for quad in tree[:4]:
+        if quad:
+            for x in gen_all (quad):
+                yield x
+
+def dump (rect, tree, depth=0):
+    print '  ' * depth, rect,  tree[4]
+    quads = split (rect)
+    for i in range (4):
+        if tree[i]:
+            dump (quads[i], tree[i], depth+1)
+
+# wrapper for a quadtree, maintains bounds, keeps track of the
+# number of objects, etc...
+
+class quadtree:
+    def __init__ (self, rect=(0,0,16,16)):
+        self.rect = rect
+        self.tree = [None,None,None,None,set()]
+        self.num_obs = 0
+        self.bounds = (0,0,0,0)
+
+    def __repr__ (self):
+        return '<quad tree (objects:%d) bounds:%s >' % (
+            self.num_obs,
+            repr(self.bounds)
+            )
+
+    def check_bounds (self, rect):
+        l,t,r,b = self.bounds
+        L,T,R,B = rect
+        if L < l:
+            l = L
+        if T < t:
+            t = T
+        if R > r:
+            r = R
+        if B > b:
+            b = B
+        self.bounds = l,t,r,b
+
+    def get_bounds (self):
+        return self.bounds
+
+    def insert (self, ob):
+        rect = ob.get_rect()
+        while not contains (self.rect, rect):
+            l,t,r,b = self.rect
+            w, h = r-l, b-t
+            # favor growing right and down
+            if (rect[2] > r) or (rect[3] > b):
+                # resize, placing original in the upper left
+                self.rect = l, t, (r+w), (b+h)
+                self.tree = [self.tree, None, None, None, set()]
+            elif (rect[0] < l) or (rect[1] < t):
+                # resize, placing original in lower right
+                self.rect = (l-w,t-h,r,b)
+                self.tree = [None, None, None, self.tree, set()]
+        # we know the target rect fits in our space
+        insert (self.tree, self.rect, ob, rect)
+        self.check_bounds (rect)
+        self.num_obs += 1
+        
+    def gen_all (self):
+        for ob in gen_all (self.tree):
+            yield ob
+
+    def search_gen (self, rect):
+        for ob in search_gen (self.tree, self.rect, rect):
+            yield ob
+
+    def delete (self, ob):
+        # we ignore the return, because we can't 'forget'
+        # the root node.
+        delete (self.tree, self.rect, ob, ob.get_rect())
+        # XXX this is bad, it assumes the object was deleted
+        self.num_obs -= 1
+
+    def dump (self):
+        print self
+        dump (self.rect, self.tree, 0)
+
+# sample 'box' object.
+class box:
+    def __init__ (self, rect):
+        self.rect = rect
+    def get_rect (self):
+        return self.rect
+    def __repr__ (self):
+        return '<box (%d,%d,%d,%d)>' % self.rect

coro/http/demo/websocket/field/region.py

+# -*- Mode: Python; tab-width: 4 -*-
+
+# SMR 2012: originally from dynwin, circa 1995
+
+###########################################################################
+# regions
+###########################################################################
+
+#
+# do two rectangular regions intersect?
+#
+# +--->
+# |           lb,tb-------+
+# |           |           |
+# |    la,ta--+-----+     |
+# |   |       |     |     |
+# |   |       +-----+-rb,bb
+# V   |             |
+#     +--------ra,ba
+
+# a rect is (left,top,right,bottom)
+# top is < bottom, left is < right
+
+# proof: imagine all possible cases in 1 dimension,
+# (there are six) and then generalize.  simplify the
+# expression and you get this.  (trust me 8^)
+
+def region_intersect_p (a,b):
+    return (a[2] >= b[0]) and \
+           (b[2] >= a[0]) and \
+           (a[3] >= b[1]) and \
+           (b[3] >= a[1])
+
+def point_in_region_p (x,y,r):
+    return (r[0] <= x <= r[2]) and (r[1] <= y <= r[3])
+
+# does region <a> fully contain region <b>?
+def region_contains_region_p (a,b):
+    return (a[0] <= b[0]) and \
+           (a[2] >= b[2]) and \
+           (a[1] <= b[1]) and \
+           (a[3] >= b[3])
+
+def union (a, b):
+    x0, y0, x1, y1 = a
+    x2, y2, x3, y3 = b
+    return (
+        min (x0, x2),
+        min (y0, y2),
+        max (x1, x3),
+        max (y1, y3)
+        )

coro/http/demo/websocket/field/setup.py

+from distutils.core import setup
+from distutils.extension import Extension
+from Cython.Distutils import build_ext
+
+setup (
+    name='quadtree',
+    description='Quad Tree - Spatial Search Data Structure',
+    cmdclass = {'build_ext': build_ext},
+    ext_modules = [Extension ("quadtree", ["quadtree.pyx"],)]
+    )

coro/http/demo/websocket/simple/simple.html

+<!doctype html>
+<html>
+<head>
+<title>WebSocket Test</title>
+<meta charset="utf-8">
+</head>
+<body>
+<canvas id="canvas" width="1024" height="1024"></canvas>
+<div id="outer_div">
+  <table style="width: 100%; padding: 10px;">
+    <tr><td>
+	Draw with your mouse.
+    </td></tr>
+    <tr><td id="messages" style="vertical-align: bottom">
+    <div id="msgs" style="overflow: hidden"> </div>
+    </td></tr>
+    <tr><td>
+      <button type="button" onclick="connection.send('CD')">New Drawing</button>
+      <button type="button" onclick="connection.send('PD')">Previous Drawing</button>
+      <button type="button" onclick="connection.send('ND')">Next Drawing</button>
+    </td></tr>
+    <tr><td>
+      Keyboard shortcuts: [U]ndo [R]edraw
+    </td></tr>
+  </table>
+</div>
+<script src="simple.js"></script>
+</body>
+</html>

coro/http/demo/websocket/simple/simple.js

+
+var connection;
+var msgs_div = document.getElementById ('msgs')
+
+function message (msg) {
+    msgs_div.innerHTML = msg;
+}
+
+if (window["WebSocket"]) {
+    connection = new WebSocket("ws://dark.nightmare.com:9001/sketch")
+
+    connection.onopen = function () {
+	message ('connected')
+    };
+
+    connection.onclose = function (event) {
+	message ('disconnected')
+    }
+    
+    connection.onmessage = function (event) {
+	message ('websocket event: ' + event.data)
+	var data = event.data.split (',')
+	switch (data[0]) {
+	case 'D':
+	    draw (parseInt(data[1]), parseInt(data[2]), parseInt(data[3]), parseInt(data[4]))
+	    break;
+	case 'E':
+	    undraw (parseInt(data[1]), parseInt(data[2]), parseInt(data[3]), parseInt(data[4]))
+	    break;
+	case 'CD':
+	    context.fillStyle = 'rgb(255, 255, 255)'
+	    context.fillRect (0, 0, 1024, 1024)
+	    break;
+	}
+    }
+}
+
+var canvas = document.getElementById ('canvas')
+canvas.style.cursor = 'crosshair'
+var context = canvas.getContext ('2d')
+
+function draw (x0, y0, x1, y1) {
+    context.strokeStyle = 'rgba(0,0,0,255)'
+    context.beginPath()
+    context.moveTo (x0, y0)
+    context.lineTo (x1, y1)
+    context.closePath()
+    context.stroke()
+}
+
+function undraw (x0, y0, x1, y1) {
+    context.strokeStyle = 'rgba(255,255,255,255)'
+    context.beginPath()
+    context.moveTo (x0, y0)
+    context.lineTo (x1, y1)
+    context.closePath()
+    context.stroke()
+}
+
+document.addEventListener ('mousedown', on_mouse_down, false)
+document.addEventListener ('mouseup', on_mouse_up, false)
+document.addEventListener ('mousemove', on_mouse_move, false)
+document.addEventListener ('keydown', on_key_down, false)
+document.addEventListener ('keyup', on_key_up, false)
+
+function clear_drawing() {
+    connection.send ('CD')
+}
+function on_mouse_down (event) {
+    connection.send ('MD,' + event.clientX + ',' + event.clientY)
+}
+function on_mouse_up (event) {
+    connection.send ('MU,' + event.clientX + ',' + event.clientY)
+}
+function on_mouse_move (event) {
+    connection.send ('MM,' + event.clientX + ',' + event.clientY)
+}
+function on_key_down (event) {
+    connection.send ('KD,' + event.keyCode)
+}
+function on_key_up (event) {
+    connection.send ('KU,' + event.keyCode)
+}

coro/http/demo/websocket/simple/simple.py

+# -*- Mode: Python -*-
+
+from coro.http.websocket import handler, websocket
+import pickle
+import re
+
+import coro
+W = coro.write_stderr
+
+def timestamp():
+    return coro.now / 1000000
+
+drawing_re = re.compile ('drawing_([0-9]+).bin')
+draw_re = re.compile ('D,([0-9]+),([0-9]+),([0-9]+),([0-9]+)')
+
+class server:
+
+    def __init__ (self):
+        self.clients = set()
+        self.dead = set()
+        self.drawing = []
+        self.timestamp = timestamp()
+        all = self.get_all_drawings()
+        if len (all):
+            self.set_drawing (all[0])
+
+    def new_session (self, proto, http_request):
+        client = sketch_conn (self, proto, http_request)
+        self.clients.add (client)
+        for payload in self.drawing:
+            client.send_text (payload)
+
+    def clear_drawing (self):
+        self.save_drawing()
+        self.broadcast ('CD', False)
+
+    def get_all_drawings (self):
+        import os
+        files = os.listdir ('.')
+        r = []
+        for path in files:
+            m = drawing_re.match (path)
+            if m:
+                stamp = int (m.group(1))
+                r.append (stamp)
+        r.sort()
+        return r
+
+    def get_path (self, stamp):
+        return 'drawing_%d.bin' % (stamp,)
+
+    def save_drawing (self):
+        if len(self.drawing):
+            W ('saving %r\n' % (self.timestamp,))
+            f = open (self.get_path (self.timestamp), 'wb')
+            drawing, self.drawing = self.drawing, []
+            self.timestamp = timestamp()
+            pickle.dump (drawing, f)
+            f.close()
+        else:
+            W ('empty drawing [no save]\n')
+
+    def next_drawing (self):
+        all = self.get_all_drawings()
+        for t in all:
+            if t > self.timestamp:
+                self.set_drawing (t)
+                return
+
+    def prev_drawing (self):
+        all = self.get_all_drawings()
+        all.reverse()
+        for t in all:
+            if t < self.timestamp:
+                self.set_drawing (t)
+                return
+
+    def set_drawing (self, stamp):
+        self.save_drawing()
+        self.timestamp = stamp
+        self.drawing = pickle.load (open (self.get_path (stamp)))
+        self.broadcast ('CD', False)
+        W ('set drawing %d [%d lines]\n' % (stamp, len(self.drawing)))
+        for payload in self.drawing:
+            self.broadcast (payload, False)
+        W ('done\n')
+
+    def broadcast (self, payload, save=True):
+        if save:
+            self.drawing.append (payload)
+        # copy to avoid "Set changed size during iteration"
+        for client in list (self.clients):
+            try:
+                client.send_text (payload)
+            except:
+                self.dead.add (client)
+                tb = coro.compact_traceback()
+                W ('error: tb=%r' % (tb,))
+        self.clients.difference_update (self.dead)
+        self.dead = set()
+
+    def undo (self):
+        if len (self.drawing):
+            last = self.drawing.pop()
+            m = draw_re.match (last)
+            if m:
+                self.broadcast ('E,%s,%s,%s,%s' % m.groups(), save=False)
+
+class sketch_conn (websocket):
+
+    def __init__ (self, server, *args, **kwargs):
+        self.send_mutex = coro.mutex()
+        self.server = server
+        self.mouse_down = False
+        self.line_start = None, None
+        websocket.__init__ (self, *args, **kwargs)
+
+    def handle_close (self):
+        self.server.dead.add (self)
+
+    def send_text (self, payload):
+        with self.send_mutex:
+            websocket.send_text (self, payload)
+
+    def handle_packet (self, p):
+        event = p.unpack().split (',')
+        #W ('packet = %r event=%r\n' % (p, event))
+        if event[0] == 'MD':
+            self.mouse_down = True
+            self.line_start = int (event[1]), int (event[2])
+        elif event[0] == 'MU':
+            self.mouse_down = False
+        elif event[0] == 'MM':
+            if self.mouse_down:
+                x1, y1 = int (event[1]), int (event[2])
+                self.server.broadcast (
+                    'D,%d,%d,%s,%s' % (
+                        self.line_start[0], self.line_start[1], x1, y1
+                        )
+                    )
+                self.line_start = x1, y1
+        elif event[0] == 'CD':
+            self.server.clear_drawing()
+        elif event[0] == 'ND':
+            self.server.next_drawing()
+        elif event[0] == 'PD':
+            self.server.prev_drawing()
+        elif event[0] == 'KD':
+            if event[1] == '85': # 'U'
+                self.server.undo()
+            elif event[1] == '82': # 'R'
+                self.server.set_drawing (self.server.timestamp)
+        else:
+            W ('unknown event: %r\n' % (event,))
+        return False
+
+if __name__ == '__main__':
+    import coro.http
+    import coro.backdoor
+    import os
+    cwd = os.getcwd()
+    sketch_server = server()
+    ih = coro.http.handlers.favicon_handler()
+    sh = coro.http.handlers.coro_status_handler()
+    wh = handler ('/sketch', sketch_server.new_session)
+    fh = coro.http.handlers.file_handler (cwd)
+    handlers = [wh, ih, sh, fh]
+    #server = coro.http.server (('0.0.0.0', 9001))
+    server = coro.http.server()
+    for h in handlers:
+        server.push_handler (h)
+    #coro.spawn (server.start)
+    coro.spawn (server.start, ('0.0.0.0', 9001))
+    coro.spawn (coro.backdoor.serve, unix_path='/tmp/ws.bd')
+    coro.event_loop (30.0)

coro/http/demo/websocket/term/term.html

+<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML//EN">
+<html> <head>
+<title></title>
+</head>
+<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.7.2/jquery.min.js"></script>
+<body>
+<h1></h1>
+
+<div id="msgs"></div>
+<pre width="99%"><span id="term">Terminal.
+</span><span id="input" tabindex="0"></span><span id="cursor" style="background-color:green"> </span></pre>
+<script src="term.js"></script>
+</body>
+</html>

coro/http/demo/websocket/term/term.js

+
+var connection;
+var msgs_div = document.getElementById ('msgs')
+
+function message (msg) {
+    msgs_div.innerHTML = msg;
+}
+
+if (window["WebSocket"]) {
+    connection = new WebSocket ("ws://127.0.0.1:9001/term")
+
+    connection.onopen = function () {
+	message ('connected')
+    };
+
+    connection.onclose = function (event) {
+	message ('disconnected')
+    }
+    
+    connection.onmessage = function (event) {
+	message ('event: ' + event.data[0])
+	var data = event.data;
+	switch (data[0]) {
+	case 'D':
+	    $('#term').append (data.substring (1, data.length));
+	    break;
+	case 'I':
+	    $('#input').append (data.substring (1, data.length));
+	    break;
+	case 'B':
+	    $('#input').html (data.substring (1, data.length));
+	    break;
+	case 'C':
+	    console.log ('clearing?');
+	    $('#input').empty();
+	    break;
+	}
+    }
+}
+
+var term = document.getElementById ('term');
+
+$(document).keydown (
+    function() {
+	if (event.which == 8) {
+	    // backspace
+	    connection.send ('B');
+	}
+    }
+);
+
+$(document).keypress (
+    function () {
+	connection.send ('K,' + event.which);
+    }
+);
+

coro/http/demo/websocket/term/term.py

+# -*- Mode: Python -*-
+
+from coro.http.websocket import handler, websocket
+import pickle
+import pprint
+import re
+
+import coro
+W = coro.write_stderr
+
+# python3 includes an 'html' module that does this, but it'd be nice
+#   if we could just stream this transform somehow... maybe using the
+#   string StreamWriter stuff?
+
+def escape (s):
+    return s.replace ('&', '&amp;').replace ('<', '&lt;').replace ('>', '&gt;')
+
+class terminal (websocket):
+
+    def __init__ (self, *args, **kwargs):
+        websocket.__init__ (self, *args, **kwargs)
+        self.send_mutex = coro.mutex()
+        self.repl = repl (self)
+        self.history = []
+        self.history_index = 0
+        self.line = []
+        try:
+            self.repl.read_eval_print_loop()
+        except Logout:
+            pass
+
+    def handle_close (self):
+        pass
+
+    def send_text (self, payload):
+        with self.send_mutex:
+            websocket.send_text (self, payload)
+
+    def handle_packet (self, p):
+        data = p.unpack()
+        event = p.unpack().split (',')
+        #W ('packet = %r event=%r\n' % (p, event))
+        if event[0] == 'K':
+            ascii = int (event[1])
+            #W ('ascii=%d\n' % (ascii,))
+            if ascii in (10, 13): # lf cr
+                ascii = 10
+                line = ''.join (self.line)
+                self.history.append (line)
+                self.line = []
+                self.send_text ('C')
+                self.send_text ('D' + escape (line) + '\n')
+                self.repl.inlines.push (line)
+            elif ascii == 4: # ctrl-d
+                self.repl.inlines.push (None)
+            elif ascii == 16: # ctrl-p
+                self.history_index = (self.history_index + 1) % len(self.history)
+                line = self.history[0 - self.history_index]
+                # turn into a list of chars...
+                self.line = [x for x in line]
+                self.send_text ('B' + escape (line))
+            else:
+                self.line.append (chr (ascii))
+                self.send_text ('I' + escape (chr (ascii)))
+        elif event[0] == 'B':
+            if len(self.line):
+                del self.line[-1]
+                self.send_text ('B' + ''.join (self.line))
+        else:
+            W ('unknown event: %r\n' % (event,))
+        return False
+
+from coro.backdoor import backdoor
+
+class NotAuthorized (Exception):
+    pass
+
+class repl (backdoor):
+
+    def __init__ (self, term):
+        backdoor.__init__ (self, term, '\n')
+        self.inlines = coro.fifo()
+
+    def login (self):
+        self.send ('Username: ')
+        u = self.read_line()
+        self.send ('Password: ')
+        p = self.read_line()
+        # XXX self.sock should really be called self.conn
+        if self.sock.handler.auth_dict.get (u, None) != p:
+            coro.sleep_relative (3)
+            self.send ('Sorry, Charlie\n')
+            self.sock.conn.close()
+            raise NotAuthorized (u)
+
+    def print_result (self, result):
+        pprint.pprint (result)
+
+    def read_line (self):
+        line = self.inlines.pop()
+        if line is None:
+            self.sock.conn.close()
+        else:
+            return line
+
+    def send (self, data):
+        # Note: sock is really a terminal object
+        self.sock.send_text ('D' + escape (data))
+
+if __name__ == '__main__':
+    import coro.http
+    import coro.backdoor
+    import os
+    cwd = os.getcwd()
+    ih = coro.http.handlers.favicon_handler()
+    sh = coro.http.handlers.coro_status_handler()
+    th = handler ('/term', terminal)
+    th.auth_dict = {'foo':'bar'}
+    fh = coro.http.handlers.file_handler (cwd)
+    handlers = [th, ih, sh, fh]
+    #server = coro.http.server (('0.0.0.0', 9001))
+    server = coro.http.server()
+    #server = coro.http.tlslite_server (
+    #    '/home/rushing/src/spdy/cert/server.crt',
+    #    '/home/rushing/src/spdy/cert/server.key',
+    #    )
+    for h in handlers:
+        server.push_handler (h)
+    #coro.spawn (server.start)
+    coro.spawn (server.start, ('0.0.0.0', 9001))
+    coro.spawn (coro.backdoor.serve, unix_path='/tmp/ws.bd')
+    coro.event_loop (30.0)

coro/http/websocket.py

+# -*- Mode: Python -*-
+
+import base64
+import struct
+import coro
+import os
+import sys
+import hashlib
+
+W = coro.write_stderr
+
+from coro.http.protocol import HTTP_Upgrade
+from coro import read_stream
+
+# RFC 6455
+
+class WebSocketError (Exception):
+    pass
+
+class TooMuchData (WebSocketError):
+    pass
+
+class UnknownOpcode (WebSocketError):
+    pass
+
+def do_mask (data, mask):
+    n = len (data)
+    r = bytearray (n)
+    i = 0
+    while i < len (data):
+        r[i] = chr (ord (data[i]) ^ mask[i%4])
+        i += 1
+    return bytes (r)
+
+class ws_packet:
+    fin = 0
+    opcode = 0
+    mask = 0
+    plen = 0
+    masking = []
+    payload = ''
+    def __repr__ (self):
+        return '<fin=%r opcode=%r mask=%r plen=%r masking=%r payload=%d bytes>' % (
+            self.fin,
+            self.opcode,
+            self.mask,
+            self.plen,
+            self.masking,
+            len (self.payload),
+            )
+
+    def unpack (self):
+        if self.mask:
+            return do_mask (self.payload, self.masking)
+        else:
+            return self.payload
+
+class handler:
+
+    magic = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"
+
+    def __init__ (self, path, handler):
+        self.path = path
+        self.handler = handler
+                  
+    def match (self, request):
+        # try to catch both versions of the protocol
+        return (
+            request.path == self.path
+            and request.method == 'get'
+            and request['upgrade']
+            and request['upgrade'].lower() == 'websocket'
+            )
+
+    def h76_frob (self, key):
+        digits = int (''.join ([ x for x in key if x in '0123456789' ]))
+        spaces = key.count (' ')
+        return digits / spaces
+
+    def handle_request (self, request):
+        rh = request.request_headers
+        key = rh.get_one ('sec-websocket-key')
+        conn = request.client.conn
+        if key:
+            d = hashlib.new ('sha1')
+            d.update (key + self.magic)
+            reply = base64.encodestring (d.digest()).strip()
+            r = [
+                'HTTP/1.1 101 Switching Protocols',
+                'Upgrade: websocket',
+                'Connection: Upgrade',
+                'Sec-WebSocket-Accept: %s' % (reply,),
+                ]
+            if rh.has_key ('sec-websocket-protocol'):
+                # XXX verify this
+                r.append (
+                    'Sec-WebSocket-Protocol: %s' % (
+                        rh.get_one ('sec-websocket-protocol')
+                        )
+                    )
+            conn.send ('\r\n'.join (r) + '\r\n\r\n')
+            protocol = 'rfc6455'
+        else:
+            # for Safari, this implements the obsolete hixie-76 protocol
+            # http://tools.ietf.org/html/draft-hixie-thewebsocketprotocol-76
+            key1 = self.h76_frob (rh.get_one ('sec-websocket-key1'))
+            key2 = self.h76_frob (rh.get_one ('sec-websocket-key2'))
+            tail = request.client.stream.read_exact (8)
+            key = struct.pack ('>L', key1) + struct.pack ('>L', key2) + tail
+            d = hashlib.new ('md5')
+            d.update (key)
+            reply = d.digest()
+            host = rh.get_one ('host')
+            r = [
+                'HTTP/1.1 101 WebSocket Protocol Handshake',
+                'Upgrade: WebSocket',
+                'Connection: Upgrade',
+                'Sec-WebSocket-Origin: http://%s' % (host,),
+                'Sec-WebSocket-Location: ws://%s%s' % (host, request.uri),
+                ]
+            all = '\r\n'.join (r) + '\r\n\r\n' + reply
+            conn.send (all)
+            protocol = 'hixie_76'
+        # pass this websocket off to its new life...
+        coro.spawn (self.handler, protocol, request, self)
+        raise HTTP_Upgrade
+
+class websocket:
+
+    def __init__ (self, proto, http_request, handler):
+        self.request = http_request
+        self.handler = handler
+        self.stream = http_request.client.stream
+        self.conn = http_request.client.conn
+        self.proto = proto
+        if proto == 'rfc6455':
+            coro.spawn (self.read_thread)
+        else:
+            coro.spawn (self.read_thread_hixie_76)
+
+    # ------------ RFC 6455 ------------
+    def read_thread (self):
+        close_it = False
+        try:
+            while 1:
+                try:
+                    close_it = coro.with_timeout (30, self.read_packet)
+                except coro.TimeoutError:
+                    self.send_pong ('bleep')
+                except coro.ClosedError:
+                    break
+                if close_it:
+                    break
+        finally:
+            self.handle_close()
+            self.conn.close()
+        
+    def read_packet (self):
+        head = self.stream.read_exact (2)
+        if not head:
+            return True
+        head, = struct.unpack ('>H', head)
+        p = ws_packet()
+        p.fin    = (head & 0x8000) >> 15
+        p.opcode = (head & 0x0f00) >> 8
+        p.mask   = (head & 0x0080) >> 7
+        plen     = (head & 0x007f) >> 0
+        if plen < 126:
+            pass
+        elif plen == 126:
+            plen, = struct.unpack ('>H', self.stream.read_exact (2))
+        else: # plen == 127:
+            plen, = struct.unpack ('>Q', self.stream.read_exact (8))
+        p.plen = plen
+        if plen > 1<<20:
+            raise TooMuchData (plen)
+        if p.mask:
+            p.masking = struct.unpack ('>BBBB', self.stream.read_exact (4))
+        else:
+            p.masking = None
+        p.payload = self.stream.read_exact (plen)
+        if p.opcode in (0, 1, 2):
+            return self.handle_packet (p)
+        elif p.opcode == 8:
+            # close
+            return True
+        elif p.opcode == 9:
+            # ping
+            assert (p.fin) # probably up to no good...
+            self.send_pong (self, p.payload)
+            return False
+        else:
+            raise UnknownOpcode (p)
+
+    # ----------- hixie-76 -------------
+    def read_thread_hixie_76 (self):
+        self.stream = self.request.client.stream
+        close_it = False
+        try:
+            while 1:
+                try:
+                    close_it = coro.with_timeout (30, self.read_packet_hixie_76)
+                except coro.TimeoutError:
+                    self.send_pong ('bleep')
+                except coro.ClosedError:
+                    break
+                if close_it:
+                    break
+        finally:
+            self.conn.close()
+        
+    def read_packet_hixie_76 (self):
+        ftype = self.stream.read_exact (1)
+        if not ftype:
+            return True
+        ftype = ord (ftype)
+        if ftype & 0x80:
+            length = 0
+            while 1:
+                b = ord (self.stream.read_exact (1))
+                length = (length << 7) | (b & 0x7f)
+                if not b & 0x80:
+                    break
+                if length > 1<<20:
+                    raise TooMuchData (length)
+            if length:
+                payload = self.stream.read_exact (length)
+            if ftype == 0xff:
+                return True
+        else:
+            data = self.stream.read_until (b'\xff')
+            if ftype == 0x00:
+                p = ws_packet()
+                p.fin = 1
+                p.opcode = 0x01
+                p.mask = None
+                p.payload = data[:-1]
+                self.handle_packet (p)
+        
+    # ---
+
+    def handle_packet (self, p):
+        # abstract method, override to implement your own logic
+        return False
+
+    def handle_close (self):
+        # abstract method
+        pass
+
+    def send_text (self, data, fin=True):
+        return self.send_packet (0x01, data, fin)
+
+    def send_binary (self, data, fin=True):
+        return self.send_packet (0x02, data, fin)
+
+    def send_pong (self, data):
+        return self.send_packet (0x0a, data, True)
+
+    def send_packet (self, opcode, data, fin=True):
+        if self.proto == 'rfc6455':
+            head = 0
+            if fin:
+                head |= 0x8000
+            assert opcode in (0, 1, 2, 8, 9, 10)
+            head |= opcode << 8
+            ld = len (data)
+            if ld < 126:
+                head |= ld
+                p = [ struct.pack ('>H', head), data ]
+            elif ld < 1<<16:
+                head |= 126
+                p = [ struct.pack ('>HH', head, ld), data ]
+            elif ld < 1<<32:
+                head |= 127
+                p = [ struct.pack ('>HQ', head, ld), data ]
+            else:
+                raise TooMuchData (ld)
+            # RFC6455: A server MUST NOT mask any frames that it sends to the client.
+            self.conn.writev (p)
+        else:
+            self.conn.writev (['\x00', data, '\xff'])

coro/ldap/test/t0.py

+# -*- Mode: Python -*-
+
+import unittest
+import sys
+from coro.asn1.ber import *
+from coro.ldap.query import *
+
+C = 'context'
+
+pq_tests = [
+    # simple equality
+    ('(xxx=yyy)',
+     ((C, 3, ['xxx', 'yyy']),
+      12)),
+    # simple expression, plus 'present'
+    ('(|(xx=y)(zz=*))',
+     ((C, 1, [(C, 3, ['xx', 'y']), (C, 7, 'zz')]),
+      15)),
+    # nary expressions
+    ('(|(a=b)(b=c)(c=d)(e=f)(f=g)(h=i))',
+     ((C, 1, [(C, 3, ['a', 'b']), (C, 3, ['b', 'c']), (C, 3, ['c', 'd']), (C, 3, ['e', 'f']), (C, 3, ['f', 'g']), (C, 3, ['h', 'i'])]),
+      50)),
+    ('(|(!(a=*))(&(b=c)(d=e))(x<=y))',
+     ((C, 1, [(C, 2, [(C, 7, 'a')]), (C, 0, [(C, 3, ['b', 'c']), (C, 3, ['d', 'e'])]), (C, 6, ['x', 'y'])]),
+      33)),
+    # approximate match
+    ('(zz~=yy)', ((C, 8, ['zz', 'yy']), 10)),
+    # substring
+    ('(a=ins*tiga*tor)', ((C, 4, ['a', [(C, 0, 'ins'), (C, 1, 'tiga'), (C, 2, 'tor')]]), 23)),
+    ('(a=*y)', ((C, 4, ['a', [(C, 2, 'y')]]), 10)),
+    ('(a=y*)', ((C, 4, ['a', [(C, 0, 'y')]]), 10)),
+    ('(a=*y*)', ((C, 4, ['a', [(C, 1, 'y')]]), 10)),
+    ('(a=*x*y)', ((C, 4, ['a', [(C, 1, 'x'), (C, 2, 'y')]]), 13)),
+    ('(a=*x*y*)', ((C, 4, ['a', [(C, 1, 'x'), (C, 1, 'y')]]), 13)),
+    ('(a=*x*y*z)', ((C, 4, ['a', [(C, 1, 'x'), (C, 1, 'y'), (C, 2, 'z')]]), 16)),
+    # syntax errors
+    ('(a=', QuerySyntaxError),
+    ('(a<b)', QuerySyntaxError),
+    # good hex escape
+    ('(a=some\\AAthing)',((C, 3, ['a', 'some\252thing']), 17)),
+    # bad hex escape
+    ('(a=some\\AZthing)', QuerySyntaxError),
+    # upper/lower case hex escape
+    ('(a=xy\\Aaz)', ((C, 3, ['a', 'xy\252z']), 11)),
+    # escaped splat
+    ('(a=x*y\\2az)', ((C, 4, ['a', [(C, 0, 'x'), (C, 2, 'y*z')]]), 15)),
+    # illegal splat
+    ('(a~=sam*son)', QuerySyntaxError),
+    # junk/illegal
+    ('junk', QuerySyntaxError),
+    # lots of parens
+    (('('*100), QuerySyntaxError),
+    # expression too complex
+    (('(!' * 55) + '(x=y)' + (')' * 55), QuerySyntaxError),
+    # expression not too complex
+    (('(!' * 10) + '(x=y)' + (')' * 10),
+     ((C, 2, [(C, 2, [(C, 2, [(C, 2, [(C, 2, [(C, 2, [(C, 2, [(C, 2, [(C, 2, [(C, 2, [(C, 3, ['x', 'y'])])])])])])])])])])]),
+      28)),
+    ]
+
+class parse_query_test (unittest.TestCase):
+    def runTest (self):
+        for q, e in pq_tests:
+            try:
+                self.assertEqual (decode (parse_query (q)), e)
+            except AssertionError:
+                raise
+            except:
+                self.assertEqual (sys.exc_info()[0], e)
+
+def suite():
+    suite = unittest.TestSuite()
+    suite.addTest (parse_query_test())
+    return suite
+
+if __name__ == '__main__':
+    unittest.main (defaultTest='suite')

coro/sync.pyx

@@ -374,6 +374,10 @@ cdef class mutex:
                 # nothing to schedule
                 return False
 
+    # for 'with'
+    __enter__ = lock
+    def __exit__ (self, t, v, tb):
+        self.unlock()
 
 # ===========================================================================
 #                         Read/Write Lock

setup.py

@@ -104,6 +104,8 @@ setup (
             ['coro/oserrors.pyx', ],
             ),
         Extension ('coro.dns.packet', ['coro/dns/packet.pyx', ],),
+        Extension ('coro.asn1.ber', ['coro/asn1/ber.pyx'],),
+        Extension ('coro.ldap.query', ['coro/ldap/query.pyx'],),
         Extension (
             'coro.clocks.tsc_time',
             ['coro/clocks/tsc_time.pyx', ],
@@ -114,12 +116,13 @@ setup (
                 ],
             ),
         ],
-    packages=['coro', 'coro.clocks', 'coro.http', 'coro.dns'],
+    packages=['coro', 'coro.clocks', 'coro.http', 'coro.dns', 'coro.emulation'],
     package_dir = {
 #        '': 'coroutine',
         'coro': 'coro',
         'coro.clocks': 'coro/clocks',
         'coro.dns': 'coro/dns',
+        'coro.emulation': 'coro/emulation',
     },
     py_modules = ['backdoor', 'coro.read_stream', 'coro_process', 'coro_unittest',],
     download_url = 'http://github.com/ironport/shrapnel/tarball/master#egg=coro-1.0.2',