Merge branch 'new-httpd'

Conflicts:
	setup.py

coro/http/__init__.py

+# -*- Mode: Python -*-
+
+from server import server, tlslite_server
+import handlers
+import coro
+from coro import read_stream
+import http_date
+import session_handler

coro/http/client.py

+# -*- Mode: Python -*-
+
+import coro
+import coro.read_stream
+from protocol import http_file, header_set, latch
+
+W = coro.write_stderr
+
+class HTTP_Protocol_Error (Exception):
+    pass
+
+# viewed at its core, HTTP is a two-way exchange of messages,
+#   some of which may have content associated with them.
+
+# two different usage patterns for pipelined requests:
+# 1) requests are made by different threads
+# 2) requests are made by a single thread
+#
+# we accommodate both patterns here.
+# for #2, use the lower-level send_request() method, for #1,
+#   use GET, PUT, etc...
+
+class request:
+
+    def __init__ (self, force=True):
+        self.latch = latch()
+        self.force = force
+        self.content = None
+        self.response = None
+        self.rheader = None
+        self.rfile = None
+
+    def wake (self):
+        if self.rfile and self.force:
+            self.content = self.rfile.read()
+        self.latch.wake_all()
+        if self.rfile and not self.force:
+            self.rfile.wait()
+
+    def wait (self):
+        return self.latch.wait()
+
+class client:
+
+    def __init__ (self, host, port=80, conn=None, inflight=100):
+        self.host = host
+        self.inflight = coro.semaphore (inflight)
+        if conn is None:
+            self.conn = coro.tcp_sock()
+            self.conn.connect ((host, port))
+        else:
+            self.conn = conn
+        self.stream = coro.read_stream.sock_stream (self.conn)
+        self.pending = coro.fifo()
+        coro.spawn (self.read_thread)
+
+    def read_thread (self):
+        while 1:
+            req = self.pending.pop()
+            self._read_message (req)
+            if not req.response:
+                break
+            else:
+                req.wake()
+
+    def _read_message (self, req):
+        req.response = self.stream.read_line()[:-2]
+        lines = []
+        while 1:
+            line = self.stream.read_line()
+            if not line:
+                raise HTTP_Protocol_Error ('unexpected close')
+            elif line == '\r\n':
+                break
+            else:
+                lines.append (line[:-2])
+        req.rheader = h = header_set (lines)
+        if h['content-length'] or h['transfer-encoding']:
+            req.rfile = http_file (h, self.stream)
+
+    def send_request (self, method, uri, headers, content=None, force=False):
+        try:
+            self.inflight.acquire (1)
+            req = request (force)
+            self._send_request (method, uri, headers, content)
+            self.pending.push (req)
+            return req
+        finally:
+            self.inflight.release (1)
+
+    def _send_request (self, method, uri, headers, content):
+        if not headers.has_key ('host'):
+            headers['host'] = self.host
+        if content:
+            if type(content) is str:
+                headers['content-length'] = len(content)
+            elif not headers.has_key ('content-length'):
+                headers['transfer-encoding'] = 'chunked'
+        req = (
+            '%s %s HTTP/1.1\r\n'
+            '%s\r\n' % (method, uri, headers)
+            )
+        self.conn.send (req)
+        # XXX 100 continue
+        if content:
+            if type(content) is str:
+                self.conn.send (content)
+            elif headers.has_key ('content-length'):
+                clen = int (headers.get_one ('content-length'))
+                slen = 0
+                for block in content:
+                    self.conn.send (block)
+                    slen += len(block)
+                    if slen > clen:
+                        raise HTTP_Protocol_Error ("content larger than declared length", clen, slen)
+                else:
+                    if slen != clen:
+                        raise HTTP_Protocol_Error ("content smaller than declared length", clen, slen)
+            else:
+                # chunked encoding
+                for block in content:
+                    if block:
+                        self.conn.writev (['%x\r\n' % (len (block),), block])
+                self.conn.send ('0\r\n')
+
+    def GET (self, uri, **headers):
+        headers = header_set().from_keywords (headers)
+        req = self.send_request ('GET', uri, headers, force=True)
+        req.wait()
+        return req
+
+    def GET_file (self, uri, **headers):
+        headers = header_set().from_keywords (headers)
+        req = self.send_request ('GET', uri, headers, force=False)
+        req.wait()
+        return req
+
+    def PUT (self, uri, content, **headers):
+        headers = header_set().from_keywords (headers)
+        req = self.send_request ('PUT', uri, headers, content, force=True)
+        req.wait()
+        return req
+
+    def POST (self, uri, content, **headers):
+        headers = header_set().from_keywords (headers)
+        req = self.send_request ('POST', uri, headers, content, force=True)
+        req.wait()
+        return req
+    
+    

coro/http/demo/grid.py

+# -*- Mode: Python -*-
+
+import coro
+import coro.http
+import backdoor
+
+# toy: move an X through a grid.
+# tests: POST data, compression, persistent connections, shared state
+
+import sys
+W = sys.stderr.write
+
+class grid_handler:
+
+    def __init__ (self, w, h):
+        self.w = w
+        self.h = h
+        self.grid = [['.' for x in range (w)] for y in range (h)]
+        self.pos = [w/2, h/2]
+        self.grid[self.pos[1]][self.pos[0]] = 'X'
+
+    def match (self, request):
+        return request.path.startswith ('/grid')
+
+    def handle_request (self, request):
+        if request.path == '/grid/source':
+            request['content-type'] = 'text/plain'
+            request.set_deflate()
+            request.push (open ('grid.py', 'rb').read())
+            request.done()
+            return
+        request['content-type'] = 'text/html'
+        request.set_deflate()
+        if request.file:
+            data = request.file.read()
+            pairs = [ x.split('=') for x in data.split ('&') ]
+            for k, v in pairs:
+                if k == 'dir':
+                    x0, y0 = self.pos
+                    x1, y1 = self.pos
+                    if v == 'left':
+                        x1 = max (x0-1, 0)
+                    elif v == 'right':
+                        x1 = min (x0+1, self.w-1)
+                    elif v == 'up':
+                        y1 = max (y0-1, 0)
+                    elif v == 'down':
+                        y1 = min (y0+1, self.h-1)
+                    else:
+                        pass
+                    self.grid[y0][x0] = '*'
+                    self.grid[y1][x1] = 'X'
+                    self.pos = [x1, y1]
+        else:
+            pass
+        l = []
+        for y in self.grid:
+            l.append (''.join (y))
+        request.push ('<pre>')
+        request.push ('\n'.join (l))
+        request.push ('\n</pre>\n')
+        request.push (
+            '<form name="input" action="grid" method="post">'
+            '<input type="submit" name="dir" value="left" />'
+            '<input type="submit" name="dir" value="right" />'
+            '<input type="submit" name="dir" value="up" />'
+            '<input type="submit" name="dir" value="down" />'
+            '</form>'
+            '<a href="/grid/source">source for this handler</a>'
+            )
+        request.done()
+
+server = coro.http.server()
+server.push_handler (grid_handler (50, 30))
+server.push_handler (coro.http.handlers.coro_status_handler())
+server.push_handler (coro.http.handlers.favicon_handler())
+coro.spawn (server.start, ('0.0.0.0', 9001))
+coro.spawn (backdoor.serve, unix_path='/tmp/httpd.bd')
+coro.event_loop (30.0)

coro/http/demo/session.py

+# -*- Mode: Python -*-
+
+import coro
+import coro.http
+import backdoor
+
+# demonstrate the session handler
+
+import sys
+W = sys.stderr.write
+
+def session (sid, fifo):
+    i = 0
+    while 1:
+        try:
+            # wait a half hour for a new hit
+            request = coro.with_timeout (1800, fifo.pop)
+        except coro.TimeoutError:
+            break
+        else:
+            request['content-type'] = 'text/html'
+            if i == 10:
+                request.push (
+                    '<html><h1>Session Over!  Bye!</h1>'
+                    '<a href="session">start over</a>'
+                    '</html>'
+                    )
+                request.done()
+                break
+            else:
+                request.push (
+                    '<html><h1>Session Demo</h1><br><h2>Hit=%d</h2>'
+                    '<a href="session">hit me!</a>'
+                    '</html>' % (i,)
+                    )
+                request.done()
+            i += 1
+
+server = coro.http.server()
+server.push_handler (coro.http.handlers.coro_status_handler())
+server.push_handler (coro.http.session_handler.session_handler ('session', session))
+server.push_handler (coro.http.handlers.favicon_handler())
+coro.spawn (server.start, ('0.0.0.0', 9001))
+coro.spawn (backdoor.serve, unix_path='/tmp/httpd.bd')
+coro.event_loop (30.0)

coro/http/demo/tclient.py

+# -*- Mode: Python -*-
+
+import coro
+W = coro.write_stderr
+
+from coro.httpd.client import client as http_client
+
+def t0():
+    c = http_client ('127.0.0.1', 80)
+    l = [ c.send_request ('GET', '/postgresql/html/', {}, content=None, force=True) for x in range (10) ]
+    for req in l:
+        req.wait()
+        W ('%s\n' % (req.response,))
+
+def t1():
+    c = http_client ('127.0.0.1', 80)
+    rl = coro.in_parallel ([(c.GET, ('/postgresql/html/',))] * 10)
+    for x in rl:
+        W ('%s\n' % (x.response,))
+    return rl
+
+if __name__ == '__main__':
+    import coro.backdoor
+    coro.spawn (t0)
+    coro.spawn (coro.backdoor.serve, unix_path='/tmp/xx.bd')
+    coro.event_loop()
+

coro/http/demo/tlslite_server.py

+# -*- Mode: Python -*-
+
+# demo an https server using the TLSLite package.
+
+import coro
+import coro.http
+import coro.backdoor
+
+# -----------------------------------------------------------------------
+# --- change the location of the chain and key files on the next line ---
+# -----------------------------------------------------------------------
+server = coro.http.tlslite_server (
+    'cert/server.crt',
+    'cert/server.key',
+    )
+server.push_handler (coro.http.handlers.coro_status_handler())
+server.push_handler (coro.http.handlers.favicon_handler())
+coro.spawn (server.start, ('0.0.0.0', 9443))
+coro.spawn (coro.backdoor.serve, unix_path='/tmp/httpsd.bd')
+coro.event_loop (30.0)

coro/http/handlers.py

+# -*- Mode: Python -*-
+
+import coro
+import os
+import re
+import sys
+import time
+import zlib
+
+from coro.http.http_date import build_http_date
+
+W = sys.stderr.write
+
+# these two aren't real handlers, they're more like templates
+#  to give you an idea how to write one.
+class post_handler:
+
+    def match (self, request):
+        # override to do a better job of matching
+        return request._method == 'post'
+
+    def handle_request (self, request):
+        data = request.file.read()
+        W ('post handler, data=%r\n' % (data,))
+        request.done()
+
+class put_handler:
+
+    def match (self, request):
+        # override to do a better job of matching
+        return request.method == 'put'
+
+    def handle_request (self, request):
+        fp = request.file
+        while 1:
+            line = fp.readline()
+            if not line:
+                W ('line: DONE!\n')
+                break
+            else:
+                W ('line: %r\n' % (line,))
+        request.done()
+
+class coro_status_handler:
+
+    def match (self, request):
+        return request.path.split ('/')[1] == 'status'
+
+    def clean (self, s):
+        s = s.replace ('<','&lt;')
+        s = s.replace ('>','&gt;')
+        return s
+
+    def handle_request (self, request):
+        request['content-type'] = 'text/html; charset=utf-8'
+        request.set_deflate()
+        request.push (
+            '<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" '
+            '"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">'
+            '<html xmlns="http://www.w3.org/1999/xhtml">\r\n'
+            )
+        request.push ('<head><title>status</title></head><body>\r\n')
+        request.push ('<p>Listening on\r\n')
+        request.push (repr (request.server.addr))
+        request.push ('</p>\r\n')
+        request.push ('<table border="1">\r\n')
+        all_threads = ( (x, coro.where(x)) for x in coro.all_threads.values() )
+        for thread, traceback in all_threads:
+            request.push ('<tr><td>%s\r\n' % self.clean (repr(thread)))
+            request.push ('<pre>\r\n')
+            # traceback format seems to have changed
+            for level in traceback[1:-1].split ('] ['):
+                [file, fun] = level.split (' ')
+                fun, line = fun.split ('|')
+                request.push ('<b>%20s</b>:%3d %s\r\n' % (self.clean (fun), int(line), self.clean (file)))
+            request.push ('</pre></td></tr>')
+        request.push ('</table>\r\n')
+        request.push ('<p><a href="status">Update</a></p>')
+        request.push ('</body></html>')
+        request.done()                                        
+
+class file_handler:
+
+    block_size = 16000
+
+    def __init__ (self, doc_root):
+        self.doc_root = doc_root
+
+    def match (self, request):
+        path = request.path
+        filename = os.path.join (self.doc_root, path[1:])
+        return os.path.exists (filename)
+
+    crack_if_modified_since = re.compile ('([^;]+)(; length=([0-9]+))?$', re.IGNORECASE)
+
+    def handle_request (self, request):
+        path = request.path
+        filename = os.path.join (self.doc_root, path[1:])
+
+        if request.method not in ('get', 'head'):
+            request.error (405)
+            return
+
+        if os.path.isdir (filename):
+            filename = os.path.join (filename, 'index.html')
+
+        if not os.path.isfile (filename):
+            request.error (404)
+        else:
+            stat_info = os.stat (filename)
+            mtime = stat_info[stat.ST_MTIME]
+            file_length = stat_info[stat.ST_SIZE]
+
+            ims = request['if-modified-since']
+            if ims:
+                length_match = 1
+                m = self.crack_if_modified_since.match (ims)
+                if m:
+                    length = m.group (3)
+                    if length:
+                        if int(length) != file_length:
+                            length_match = 0
+
+                ims_date = http_date.parse_http_date (m.group(1))
+
+                if length_match and ims_date:
+                    if mtime <= ims_date:
+                        request.error (304)
+                        return
+
+            ftype, fencoding = mimetypes.guess_type (filename)
+            request['Content-Type'] = ftype or 'text/plain'
+            request['Last-Modified'] = build_http_date (mtime)
+
+            # Note: these are blocking file operations.
+            if request.method == 'get':
+                f = open (filename, 'rb')
+                block = f.read (self.block_size)
+                if not block:
+                    request.error (204) # no content
+                else:
+                    while 1:
+                        request.push (block)
+                        block = f.read (self.block_size)
+                        if not block:
+                            break
+            elif request.method == 'head':
+                pass
+            else:
+                # should be impossible
+                request.error (405)
+
+sample = (
+    'AAABAAEAICAQAAEABADoAgAAFgAAACgAAAAgAAAAQAAAAAEABAAAAAAAAAAAAAAAAAAAAAAAAAAA'
+    'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA'
+    'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA'
+    'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA'
+    'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA'
+    'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA'
+    'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA'
+    'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA'
+    'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA'
+    'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA'
+    'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA'
+    'AAAAAAAAAAAAAAD/+K///8AH//+iI///QAH//r4g//x3AH//Z6J//UABP/ovgD/458Ef+u+wv/Tn'
+    '0R/+79if9OXZH/6gCJ/2BwAf/u/8n/h33R/7Z7kf/ReQH/+qu7//BUW//7vrv//RR3//7r///80d'
+    '///pq///8EP//+rH///d9///6j///9Af/w=='
+    ).decode ('base64')
+
+zsample = zlib.compress (sample, 9)
+
+last_modified = build_http_date (time.time())
+
+class favicon_handler:
+
+    def __init__ (self, data=None):
+        if data is None:
+            self.data = zsample
+        else:
+            self.data = data
+
+    def match (self, request):
+        return request.path == '/favicon.ico'
+
+    def handle_request (self, request):
+        if request['if-modified-since']:
+            # if we cared, we could check that timestamp.
+            request.error (304)
+        else:
+            request['content-type'] = 'image/x-icon'
+            request['last-modified'] = last_modified
+            # are there browsers that don't accept deflate?
+            request['content-encoding'] = 'deflate'
+            request.push (self.data)
+            request.done()

coro/http/protocol.py

+# -*- Mode: Python -*-
+
+import coro
+from coro import read_stream
+
+W = coro.write_stderr
+
+# candidate for sync.pyx?
+class latch:
+
+    "Like a CV, except without the race - if the event has already fired then wait() will return immediately."
+
+    def __init__ (self):
+        self.cv = coro.condition_variable()
+        self.done = False
+
+    def wake_all (self, args=()):
+        self.done = True
+        self.args = args
+        return self.cv.wake_all (args)
+
+    def wait (self):
+        if not self.done:
+            return self.cv.wait()
+        else:
+            return self.args
+
+class http_file:
+
+    "HTTP message content, as a file-like object."
+
+    buffer_size = 8000
+
+    def __init__ (self, headers, stream):
+        self.streami = stream
+        self.done_cv = latch()
+        if headers.get_one ('transfer-encoding') == 'chunked':
+            self.streamo = read_stream.buffered_stream (self._gen_read_chunked().next)
+        else:
+            content_length = headers.get_one ('content-length')
+            if content_length:
+                self.content_length = int (content_length)
+                self.streamo = read_stream.buffered_stream (self._gen_read_fixed().next)
+            else:
+                raise HTTP_Protocol_Error ("no way to determine length of HTTP data")
+            
+    def _gen_read_chunked (self):
+        "generator: decodes chunked transfer-encoding."
+        s = self.streami
+        while 1:
+            chunk_size = int (s.read_line()[:-2], 16)
+            if chunk_size == 0:
+                self.done_cv.wake_all()
+                return
+            else:
+                remain = chunk_size
+                while remain:
+                    ask = min (remain, self.buffer_size)
+                    block = s.read_exact (ask)
+                    assert (s.read_exact (2) == '\r\n')
+                    remain -= ask
+                    yield block
+
+    def _gen_read_fixed (self):
+        "generate fixed-size blocks of content."
+        s = self.streami
+        remain = self.content_length
+        while remain:
+            ask = min (remain, self.buffer_size)
+            block = s.read_exact (ask)
+            remain -= ask
+            yield block
+        self.done_cv.wake_all()
+        return
+
+    # XXX implement <size> argument
+    def read (self, join=True):
+        "read the entire contents.  join=False returns a generator, join=True returns a string."
+        r = (x for x in self.streamo.read_all())
+        if join:
+            return ''.join (r)
+        else:
+            return r
+        
+    def readline (self):
+        "read a newline-delimited line."
+        if self.done_cv.done:
+            return ''
+        else:
+            return self.streamo.read_until ('\n')
+
+    def wait (self):
+        "wait until all the content has been read."
+        self.done_cv.wait()
+
+class header_set:
+
+    def __init__ (self, headers=()):
+        self.headers = {}
+        for h in headers:
+            self.crack (h)
+
+    def from_keywords (self, kwds):
+        "Populate this header set from a dictionary of keyword arguments (e.g., 'content_length' becomes 'content-length')"
+        r = []
+        for k, v in kwds.items():
+            k = k.replace ('_', '-')
+            self[k] = v
+        return self
+
+    def crack (self, h):
+        "Crack one header line."
+        # deliberately ignoring 822 crap like continuation lines.
+        try:
+            i = h.index (': ')
+            name, value = h[:i], h[i+2:]
+            self[name] = value
+        except ValueError:
+            coro.write_stderr ('dropping bogus header %r\n' % (h,))
+            pass
+
+    def get_one (self, key):
+        "Get the value of a header expected to have at most one value. If not present, return None.  If more than one, raise ValueError."
+        r = self.headers.get (key, None)
+        if r is None:
+            return r
+        elif isinstance (r, list) and len (r) > 1:
+            raise ValueError ("expected only one %s header, got %r" % (key, r))
+        else:
+            return r[0]
+
+    def has_key (self, key):
+        "Is this header present?"
+        return self.headers.has_key (key.lower())
+
+    def __getitem__ (self, key):
+        "Returns the list of values for this header, or None."
+        return self.headers.get (key, None)
+
+    def __setitem__ (self, name, value):
+        "Add a value to the header <name>."
+        name = name.lower()
+        probe = self.headers.get (name)
+        if probe is None:
+            self.headers[name] = [value]
+        else:
+            probe.append (value)
+
+    def __str__ (self):
+        "Render the set of headers."
+        r = []
+        for k, vl in self.headers.iteritems():
+            for v in vl:
+                r.append ('%s: %s\r\n' % (k, v))
+        return ''.join (r)

coro/http/session_handler.py

+# -*- Mode: Python -*-
+
+import coro
+import time
+import uuid
+
+import sys
+W = sys.stderr.write
+
+# See: http://en.wikipedia.org/wiki/HTTP_cookie#Session_cookie
+
+def extract_session (cookie):
+    parts = cookie.split (';')
+    for part in parts:
+        pair = [x.strip() for x in part.split ('=')]
+        if len (pair) == 2:
+            if pair[0] == 'session':
+                return pair[1]
+    return None
+
+class session_handler:
+
+    def __init__ (self, name, function):
+        self.name = name
+        self.function = function
+        self.sessions = {}
+
+    def match (self, request):
+        path = request.path.split ('/')
+        if (len(path) > 1) and (path[1] == self.name):
+            return 1
+        else:
+            return 0
+
+    def find_session (self, request):
+        # XXX does http allow more than one cookie header?
+        cookie = request['cookie']
+        if cookie:
+            sid = extract_session (cookie)
+            return sid, self.sessions.get (sid, None)
+        else:
+            return None, None
+
+    def gen_session_id (self):
+        return str (uuid.uuid4())
+
+    def handle_request (self, request):
+        sid, fifo = self.find_session (request)
+        if fifo is None:
+            # login
+            fifo = coro.fifo()
+            fifo.push (request)
+            sid = self.gen_session_id()
+            request['set-cookie'] = 'session=%s' % (sid,)            
+            self.sessions[sid] = fifo
+            coro.spawn (self.wrap, sid, fifo)
+        else:
+            fifo.push (request)
+
+    def wrap (self, sid, fifo):
+        try:
+            self.function (sid, fifo)
+        finally:
+            del self.sessions[sid]

coro/read_stream.py

+# -*- Mode: Python -*-
+
+class socket_producer:
+    def __init__ (self, conn, buffer_size=8000):
+        self.conn = conn
+        self.buffer_size = buffer_size
+    def next (self):
+        return self.conn.recv (self.buffer_size)
+
+def sock_stream (sock):
+    return buffered_stream (socket_producer (sock).next)
+
+class buffered_stream:
+
+    def __init__ (self, producer):
+        self.producer = producer
+        self.buffer = ''
+
+    def gen_read_until (self, delim):
+        "generate pieces of input up to and including <delim>, then StopIteration"
+        ld = len(delim)
+        m = 0
+        while 1:
+            if not self.buffer:
+                self.buffer = self.producer()
+                if not self.buffer:
+                    # eof
+                    yield ''
+                    return
+            i = 0
+            while i < len (self.buffer):
+                if self.buffer[i] == delim[m]:
+                    m += 1
+                    if m == ld:
+                        result, self.buffer = self.buffer[:i+1], self.buffer[i+1:]
+                        yield result
+                        return
+                else:
+                    m = 0
+                i += 1
+            block, self.buffer = self.buffer, ''
+            yield block
+
+    def gen_read_until_dfa (self, dfa):
+        "generate pieces of input up to and including a match on <dfa>, then StopIteration"
+        m = 0
+        while 1:
+            if not self.buffer:
+                self.buffer = self.producer()
+                if not self.buffer:
+                    # eof
+                    yield ''
+                    return
+            i = 0
+            while i < len (self.buffer):
+                if dfa.consume (self.buffer[i]):
+                    result, self.buffer = self.buffer[:i+1], self.buffer[i+1:]
+                    yield result
+                    return
+                i += 1
+            block, self.buffer = self.buffer, ''
+            yield block
+
+    def gen_read_exact (self, size):
+        "generate pieces of input up to <size> bytes, then StopIteration"
+        remain = size
+        while remain:
+            if len (self.buffer) >= remain:
+                result, self.buffer = self.buffer[:remain], self.buffer[remain:]
+                yield result
+                return
+            else:
+                piece, self.buffer = self.buffer, self.producer()
+                remain -= len (piece)
+                yield piece
+                if not self.buffer:
+                    # eof
+                    yield ''
+                    return
+
+    def read_until (self, delim, join=True):
+        "read until <delim>.  return a list of parts unless <join> is True"
+        result = ( x for x in self.gen_read_until (delim) )
+        if join:
+            return ''.join (result)
+        else:
+            return result
+
+    def read_exact (self, size, join=True):
+        "read exactly <size> bytes.  return a list of parts unless <join> is True"
+        result = ( x for x in self.gen_read_exact (size) )
+        if join:
+            return ''.join (result)
+        else:
+            return result
+
+    def flush (self):
+        "flush this stream's buffer"
+        result, self.buffer = self.buffer, ''
+        return result
+
+    def read_line (self, delim='\r\n'):
+        "read a CRLF-delimited line from this stream"
+        return self.read_until (delim)
+
+    def read_all (self):
+        "read from self.producer until the stream terminates"
+        if self.buffer:
+            yield self.flush()
+        while 1:
+            block = self.producer()
+            if not block:
+                return
+            else:
+                yield block
+

coro/socket.pyx

@@ -340,11 +340,11 @@ cdef public class sock [ object sock_object, type sock_type ]:
         return self.fd
 
     cdef _set_reuse_addr (self):
-        cdef int old
+        cdef int old = 0
         cdef socklen_t optlen
-        optlen = sizeof (old);
+        optlen = sizeof (old)
         getsockopt (self.fd, SOL_SOCKET, SO_REUSEADDR, <void*> &old, &optlen)
-        old = old | 1;
+        old = old | 1
         setsockopt (self.fd, SOL_SOCKET, SO_REUSEADDR, <void*> &old, optlen)
 
     def set_reuse_addr (self):

coroutine/httpd/mime_type_table.py

-# -*- Python -*-
-# Converted by ./convert_mime_type_table.py from:
-# /usr/src2/apache_1.2b6/conf/mime.types
-#
-content_type_map = \
-  {
-        'ai':  'application/postscript',
-       'aif':  'audio/x-aiff',
-      'aifc':  'audio/x-aiff',
-      'aiff':  'audio/x-aiff',
-        'au':  'audio/basic',
-       'avi':  'video/x-msvideo',
-     'bcpio':  'application/x-bcpio',
-       'bin':  'application/octet-stream',
-       'cdf':  'application/x-netcdf',
-     'class':  'application/octet-stream',
-      'cpio':  'application/x-cpio',
-       'cpt':  'application/mac-compactpro',
-       'csh':  'application/x-csh',
-       'dcr':  'application/x-director',
-       'dir':  'application/x-director',
-       'dms':  'application/octet-stream',
-       'doc':  'application/msword',
-       'dvi':  'application/x-dvi',
-       'dxr':  'application/x-director',
-       'eps':  'application/postscript',
-       'etx':  'text/x-setext',
-       'exe':  'application/octet-stream',
-       'gif':  'image/gif',
-      'gtar':  'application/x-gtar',
-        'gz':  'application/x-gzip',
-       'hdf':  'application/x-hdf',
-       'hqx':  'application/mac-binhex40',
-       'htm':  'text/html',
-      'html':  'text/html',
-       'ice':  'x-conference/x-cooltalk',
-       'ief':  'image/ief',
-       'jpe':  'image/jpeg',
-      'jpeg':  'image/jpeg',
-       'jpg':  'image/jpeg',
-       'kar':  'audio/midi',
-     'latex':  'application/x-latex',
-       'lha':  'application/octet-stream',
-       'lzh':  'application/octet-stream',
-       'man':  'application/x-troff-man',
-        'me':  'application/x-troff-me',
-       'mid':  'audio/midi',
-      'midi':  'audio/midi',
-       'mif':  'application/x-mif',
-       'mov':  'video/quicktime',
-     'movie':  'video/x-sgi-movie',
-       'mp2':  'audio/mpeg',
-       'mpe':  'video/mpeg',
-      'mpeg':  'video/mpeg',
-       'mpg':  'video/mpeg',
-      'mpga':  'audio/mpeg',
-       'mp3':  'audio/mpeg',
-        'ms':  'application/x-troff-ms',
-        'nc':  'application/x-netcdf',
-       'oda':  'application/oda',
-       'pbm':  'image/x-portable-bitmap',
-       'pdb':  'chemical/x-pdb',
-       'pdf':  'application/pdf',
-       'pgm':  'image/x-portable-graymap',
-       'png':  'image/png',
-       'pnm':  'image/x-portable-anymap',
-       'ppm':  'image/x-portable-pixmap',
-       'ppt':  'application/powerpoint',
-        'ps':  'application/postscript',
-        'qt':  'video/quicktime',
-        'ra':  'audio/x-realaudio',
-       'ram':  'audio/x-pn-realaudio',
-       'ras':  'image/x-cmu-raster',
-       'rgb':  'image/x-rgb',
-      'roff':  'application/x-troff',
-       'rpm':  'audio/x-pn-realaudio-plugin',
-       'rtf':  'application/rtf',
-       'rtx':  'text/richtext',
-       'sgm':  'text/x-sgml',
-      'sgml':  'text/x-sgml',
-        'sh':  'application/x-sh',
-      'shar':  'application/x-shar',
-       'sit':  'application/x-stuffit',
-       'skd':  'application/x-koan',
-       'skm':  'application/x-koan',
-       'skp':  'application/x-koan',
-       'skt':  'application/x-koan',
-       'snd':  'audio/basic',
-       'src':  'application/x-wais-source',
-   'sv4cpio':  'application/x-sv4cpio',
-    'sv4crc':  'application/x-sv4crc',
-         't':  'application/x-troff',
-       'tar':  'application/x-tar',
-       'tcl':  'application/x-tcl',
-       'tex':  'application/x-tex',
-      'texi':  'application/x-texinfo',
-   'texinfo':  'application/x-texinfo',
-       'tif':  'image/tiff',
-      'tiff':  'image/tiff',
-        'tr':  'application/x-troff',
-       'tsv':  'text/tab-separated-values',
-       'txt':  'text/plain',
-     'ustar':  'application/x-ustar',
-       'vcd':  'application/x-cdlink',
-      'vrml':  'x-world/x-vrml',
-       'wav':  'audio/x-wav',
-       'wrl':  'x-world/x-vrml',
-       'xbm':  'image/x-xbitmap',
-       'xpm':  'image/x-xpixmap',
-       'xwd':  'image/x-xwindowdump',
-       'xyz':  'chemical/x-pdb',
-       'zip':  'application/zip',
-  }

coroutine/httpd/session_handler.py

-# -*- Mode: Python; tab-width: 4 -*-
-
-import coro
-import string
-import re
-import time
-
-h_re = re.compile (r'([^: ]+): (.*)')
-
-def get_header (header, headers):
-        for h in headers:
-                m = h_re.match (h)
-                if m:
-                        name, value = m.groups()
-                        if string.lower (name) == header:
-                                return value
-        return None
-
-def extract_session (cookie):
-        parts = string.split (cookie, ';')
-        for part in parts:
-                pair = string.split (part, '=')
-                if len(pair) == 2:
-                        if pair[0] == 'session':
-                                return pair[1]
-        return None
-
-class session_handler:
-
-        def __init__ (self, name, function):
-                self.name = name
-                self.function = function
-                self.sessions = {}
-
-        def match (self, request):
-                path = string.split (request._path, '/')
-                if (len(path) > 1) and (path[1] == self.name):
-                        return 1
-                else:
-                        return 0
-
-        def get_next_request (self):
-                return coro._yield()
-
-        def find_session (self, request):
-                cookie = get_header ('cookie', request._request_headers)
-                if cookie:
-                        sid = extract_session (cookie)
-                        return sid, self.sessions.get (sid, None)
-                else:
-                        return None, None
-
-        def gen_session_id (self):
-                import random
-                import sys
-                sid = None
-                while self.sessions.has_key (sid):
-                        n = random.randint (0,sys.maxint-1)
-                        sid = hex(n)[2:]
-                return sid
-
-        expires_delta = 100 * 86400
-
-        def handle_request (self, request):
-                sid, c = self.find_session (request)
-                # The sid=='None' test is temporary hack, can probably remove it
-                if (not sid) or (sid=='None'):
-                        sid = self.gen_session_id()
-                if c and c.isAlive():
-                        # is c already running?
-                        # hack, must grok this
-                        coro.schedule (c, request)
-                        request._done = 1
-                else:
-                        # login
-                        c = coro.new (self.function, self, request, sid)
-                        # Wdy, DD-Mon-YYYY HH:MM:SS GMT
-                        expires = time.strftime ('%a, %d-%b-%Y 00:00:00 GMT', time.gmtime (int (time.time()) + self.expires_delta))
-                        request['Set-Cookie'] = 'session=%s; path=/; expires=%s' % (sid, expires)
-                        # hack, must grok this
-                        request._done = 1
-                        c.start()

setup.py

@@ -108,13 +108,13 @@ setup (
                 ],
             ),
         ],
-    packages=['coro', 'coro.clocks'],
+    packages=['coro', 'coro.clocks', 'coro.http'],
     package_dir = {
         '': 'coroutine',
         'coro': 'coro',
         'coro.clocks': 'coro/clocks'
     },
-    py_modules = ['backdoor', 'coro_process', 'coro_unittest'],
+    py_modules = ['backdoor', 'coro.read_stream', 'coro_process', 'coro_unittest',],
     download_url = 'http://github.com/ironport/shrapnel/tarball/master#egg=coro-1.0.2',
     install_requires = ['Cython>=0.12.1', 'distribute>=0.6.16'],
     cmdclass={'build_ext': build_ext},