Compile DFA.py and tie it into the other lexer classes.

Cython/Plex/DFA.pxd

+# cython:
+
+cimport cython
+
+from . cimport Machines
+from .Transitions cimport TransitionMap
+
+
+@cython.final
+cdef class StateMap:
+    cdef Machines.FastMachine new_machine
+    cdef dict old_to_new_dict
+    cdef dict new_to_old_dict
+
+    cdef old_to_new(self, dict old_state_set)
+
+    @cython.locals(state=Machines.Node)
+    cdef highest_priority_action(self, dict state_set)
+
+    cdef make_key(self, dict state_set)
+
+
+@cython.locals(new_machine=Machines.FastMachine, transitions=TransitionMap)
+cpdef nfa_to_dfa(Machines.Machine old_machine, debug=*)
+
+cdef set_epsilon_closure(dict state_set)
+cdef dict epsilon_closure(Machines.Node state)
+
+@cython.locals(state_set_2=dict, state2=Machines.Node)
+cdef add_to_epsilon_closure(dict state_set, Machines.Node state)

Cython/Plex/DFA.py

+# cython: auto_cpdef=True
 """
 Python Lexical Analyser
 
@@ -95,14 +96,11 @@ class StateMap(object):
     Helper class used by nfa_to_dfa() to map back and forth between
     sets of states from the old machine and states of the new machine.
     """
-    new_machine = None      # Machine
-    old_to_new_dict = None  # {(old_state,...) : new_state}
-    new_to_old_dict = None  # {id(new_state) : old_state_set}
 
     def __init__(self, new_machine):
-        self.new_machine = new_machine
-        self.old_to_new_dict = {}
-        self.new_to_old_dict = {}
+        self.new_machine = new_machine  # Machine
+        self.old_to_new_dict = {}  # {(old_state,...) : new_state}
+        self.new_to_old_dict = {}  # {id(new_state) : old_state_set}
 
     def old_to_new(self, old_state_set):
         """
@@ -140,9 +138,7 @@ class StateMap(object):
         Convert a set of states into a uniquified
         sorted tuple suitable for use as a dictionary key.
         """
-        lst = list(state_set)
-        lst.sort()
-        return tuple(lst)
+        return tuple(sorted(state_set))
 
     def dump(self, file):
         from .Transitions import state_set_str

Cython/Plex/Machines.pxd

 cimport cython
 
+from .Actions cimport Action
+from .Transitions cimport TransitionMap
+
+
+@cython.final
+cdef class Machine:
+    cdef readonly list states
+    cdef readonly dict  initial_states
+    cdef readonly Py_ssize_t next_state_number
+
+    cpdef new_state(self)
+    cpdef new_initial_state(self, name)
+
+
 @cython.final
 cdef class Node:
-    cdef readonly object transitions  # TransitionMap
-    cdef readonly object action  # Action
-    cdef public object epsilon_closure  # dict
+    cdef readonly TransitionMap transitions
+    cdef readonly Action action
+    cdef public dict epsilon_closure
     cdef readonly Py_ssize_t number
     cdef readonly long action_priority
 

Cython/Plex/Machines.py

@@ -24,13 +24,10 @@ LOWEST_PRIORITY = -maxint
 
 class Machine(object):
     """A collection of Nodes representing an NFA or DFA."""
-    states = None          # [Node]
-    next_state_number = 1
-    initial_states = None  # {(name, bol): Node}
-
     def __init__(self):
-        self.states = []
-        self.initial_states = {}
+        self.states = []  # [Node]
+        self.initial_states = {}  # {(name, bol): Node}
+        self.next_state_number = 1
 
     def __del__(self):
         for state in self.states:

Cython/Plex/Transitions.pxd

+cimport cython
+
+cdef long maxint
+
+@cython.final
+cdef class TransitionMap:
+    cdef list map
+    cdef dict special
+
+    @cython.locals(i=cython.Py_ssize_t, j=cython.Py_ssize_t)
+    cpdef add(self, event, new_state)
+
+    @cython.locals(i=cython.Py_ssize_t, j=cython.Py_ssize_t)
+    cpdef add_set(self, event, new_set)
+
+    @cython.locals(i=cython.Py_ssize_t, n=cython.Py_ssize_t, else_set=cython.bint)
+    cpdef iteritems(self)
+
+    @cython.locals(map=list, lo=cython.Py_ssize_t, mid=cython.Py_ssize_t, hi=cython.Py_ssize_t)
+    cdef split(self, long code)
+
+    cdef get_special(self, event)

Cython/Plex/Transitions.py

@@ -5,18 +5,12 @@ This version represents state sets directly as dicts for speed.
 """
 from __future__ import absolute_import
 
-import cython
-
-cython.declare(maxint=cython.long)
-
 try:
     from sys import maxsize as maxint
 except ImportError:
     from sys import maxint
 
 
-@cython.final
-@cython.cclass
 class TransitionMap(object):
     """
     A TransitionMap maps an input event to a set of states.
@@ -45,8 +39,6 @@ class TransitionMap(object):
     kept separately in a dictionary.
     """
 
-    cython.declare(map=list, special=dict)
-
     def __init__(self, map=None, special=None):
         if not map:
             map = [-maxint, {}, maxint]
@@ -55,7 +47,6 @@ class TransitionMap(object):
         self.map = map          # The list of codes and states
         self.special = special  # Mapping for special events
 
-    @cython.locals(i=cython.Py_ssize_t, j=cython.Py_ssize_t, map=list)
     def add(self, event, new_state):
         """
         Add transition to |new_state| on |event|.
@@ -71,7 +62,6 @@ class TransitionMap(object):
         else:
             self.get_special(event)[new_state] = 1
 
-    @cython.locals(i=cython.Py_ssize_t, j=cython.Py_ssize_t, map=list)
     def add_set(self, event, new_set):
         """
         Add transitions to the states in |new_set| on |event|.
@@ -93,7 +83,6 @@ class TransitionMap(object):
         """
         return self.special.get('')
 
-    @cython.locals(map=list, i=cython.Py_ssize_t, n=cython.Py_ssize_t, else_set=cython.bint)
     def iteritems(self):
         """
         Return the mapping as an iterable of ((code1, code2), state_set) and
@@ -121,8 +110,6 @@ class TransitionMap(object):
 
     # ------------------- Private methods --------------------
 
-    @cython.ccall
-    @cython.locals(map=list, lo=cython.Py_ssize_t, mid=cython.Py_ssize_t, hi=cython.Py_ssize_t, code=cython.long)
     def split(self, code):
         """
         Search the list for the position of the split point for |code|,
@@ -153,7 +140,6 @@ class TransitionMap(object):
             map[hi:hi] = [code, map[hi - 1].copy()]
             return hi
 
-    @cython.ccall
     def get_special(self, event):
         """
         Get state set for special event, adding a new entry if necessary.

setup.py

@@ -86,6 +86,7 @@ def compile_cython_modules(profile=False, compile_more=False, cython_with_refnan
         "Cython.Plex.Actions",
         "Cython.Plex.Machines",
         "Cython.Plex.Transitions",
+        "Cython.Plex.DFA",
         "Cython.Compiler.Scanning",
         "Cython.Compiler.Visitor",
         "Cython.Compiler.FlowControl",