Implement the tracing mechanism

.gitignore

@@ -108,3 +108,5 @@ dmypy.json
 .pyre/
 
 # End of https://www.gitignore.io/api/python
+
+traces.db

my2to3/fixes/__init__.py

@@ -4,6 +4,7 @@ import os
 
 class BaseFix(lib2to3.fixer_base.BaseFix):
   def start_tree(self, tree, filename):
+    # See my2to3.trace.apply_fixers
     sep = '-' * 5 # Arbitrary
     if sep in filename:
       filename = filename.split(sep, 1)[1].replace(sep, os.sep)

my2to3/fixes/fix_division_trace.py

 # https://lab.nexedi.com/nexedi/erp5/snippets/475
 from collections import defaultdict
+import inspect
 import lib2to3.fixer_util
 import lib2to3.pgen2
 from lib2to3.pygram import python_symbols as syms
@@ -7,27 +8,28 @@ from lib2to3.pytree import Node
 import sys
 
 from my2to3.fixes import BaseFix
+from my2to3.trace import create_table, register_tracing_function
+from my2to3.util import parse_type
+
+
+trace = create_table("division", "filename", "lineno", "id", "dividend_type", "divisor_type")
+
+@register_tracing_function
+def division_traced(id, dividend, divisor):
+  previous_frame = inspect.currentframe().f_back
+
+  trace(
+    previous_frame.f_code.co_filename,
+    previous_frame.f_lineno,
+    id,
+    parse_type(type(dividend)),
+    parse_type(type(divisor))
+  )
+  return dividend / divisor
 
 
 class FixDivisionTrace(BaseFix):
-  """Rewrites a/b into division_traced(a, b)
-
-  division_traced can be a function that looks up the stack and record the operand types, for example:
-
-      def division_traced(id, dividend, divisor):
-        import inspect
-        previous_frame = inspect.currentframe().f_back
-        print ("{}|{}|{} {} / {}".format(
-            previous_frame.f_code.co_filename,
-            previous_frame.f_lineno,
-            id,
-            type(dividend),
-            type(divisor)
-        ))
-        return dividend / divisor
-
-      import __builtin__
-      __builtin__.division_traced = division_traced
+  """Rewrites a / b into division_traced(id, a, b)
   """
   def start_tree(self, tree, filename):
     super(FixDivisionTrace, self).start_tree(tree, filename)

my2to3/trace.py

+import __builtin__, imp, os, sqlite3, sys, tempfile, types
+
+from lib2to3.refactor import get_fixers_from_package, RefactoringTool
+
+database = "traces.db"
+
+
+tracing_functions = []
+def register_tracing_function(f):
+  tracing_functions.append(f)
+  return f
+
+
+def create_table(table, *values):
+  # TODO:
+  # - refactor
+  # - optimize
+  conn = sqlite3.connect(database)
+  c = conn.cursor()
+  v = ', '.join(values)
+  c.execute("CREATE TABLE IF NOT EXISTS %s (%s, UNIQUE (%s))" % (table, v, v))
+  conn.commit()
+  conn.close()
+
+  def trace(*args):
+    conn = sqlite3.connect(database)
+    c = conn.cursor()
+    try:
+      c.execute('INSERT INTO %s VALUES (%s)' % (
+        table, ', '.join(['?'] * len(args))), args)
+    except sqlite3.IntegrityError as e:
+      if not e.message.startswith('UNIQUE constraint failed:'):
+        raise
+    else:
+      conn.commit()
+    conn.close()
+  return trace
+
+
+def get_fixers():
+  return [
+    f for f in get_fixers_from_package("my2to3.fixes") if f.endswith("_trace")
+  ]
+
+
+def apply_fixers(string, name):
+  refactoring_tool = RefactoringTool(fixer_names=get_fixers())
+  # A temporary file is used to enjoy the benefits of
+  # refactoring_tool.refactor_file (unlike directly using
+  # refactoring_tool.refactor_string).
+  #
+  # XXX: We add the original name via the suffix. Fixers are able to handle this
+  # (see my2to3.fixes.BaseFix). Let's find a better way.
+  sep = '-' * 5 # Arbitrary
+  with tempfile.NamedTemporaryFile(suffix=sep + name.replace(os.sep, sep)) as f:
+    f.write(string)
+    f.flush()
+    refactoring_tool.refactor_file(f.name, write=True)
+    f.seek(0)
+    return f.read()
+
+
+init_py = '__init__' + os.extsep + 'py'
+class ModuleImporter:
+  def __init__(self, is_whitelisted):
+    self.paths = {}
+    self.is_whitelisted = is_whitelisted
+
+  def find_module(self, fullname, path=None):
+    if not path or not self.is_whitelisted(fullname, path):
+      return
+
+    try:
+      file, pathname, description = imp.find_module(fullname.rsplit('.', 1)[-1], path)
+    except ImportError:
+      return
+    if file is not None:
+      file.close()
+
+    if description[-1] is imp.PKG_DIRECTORY:
+      # It's a package
+      assert file is None, file
+      pathname = os.path.join(pathname, init_py)
+
+    self.paths[fullname] = pathname
+    return self
+
+  def load_module(self, fullname):
+    imp.acquire_lock()
+    try:
+      if fullname in sys.modules:
+        return sys.modules[fullname]
+
+      path = self.paths[fullname]
+      with open(path) as f:
+        new_code = compile(apply_fixers(f.read(), path), path, 'exec')
+
+      sys.modules[fullname] = module = types.ModuleType(fullname)
+      module.__file__ = p = new_code.co_filename
+      if p.endswith(init_py):
+        module.__path__ = [path.rstrip(init_py)]
+      try:
+        exec new_code in module.__dict__
+      except Exception:
+        del sys.modules[fullname]
+        raise
+    finally:
+      imp.release_lock()
+    return module
+
+
+def patch_imports(whitelist):
+  sys.meta_path.append(ModuleImporter(whitelist))
+
+  for f in tracing_functions:
+    setattr(__builtin__, f.__name__, f)
+
+
+for fixer in get_fixers():
+  # Import each fixer, to make sure that "my2to3.trace.tracing_functions" is
+  # correctly populated.
+  __import__(fixer)

my2to3/util.py

@@ -2,6 +2,12 @@ from lib2to3 import fixer_util
 from lib2to3.pytree import Leaf, Node
 from lib2to3.pgen2 import token
 from lib2to3.pygram import python_symbols as syms
+import re
+
+
+type_match = re.compile(r"^<type '(.*)'>$").match
+def parse_type(type_):
+  return type_match(str(type_)).group(1)
 
 
 # https://github.com/python-modernize/python-modernize/blob/84d973cb7b8153f9f7f22c3574a59312b2372ccb/libmodernize/__init__.py#L10-49