From d93330a9ed5d55b2c4e2f78d64528cbd19b95b61 Mon Sep 17 00:00:00 2001 From: Arnaud Fontaine <arnaud.fontaine@nexedi.com> Date: Fri, 12 Dec 2014 11:11:23 +0900 Subject: [PATCH] Fix wrong lines information in bytecode (tracebacks, pdb...) generated by compiler module. --- component/python-2.7/buildout.cfg | 1 + .../fix_compiler_module_issue_20613.patch | 378 ++++++++++++++++++ 2 files changed, 379 insertions(+) create mode 100644 component/python-2.7/fix_compiler_module_issue_20613.patch diff --git a/component/python-2.7/buildout.cfg b/component/python-2.7/buildout.cfg index a7abde238..b0743db36 100644 --- a/component/python-2.7/buildout.cfg +++ b/component/python-2.7/buildout.cfg @@ -41,6 +41,7 @@ patch-options = -p1 patches = ${:_profile_base_location_}/tls_sni.patch#c95af105e6e96aaa58a50137595872a0 ${:_profile_base_location_}/tls_sni_httplib.patch#5c9d00d23b85169df792a936a056cbcc + ${:_profile_base_location_}/fix_compiler_module_issue_20613.patch#94443a77f903e9de880a029967fa6aa7 url = http://python.org/ftp/python/${:package_version}/Python-${:package_version}${:package_version_suffix}.tar.xz configure-options = diff --git a/component/python-2.7/fix_compiler_module_issue_20613.patch b/component/python-2.7/fix_compiler_module_issue_20613.patch new file mode 100644 index 000000000..685a435e5 --- /dev/null +++ b/component/python-2.7/fix_compiler_module_issue_20613.patch @@ -0,0 +1,378 @@ +From af98230024f639d4c719b55d0d330912c3ea7b97 Mon Sep 17 00:00:00 2001 +From: Arnaud Fontaine <arnaud.fontaine@nexedi.com> +Date: Thu, 11 Dec 2014 23:48:22 +0900 +Subject: [PATCH] Revert https://hg.python.org/cpython/raw-rev/42faa8054c3d. + This fixes http://bugs.python.org/issue20613. + +--- + Lib/compiler/pyassem.py | 257 ++++++++++++++++++++++++++++------------------ + Lib/compiler/pycodegen.py | 2 +- + 2 files changed, 159 insertions(+), 100 deletions(-) + +diff --git a/Lib/compiler/pyassem.py b/Lib/compiler/pyassem.py +index f52f7d0..4299a17 100644 +--- a/Lib/compiler/pyassem.py ++++ b/Lib/compiler/pyassem.py +@@ -21,7 +21,6 @@ class FlowGraph: + if self.current: + print "end", repr(self.current) + print " next", self.current.next +- print " prev", self.current.prev + print " ", self.current.get_children() + print repr(block) + self.current = block +@@ -41,12 +40,13 @@ class FlowGraph: + if block is None: + block = self.newBlock() + +- # Note: If the current block ends with an unconditional control +- # transfer, then it is techically incorrect to add an implicit +- # transfer to the block graph. Doing so results in code generation +- # for unreachable blocks. That doesn't appear to be very common +- # with Python code and since the built-in compiler doesn't optimize +- # it out we don't either. ++ # Note: If the current block ends with an unconditional ++ # control transfer, then it is incorrect to add an implicit ++ # transfer to the block graph. The current code requires ++ # these edges to get the blocks emitted in the right order, ++ # however. :-( If a client needs to remove these edges, call ++ # pruneEdges(). ++ + self.current.addNext(block) + self.startBlock(block) + +@@ -69,6 +69,8 @@ class FlowGraph: + def emit(self, *inst): + if self._debug: + print "\t", inst ++ if inst[0] in ['RETURN_VALUE', 'YIELD_VALUE']: ++ self.current.addOutEdge(self.exit) + if len(inst) == 2 and isinstance(inst[1], Block): + self.current.addOutEdge(inst[1]) + self.current.emit(inst) +@@ -78,9 +80,118 @@ class FlowGraph: + + i.e. each node appears before all of its successors + """ +- order = order_blocks(self.entry, self.exit) ++ # XXX make sure every node that doesn't have an explicit next ++ # is set so that next points to exit ++ for b in self.blocks.elements(): ++ if b is self.exit: ++ continue ++ if not b.next: ++ b.addNext(self.exit) ++ order = dfs_postorder(self.entry, {}) ++ order.reverse() ++ self.fixupOrder(order, self.exit) ++ # hack alert ++ if not self.exit in order: ++ order.append(self.exit) ++ + return order + ++ def fixupOrder(self, blocks, default_next): ++ """Fixup bad order introduced by DFS.""" ++ ++ # XXX This is a total mess. There must be a better way to get ++ # the code blocks in the right order. ++ ++ self.fixupOrderHonorNext(blocks, default_next) ++ self.fixupOrderForward(blocks, default_next) ++ ++ def fixupOrderHonorNext(self, blocks, default_next): ++ """Fix one problem with DFS. ++ ++ The DFS uses child block, but doesn't know about the special ++ "next" block. As a result, the DFS can order blocks so that a ++ block isn't next to the right block for implicit control ++ transfers. ++ """ ++ index = {} ++ for i in range(len(blocks)): ++ index[blocks[i]] = i ++ ++ for i in range(0, len(blocks) - 1): ++ b = blocks[i] ++ n = blocks[i + 1] ++ if not b.next or b.next[0] == default_next or b.next[0] == n: ++ continue ++ # The blocks are in the wrong order. Find the chain of ++ # blocks to insert where they belong. ++ cur = b ++ chain = [] ++ elt = cur ++ while elt.next and elt.next[0] != default_next: ++ chain.append(elt.next[0]) ++ elt = elt.next[0] ++ # Now remove the blocks in the chain from the current ++ # block list, so that they can be re-inserted. ++ l = [] ++ for b in chain: ++ assert index[b] > i ++ l.append((index[b], b)) ++ l.sort() ++ l.reverse() ++ for j, b in l: ++ del blocks[index[b]] ++ # Insert the chain in the proper location ++ blocks[i:i + 1] = [cur] + chain ++ # Finally, re-compute the block indexes ++ for i in range(len(blocks)): ++ index[blocks[i]] = i ++ ++ def fixupOrderForward(self, blocks, default_next): ++ """Make sure all JUMP_FORWARDs jump forward""" ++ index = {} ++ chains = [] ++ cur = [] ++ for b in blocks: ++ index[b] = len(chains) ++ cur.append(b) ++ if b.next and b.next[0] == default_next: ++ chains.append(cur) ++ cur = [] ++ chains.append(cur) ++ ++ while 1: ++ constraints = [] ++ ++ for i in range(len(chains)): ++ l = chains[i] ++ for b in l: ++ for c in b.get_children(): ++ if index[c] < i: ++ forward_p = 0 ++ for inst in b.insts: ++ if inst[0] == 'JUMP_FORWARD': ++ if inst[1] == c: ++ forward_p = 1 ++ if not forward_p: ++ continue ++ constraints.append((index[c], i)) ++ ++ if not constraints: ++ break ++ ++ # XXX just do one for now ++ # do swaps to get things in the right order ++ goes_before, a_chain = constraints[0] ++ assert a_chain > goes_before ++ c = chains[a_chain] ++ chains.remove(c) ++ chains.insert(goes_before, c) ++ ++ del blocks[:] ++ for c in chains: ++ for b in c: ++ blocks.append(b) ++ + def getBlocks(self): + return self.blocks.elements() + +@@ -94,84 +205,27 @@ class FlowGraph: + l.extend(b.getContainedGraphs()) + return l + +- +-def order_blocks(start_block, exit_block): +- """Order blocks so that they are emitted in the right order""" +- # Rules: +- # - when a block has a next block, the next block must be emitted just after +- # - when a block has followers (relative jumps), it must be emitted before +- # them +- # - all reachable blocks must be emitted ++def dfs_postorder(b, seen): ++ """Depth-first search of tree rooted at b, return in postorder""" + order = [] +- +- # Find all the blocks to be emitted. +- remaining = set() +- todo = [start_block] +- while todo: +- b = todo.pop() +- if b in remaining: +- continue +- remaining.add(b) +- for c in b.get_children(): +- if c not in remaining: +- todo.append(c) +- +- # A block is dominated by another block if that block must be emitted +- # before it. +- dominators = {} +- for b in remaining: +- if __debug__ and b.next: +- assert b is b.next[0].prev[0], (b, b.next) +- # Make sure every block appears in dominators, even if no +- # other block must precede it. +- dominators.setdefault(b, set()) +- # preceding blocks dominate following blocks +- for c in b.get_followers(): +- while 1: +- dominators.setdefault(c, set()).add(b) +- # Any block that has a next pointer leading to c is also +- # dominated because the whole chain will be emitted at once. +- # Walk backwards and add them all. +- if c.prev and c.prev[0] is not b: +- c = c.prev[0] +- else: +- break +- +- def find_next(): +- # Find a block that can be emitted next. +- for b in remaining: +- for c in dominators[b]: +- if c in remaining: +- break # can't emit yet, dominated by a remaining block +- else: +- return b +- assert 0, 'circular dependency, cannot find next block' +- +- b = start_block +- while 1: +- order.append(b) +- remaining.discard(b) +- if b.next: +- b = b.next[0] ++ seen[b] = b ++ for c in b.get_children(): ++ if c in seen: + continue +- elif b is not exit_block and not b.has_unconditional_transfer(): +- order.append(exit_block) +- if not remaining: +- break +- b = find_next() ++ order = order + dfs_postorder(c, seen) ++ order.append(b) + return order + +- + class Block: + _count = 0 + + def __init__(self, label=''): + self.insts = [] +- self.outEdges = set() ++ self.inEdges = misc.Set() ++ self.outEdges = misc.Set() + self.label = label + self.bid = Block._count + self.next = [] +- self.prev = [] + Block._count = Block._count + 1 + + def __repr__(self): +@@ -187,46 +241,51 @@ class Block: + + def emit(self, inst): + op = inst[0] ++ if op[:4] == 'JUMP': ++ self.outEdges.add(inst[1]) + self.insts.append(inst) + + def getInstructions(self): + return self.insts + ++ def addInEdge(self, block): ++ self.inEdges.add(block) ++ + def addOutEdge(self, block): + self.outEdges.add(block) + + def addNext(self, block): + self.next.append(block) + assert len(self.next) == 1, map(str, self.next) +- block.prev.append(self) +- assert len(block.prev) == 1, map(str, block.prev) + +- _uncond_transfer = ('RETURN_VALUE', 'RAISE_VARARGS', +- 'JUMP_ABSOLUTE', 'JUMP_FORWARD', 'CONTINUE_LOOP', +- ) ++ _uncond_transfer = ('RETURN_VALUE', 'RAISE_VARARGS', 'YIELD_VALUE', ++ 'JUMP_ABSOLUTE', 'JUMP_FORWARD', 'CONTINUE_LOOP') + +- def has_unconditional_transfer(self): +- """Returns True if there is an unconditional transfer to an other block +- at the end of this block. This means there is no risk for the bytecode +- executer to go past this block's bytecode.""" ++ def pruneNext(self): ++ """Remove bogus edge for unconditional transfers ++ ++ Each block has a next edge that accounts for implicit control ++ transfers, e.g. from a JUMP_IF_FALSE to the block that will be ++ executed if the test is true. ++ ++ These edges must remain for the current assembler code to ++ work. If they are removed, the dfs_postorder gets things in ++ weird orders. However, they shouldn't be there for other ++ purposes, e.g. conversion to SSA form. This method will ++ remove the next edge when it follows an unconditional control ++ transfer. ++ """ + try: + op, arg = self.insts[-1] + except (IndexError, ValueError): + return +- return op in self._uncond_transfer ++ if op in self._uncond_transfer: ++ self.next = [] + + def get_children(self): +- return list(self.outEdges) + self.next +- +- def get_followers(self): +- """Get the whole list of followers, including the next block.""" +- followers = set(self.next) +- # Blocks that must be emitted *after* this one, because of +- # bytecode offsets (e.g. relative jumps) pointing to them. +- for inst in self.insts: +- if inst[0] in PyFlowGraph.hasjrel: +- followers.add(inst[1]) +- return followers ++ if self.next and self.next[0] in self.outEdges: ++ self.outEdges.remove(self.next[0]) ++ return self.outEdges.elements() + self.next + + def getContainedGraphs(self): + """Return all graphs contained within this block. +@@ -387,18 +446,18 @@ class PyFlowGraph(FlowGraph): + elif inst[0] != "SET_LINENO": + pc = pc + 3 + opname = inst[0] +- if opname in self.hasjrel: ++ if self.hasjrel.has_elt(opname): + oparg = inst[1] + offset = begin[oparg] - pc + insts[i] = opname, offset +- elif opname in self.hasjabs: ++ elif self.hasjabs.has_elt(opname): + insts[i] = opname, begin[inst[1]] + self.stage = FLAT + +- hasjrel = set() ++ hasjrel = misc.Set() + for i in dis.hasjrel: + hasjrel.add(dis.opname[i]) +- hasjabs = set() ++ hasjabs = misc.Set() + for i in dis.hasjabs: + hasjabs.add(dis.opname[i]) + +diff --git a/Lib/compiler/pycodegen.py b/Lib/compiler/pycodegen.py +index 6515945..3f641ba 100644 +--- a/Lib/compiler/pycodegen.py ++++ b/Lib/compiler/pycodegen.py +@@ -706,7 +706,7 @@ class CodeGenerator: + self.startBlock(anchor) + self.emit('POP_BLOCK') + self.setups.pop() +- self.nextBlock(end) ++ self.startBlock(end) + + self.emit('LOAD_CONST', None) + +-- +2.2.0.rc0.207.ga3a616c + -- 2.30.9