master: notify replicating nodes of aborted watched transactions

This fixes stuck replication when a client loses connection to the master
during a commit.

neo/master/app.py

@@ -596,3 +596,12 @@ class Application(BaseApplication):
     def getStorageReadySet(self, readiness=float('inf')):
         return {k for k, v in self.storage_ready_dict.iteritems()
                   if v <= readiness}
+
+    def notifyTransactionAborted(self, ttid, uuids):
+        uuid_set = self.getStorageReadySet()
+        uuid_set.intersection_update(uuids)
+        if uuid_set:
+            p = Packets.AbortTransaction(ttid, ())
+            getByUUID = self.nm.getByUUID
+            for uuid in uuid_set:
+                getByUUID(uuid).send(p)

neo/master/handlers/client.py

@@ -27,7 +27,8 @@ class ClientServiceHandler(MasterHandler):
         app = self.app
         node = app.nm.getByUUID(conn.getUUID())
         assert node is not None, conn
-        app.tm.clientLost(node)
+        for x in app.tm.clientLost(node):
+            app.notifyTransactionAborted(*x)
         node.setUnknown()
         app.broadcastNodesInformation([node])
 
@@ -118,12 +119,7 @@ class ClientServiceHandler(MasterHandler):
         app = self.app
         involved = app.tm.abort(tid, conn.getUUID())
         involved.update(uuid_list)
-        involved.intersection_update(app.getStorageReadySet())
-        if involved:
-            p = Packets.AbortTransaction(tid, ())
-            getByUUID = app.nm.getByUUID
-            for involved in involved:
-                getByUUID(involved).send(p)
+        app.notifyTransactionAborted(tid, involved)
 
 
 # like ClientServiceHandler but read-only & only for tid <= backup_tid

neo/master/transactions.py

@@ -444,7 +444,9 @@ class TransactionManager(EventQueue):
     def clientLost(self, node):
         for txn in self._ttid_dict.values():
             if txn.clientLost(node):
-                del self[txn.getTTID()]
+                tid = txn.getTTID()
+                del self[tid]
+                yield tid, txn.getNotificationUUIDList()
 
     def log(self):
         logging.info('Transactions:')

neo/tests/master/testTransactions.py

@@ -61,8 +61,8 @@ class testTransactionManager(NeoUnitTestBase):
         tid1 = self.getNextTID()
         tid2 = self.getNextTID()
         tm.begin(node1, 0, tid1)
-        tm.clientLost(node1)
-        self.assertTrue(tid1 not in tm)
+        self.assertEqual(1, len(list(tm.clientLost(node1))))
+        self.assertNotIn(tid1, tm)
 
 if __name__ == '__main__':
     unittest.main()