wip: add more resource cleanups to remove ResourceWarnings

Python3 added automatic detection of leaked resources, which causes lots of warnings in Re6st, mostly because of unclosed subprocess streams.

demo/demo

@@ -83,7 +83,8 @@ if args.duration:
     signal.signal(signal.SIGALRM, handler)
     signal.alarm(args.duration)
 
-exec(compile(open("fixnemu.py", "rb").read(), "fixnemu.py", 'exec'))
+with open('fixnemu.py', 'rb') as f:
+    exec(compile(f.read(), 'fixnemu.py', 'exec'))
 
 
 class Re6stNode(nemu.Node):

demo/py

@@ -30,4 +30,5 @@ def __file__():
 
     return os.path.join(sys.path[0], sys.argv[1])
 __file__ = __file__()
-exec(compile(open(__file__, "rb").read(), __file__, 'exec'))
+with open(__file__) as f:
+    exec(compile(f.read(), __file__, 'exec'))
\ No newline at end of file

re6st/cli/conf.py

@@ -77,11 +77,12 @@ def main():
         print("WARNING: it is strongly recommended to use --fingerprint option.")
     network = x509.networkFromCa(ca)
     if config.is_needed:
-        route, err = subprocess.Popen(('ip', '-6', '-o', 'route', 'get',
-                                       utils.ipFromBin(network)),
-                                      stdout=subprocess.PIPE).communicate()
-        sys.exit(err or route and
-            utils.binFromIp(route.split()[8]).startswith(network))
+        with subprocess.Popen(('ip', '-6', '-o', 'route', 'get',
+                               utils.ipFromBin(network)),
+                              stdout=subprocess.PIPE) as proc:
+            route, err = proc.communicate()
+            sys.exit(err or route and
+                utils.binFromIp(route.split()[8]).startswith(network))
 
     create(ca_path, crypto.dump_certificate(crypto.FILETYPE_PEM, ca))
     if config.ca_only:
@@ -90,7 +91,7 @@ def main():
     reserved = 'CN', 'serial'
     req = crypto.X509Req()
     try:
-        with open(cert_path) as f:
+        with open(cert_path, "rb") as f:
             cert = loadCert(f.read())
         components = dict(cert.get_subject().get_components())
         components = {k.decode(): v for k, v in components.items()}

re6st/cli/node.py

@@ -266,12 +266,12 @@ def main():
 
     def call(cmd):
         logging.debug('%r', cmd)
-        p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
-                                  stderr=subprocess.PIPE)
-        stdout, stderr = p.communicate()
-        if p.returncode:
-            raise EnvironmentError("%r failed with error %u\n%s"
-                                   % (' '.join(cmd), p.returncode, stderr))
+        with subprocess.Popen(cmd, stdout=subprocess.PIPE,
+                                  stderr=subprocess.PIPE) as p:
+            stdout, stderr = p.communicate()
+            if p.returncode:
+                raise EnvironmentError("%r failed with error %u\n%s"
+                                       % (' '.join(cmd), p.returncode, stderr))
         return stdout
     def ip4(object, *args):
         args = ['ip', '-4', object, 'add'] + list(args)

re6st/registry.py

@@ -155,6 +155,11 @@ class RegistryServer:
         else:
             self.newHMAC(0)
 
+    def close(self):
+        self.sock.close()
+        self.db.close()
+        self.ctl.close()
+
     def getConfig(self, name, *default):
         r, = next(self.db.execute(
             "SELECT value FROM config WHERE name=?", (name,)), default)

re6st/tests/test_end2end/test_registry_client.py

@@ -42,6 +42,8 @@ class TestRegistryClientInteract(unittest.TestCase):
 
     def tearDown(self):
         self.server.proc.terminate()
+        with self.server.proc:
+            pass
 
     def test_1_main(self):
         """ a client interact a server, no re6stnet node test basic function"""

re6st/tests/test_network/network_build.py

@@ -62,11 +62,11 @@ class NetManager:
         """
         for reg, nodes in self.registries.items():
             for node in nodes:
-                app0 = node.Popen(["ping", "-c", "1", reg.ip], stdout=PIPE)
-                ret = app0.wait()
-                if ret:
-                    raise ConnectableError(
-                        "network construct failed {} to {}".format(node.ip, reg.ip))
+                with node.Popen(["ping", "-c", "1", reg.ip], stdout=PIPE) as app0:
+                    ret = app0.wait()
+                    if ret:
+                        raise ConnectableError(
+                            "network construct failed {} to {}".format(node.ip, reg.ip))
 
         logging.debug("each node can ping to their registry")
 

re6st/tests/test_network/re6st_wrap.py

@@ -136,6 +136,8 @@ class Re6stRegistry:
         try:
             logging.debug("teminate process %s", self.proc.pid)
             self.proc.destroy()
+            with self.proc:
+                pass
         except:
             pass
 
@@ -262,6 +264,8 @@ class Re6stNode:
         """stop running re6stnet process"""
         logging.debug("%s teminate process %s", self.name, self.proc.pid)
         self.proc.destroy()
+        with self.proc:
+            pass
 
     def __del__(self):
         """teminate process and rm temp dir"""

re6st/tests/test_network/test_net.py

@@ -31,6 +31,14 @@ def deploy_re6st(nm, recreate=False):
             nodes.append(node)
     return nodes, registries
 
+def clean_re6st(nodes, registries):
+    for node in nodes:
+        node.node.destroy()
+        node.stop()
+
+    for reg in registries:
+        reg.__del__()
+
 def wait_stable(nodes, timeout=240):
     """try use ping6 from each node to the other until ping success to all the
     other nodes
@@ -94,20 +102,22 @@ class TestNet(unittest.TestCase):
         """create a network in a net segment, test the connectivity by ping
         """
         nm = network_build.net_route()
-        nodes, _ = deploy_re6st(nm)
+        nodes, registries = deploy_re6st(nm)
 
         wait_stable(nodes, 40)
         time.sleep(10)
 
         self.assertTrue(wait_stable(nodes, 30), " ping test failed")
 
+        clean_re6st(nodes, registries)
+
     @unittest.skip("usually failed due to UPnP problem")
     def test_reboot_one_machine(self):
         """create a network demo, wait the net stable, reboot on machine,
         then test if network recover, this test seems always failed
         """
         nm = network_build.net_demo()
-        nodes, _ = deploy_re6st(nm)
+        nodes, registries = deploy_re6st(nm)
 
         wait_stable(nodes, 100)
 
@@ -121,12 +131,14 @@ class TestNet(unittest.TestCase):
 
         self.assertTrue(wait_stable(nodes, 400), "network can't recover")
 
+        clean_re6st(nodes, registries)
+
     def test_reboot_one_machine_router(self):
         """create a network router, wait the net stable, reboot on machine,
         then test if network recover,
         """
         nm = network_build.net_route()
-        nodes, _ = deploy_re6st(nm)
+        nodes, registries = deploy_re6st(nm)
 
         wait_stable(nodes, 40)
 
@@ -140,6 +152,8 @@ class TestNet(unittest.TestCase):
 
         self.assertTrue(wait_stable(nodes, 100), "network can't recover")
 
+        clean_re6st(nodes, registries)
+
 
 
 if __name__ == "__main__":

re6st/tests/test_unit/test_registry.py

@@ -70,6 +70,7 @@ class TestRegistryServer(unittest.TestCase):
 
     @classmethod
     def tearDownClass(cls):
+        cls.server.close()
         # remove database
         for file in [cls.config.db, cls.config.ca, cls.config.key]:
             try:

re6st/tests/tools.py

@@ -103,7 +103,7 @@ def decrypt(pkey, incontent):
     with open("node.key", 'w') as f:
         f.write(pkey.decode())
     args = "openssl rsautl -decrypt -inkey node.key".split()
-    p = subprocess.Popen(
-        args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    outcontent, err = p.communicate(incontent)
+    with subprocess.Popen(
+        args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p:
+        outcontent, err = p.communicate(incontent)
     return outcontent