DB and tunnel now have a separate refresh timer

Changes in the number of connection per peers

README

-vifibnet is a daemon setting up a resilient virtual private network over the internet
+Vsifibnet is a daemon setting up a resilient virtual private network over the internet
+
+The organisation of the code
+    vifibnet.py         Just contain the main loop and the init
+    plib.py             To launch server/client/routing processes
+    utils.py            Small functions to do some usefull job
+    db.py               Function to manage peers
+    tunnel.py           To choose wich connection delete/keep/...
+    upnpigd.py          To open a port
\ No newline at end of file

TODO

@@ -9,25 +9,24 @@ To be done :
     Replace comments at the beginning of functions with docstrings & give all fn docstrings
     Do a clean-up in the import
     Remove the parameters to choose the number of clients
-    Use the server events ( client connection/deconnection ) to do something useful
+    Use the server events ( client connection/deconnection ) to do something usefull
+    In peers DB, remove some peers when they are too many of them
+    Contact the server using vifibnet and not the underlying network when possible
 
 To be discuss:
-    Remove the --no-boot option since we know when no node is avalaible
-       \=> the no-boot option is only useful when the server knows no peer,
-           irl it should never happen, no-boot is a debug option
-
-    The organisation of the code
-        vifibnet.py         Just contain the main loop and the init
-        plib.py             To launch server/client/routing processes
-        utils.py            Small functions to do some usefull job
-        db.py               Function to manage peers
-        tunnelmanager.py    To choose wich connection delete/keep/...
-        upnpigd.py          To open a port and find the external IP
+    U : Remove the --no-boot option since we know when no node is avalaible
+    G : the no-boot option is only useful when the server knows no peer,
+        irl it should never happen, no-boot is a debug option
+    U : Ok, but the server knows when no peers is avalaible, doesn't he ?
 
     How we choose which protocol we use :
         IMO, we should use UDP. I've read many times than TCP other TCP can be catastrophic in terme of performance
         Every time a packet is lost, it is resend 2 times, one for each TCP tunnel
         And many GW allow UDP port forwarding (for bittorent, Xbox, ...) but not TCP port forwarding
 
-    Use peers_db.populate(100) every once in a while ? -> yes but be warry of the refresh time ( populate
-    the db once every 20s is bad.. )
+    Use peers_db.populate(100) every once in a while ?
+    G : yes but be warry of the refresh time ( populate the db once every 20s is bad.. ) 
+    U : I agree. Once evry hours should be sufficient, and when their too few peers avalaible.
+
+    G : don't reconnect to server each time we repopulate in peers_db ?
+    U : we might recontact the server evry 1H or even less. So i think it is a good thing not to keep the connection alive

db.py

 #!/usr/bin/env python
-import sqlite3, xmlrpclib
+import sqlite3, xmlrpclib, time
 import utils
 
 class PeerManager:
 
-    def __init__(self, dbPath, server, port):
+    def __init__(self, dbPath, server, port, refresh_time):
         utils.log('Connectiong to peers database', 4)
         self._db = sqlite3.connect(dbPath, isolation_level=None)
         self._server = server
         self._port = port
+        self._refresh_time = refresh_time
         utils.log('Preparing peers database', 4)
         try:
             self._db.execute("UPDATE peers SET used = 0")
         except sqlite3.OperationalError, e:
             if e.args[0] == 'no such table: peers':
                 raise RuntimeError
+        self.next_refresh = time.time()
 
     def populate(self, n, address):
         # address = (internal_ip, external_ip, port, proto)
-        # TODO: don't reconnect to server each time ?
         utils.log('Connecting to remote server', 3)
         self._proxy = xmlrpclib.ServerProxy('http://%s:%u' % (self._server, self._port))
         utils.log('Updating peers database : populating', 2)
         _, external_ip, _, _ = address
         new_peer_list = self._proxy.getPeerList(n, address)
+        utils.log('New peers recieved from %s' % self._server, 5)
         self._db.executemany("INSERT OR IGNORE INTO peers (ip, port, proto, used) VALUES (?,?,?,0)", new_peer_list)
         self._db.execute("DELETE FROM peers WHERE ip = ?", (external_ip,))
+        self.next_refresh = time.time() + self._refresh_time
+        utils.log('New peers : %s' % ', '.join(map(str, new_peer_list)), 5)
 
     def getUnusedPeers(self, nPeers):
         return self._db.execute("SELECT id, ip, port, proto FROM peers WHERE used = 0 "

tunnel.py

 #!/usr/bin/env python
-import os, random, traceback
+import os, random, traceback, time
 import plib, utils, db
 
 log = None
 
 class TunnelManager:
 
-    def __init__(self, write_pipe, peer_db, client_count, refresh_count, openvpn_args):
+    def __init__(self, write_pipe, peer_db, openvpn_args, refresh, connection_count, refresh_rate):
         self._write_pipe = write_pipe
         self._peer_db = peer_db
         self._connection_dict = {}
-        self._client_count = client_count
-        self._refresh_count = refresh_count
         self._ovpn_args = openvpn_args
+        self._refresh_time = refresh
         self.free_interface_set = set(('client1', 'client2', 'client3', 'client4', 'client5',
                                        'client6', 'client7', 'client8', 'client9', 'client10'))
+        self.next_refresh = time.time()
+
+        # TODO : choose this automatically
+        self._client_count = connection_count/2
+        self._refresh_count = refresh_rate*self._client_count
 
     def refresh(self):
+        utils.log('Refreshing the tunnels', 2)
         self._cleanDeads()
         self._removeSomeTunnels()
         self._makeNewTunnels()
+        self.next_refresh = time.time() + self._refresh_time
 
     def _cleanDeads(self):
         for id in self._connection_dict.keys():
@@ -43,6 +49,7 @@ class TunnelManager:
         self._peer_db.unusePeer(peer_id)
 
     def _makeNewTunnels(self):
+        utils.log('Making %i new tunnels' % (self._client_count - len(self._connection_dict)), 3)
         try:
             for peer_id, ip, port, proto in self._peer_db.getUnusedPeers(self._client_count - len(self._connection_dict)):
                 utils.log('Establishing a connection with id %s (%s:%s)' % (peer_id, ip, port), 2)

vifibnet.py

@@ -14,15 +14,10 @@ def getConfig():
             help='Peer discovery server port')
     _('-l', '--log', default='/var/log',
             help='Path to vifibnet logs directory')
-    _('--client-count', default=2, type=int,
-            help='Number of client connections')
-    # TODO: use maxpeer
-    _('--max-clients', default=10, type=int,
-            help='the number of peers that can connect to the server')
-    _('--refresh-time', default=300, type=int,
+    _('--tunnel-refresh', default=300, type=int,
             help='the time (seconds) to wait before changing the connections')
-    _('--refresh-count', default=1, type=int,
-            help='The number of connections to drop when refreshing the connections')
+    _('--peers-db-refresh', default=3600, type=int,
+            help='the time (seconds) to wait before refreshing the peers db')
     _('--db', default='/var/lib/vifibnet/peers.db',
             help='Path to peers database')
     _('--dh', required=True,
@@ -37,6 +32,11 @@ def getConfig():
             help='Path to the certificate file')
     _('--ip', required=True, dest='external_ip',
             help='Ip address of the machine on the internet')
+    # args to be removed ?
+    _('--connection-count', default=30, type=int,
+            help='Number of client connections')
+    _('--refresh-rate', default=0.05, type=float,
+            help='The ratio of connections to drop when refreshing the connections')
     # Openvpn options
     _('openvpn_args', nargs=argparse.REMAINDER,
             help="Common OpenVPN options (e.g. certificates)")
@@ -63,8 +63,8 @@ def main():
     read_pipe = os.fdopen(r_pipe)
 
     # Init db and tunnels
-    peer_db = db.PeerManager(config.db, config.server, config.server_port)
-    tunnel_manager = tunnel.TunnelManager(write_pipe, peer_db, config.client_count, config.refresh_count, openvpn_args)
+    peer_db = db.PeerManager(config.db, config.server, config.server_port, config.peers_db_refresh)
+    tunnel_manager = tunnel.TunnelManager(write_pipe, peer_db, openvpn_args, config.tunnel_refresh, config.connection_count, config.refresh_rate)
 
     # Launch babel on all interfaces. WARNING : you have to be root to start babeld
     interface_list = ['vifibnet'] + list(tunnel_manager.free_interface_set)
@@ -73,25 +73,21 @@ def main():
         os.O_WRONLY | os.O_CREAT | os.O_TRUNC), stderr=subprocess.STDOUT)
 
    # Establish connections
-    server_process = plib.server(internal_ip, network, config.max_clients, config.dh, write_pipe, 
+    server_process = plib.server(internal_ip, network, config.connection_count, config.dh, write_pipe, 
             '--dev', 'vifibnet', *openvpn_args,
             stdout=os.open(os.path.join(config.log, 'vifibnet.server.log'), os.O_WRONLY | os.O_CREAT | os.O_TRUNC))
-    tunnel_manager.refresh()
-
-    # Timed refresh initializing
-    next_refresh = time.time() + config.refresh_time
 
     # main loop
     try:
         while True:
             ready, tmp1, tmp2 = select.select([read_pipe], [], [],
-                    max(0, next_refresh - time.time()))
+                    max(0, min(tunnel_manager.next_refresh, peer_db.next_refresh) - time.time()))
             if ready:
                 peer_db.handle_message(read_pipe.readline())
-            if time.time() >= next_refresh:
-                peer_db.populate(100, (internal_ip, config.external_ip, port, proto))
+            if time.time() >= peer_db.next_refresh:
+                peer_db.populate(200, (internal_ip, config.external_ip, port, proto))
+            if time.time() >= tunnel_manager.next_refresh:
                 tunnel_manager.refresh()
-                next_refresh = time.time() + config.refresh_time
     except KeyboardInterrupt:
         return 0