First attempt to organise the code

TODO

+Bugs :
+    When no peer is avalaible without the --no-boot option, it crash
+
+To be discuss:
+    Remove the --no-boot option since we know when no node is avalaible
+
+    Maybe we should notify the server of our existence earlier
+
+    Find a better solution for config than utils.config = config, openv.config = config, ...
+    
+    When I created PeersDB, I thought only be used to access the DB and not do some logic. 
+        We should decide what it is suppose to do :
+            Just access the DB
+            Or manage the peers
+    
+    The organisation of the code
+        vifibnet.py             Just contain the main loop and the init
+        openpvn.py          To launch openvpn processes
+        utils.py            Small functions to do some usefull job
+        db.py               Function to access the DB (merge with utils ?)
+        tunnelmanager.py    To choose wich connection delete/keep/...
+        upnpigd.py          To open a port and find the external IP
+        config.py           If we find a way to be able to juste have to import config evrywhere,
+                                call config.init() from main and then have the config evrywhere, 
+                                it would be great
+    
+    How we choose which protocol we use :
+        IMO, we should use UDP. I've read many times than TCP other TCP can be catastrophic in terme of performance 
+        Every time a packet is lost, it is resend 2 times, one for each TCP tunnel
+        And many GW allow UDP port forwarding (for bittorent, Xbox, ...) but not TCP port forwarding
+
+    Use peers_db.populate(100) every once in a while ?

db.py

+from utils import *
+import sqlite3
+import xmlrpclib
+
+class PeersDB:
+    def __init__(self, dbPath):
+
+        log('Connectiong to peers database', 4)
+        self.db = sqlite3.connect(dbPath, isolation_level=None)
+        log('Preparing peers database', 4)
+        try:
+            self.db.execute("UPDATE peers SET used = 0")
+        except sqlite3.OperationalError, e:
+            if e.args[0] == 'no such table: peers':
+                raise RuntimeError
+
+    def populate(self, n):
+        # TODO: don't reconnect to server each time ?
+        log('Connecting to remote server', 3)
+        self.proxy = xmlrpclib.ServerProxy('http://%s:%u' % (config.server, config.server_port))
+        log('Updating peers database : populating', 2)
+        # TODO: determine port and proto
+        port = 1194
+        proto = 'udp'
+        new_peer_list = self.proxy.getPeerList(n, (config.internal_ip, config.external_ip, port, proto))
+        self.db.executemany("INSERT OR IGNORE INTO peers (ip, port, proto, used) VALUES (?,?,?,0)", new_peer_list)
+        self.db.execute("DELETE FROM peers WHERE ip = ?", (config.external_ip,))
+
+    def getUnusedPeers(self, nPeers):
+        return self.db.execute("SELECT id, ip, port, proto FROM peers WHERE used = 0 "
+                "ORDER BY RANDOM() LIMIT ?", (nPeers,))
+
+    def usePeer(self, id):
+        log('Updating peers database : using peer ' + str(id), 5)
+        self.db.execute("UPDATE peers SET used = 1 WHERE id = ?", (id,))
+
+    def unusePeer(self, id):
+        log('Updating peers database : unusing peer ' + str(id), 5)
+        self.db.execute("UPDATE peers SET used = 0 WHERE id = ?", (id,))

log.py

-import time
-
-def log(message, verbose_level):
-    if verbose >= verbose_level:
-        print time.strftime("%d-%m-%Y %H:%M:%S : " + message)
-

tunnelmanager.py

+import os, random
+import openvpn
+import utils
+import db
+
+connection_dict = {} # to remember current connections we made
+free_interface_set = set(('client1', 'client2', 'client3', 'client4', 'client5',
+                          'client6', 'client7', 'client8', 'client9', 'client10'))
+
+def startNewConnections(n, write_pipe):
+    try:
+        for peer_id, ip, port, proto in peers_db.getUnusedPeers(n):
+            utils.log('Establishing a connection with id %s (%s:%s)' % (peer_id, ip, port), 2)
+            iface = free_interface_set.pop()
+            connection_dict[peer_id] = ( openvpn.client( ip, write_pipe, '--dev', iface, '--proto', proto, '--rport', str(port),
+                stdout=os.open(os.path.join(config.log, 'vifibnet.client.%s.log' % (peer_id,)), 
+                               os.O_WRONLY|os.O_CREAT|os.O_TRUNC) ),
+                iface)
+            peers_db.usePeer(peer_id)
+    except KeyError:
+        utils.log("Can't establish connection with %s : no available interface" % ip, 2)
+    except Exception:
+        traceback.print_exc()
+
+def killConnection(peer_id):
+    try:
+        utils.log('Killing the connection with id ' + str(peer_id), 2)
+        p, iface = connection_dict.pop(peer_id)
+        p.kill()
+        free_interface_set.add(iface)
+        peers_db.unusePeer(peer_id)
+    except KeyError:
+        utils.log("Can't kill connection to " + peer_id + ": no existing connection", 1)
+        pass
+    except Exception:
+        utils.log("Can't kill connection to " + peer_id + ": uncaught error", 1)
+        pass
+
+def checkConnections():
+    for id in connection_dict.keys():
+        p, iface = connection_dict[id]
+        if p.poll() != None:
+            utils.log('Connection with %s has failed with return code %s' % (id, p.returncode), 3)
+            free_interface_set.add(iface)
+            peers_db.unusePeer(id)
+            del connection_dict[id]
+
+def refreshConnections(write_pipe):
+    checkConnections()
+    # Kill some random connections
+    try:
+        for i in range(0, max(0, len(connection_dict) - config.client_count + config.refresh_count)):
+            peer_id = random.choice(connection_dict.keys())
+            killConnection(peer_id)
+    except Exception:
+        pass
+    # Establish new connections
+    startNewConnections(config.client_count - len(connection_dict), write_pipe)
+

utils.py

+import time
+
+def log(message, verbose_level):
+    if verbose >= verbose_level:
+        print time.strftime("%d-%m-%Y %H:%M:%S : " + message)
+
+def ipFromBin(prefix):
+    prefix = hex(int(prefix, 2))[2:]
+    ip = ''
+    for i in xrange(0, len(prefix) - 1, 4):
+        ip += prefix[i:i+4] + ':'
+    return ip.rstrip(':')
+
+def ipFromPrefix(vifibnet, prefix, prefix_len):
+    prefix = bin(int(prefix))[2:].rjust(prefix_len, '0')
+    ip_t = (vifibnet + prefix).ljust(128, '0')
+    return ipFromBin(ip_t)

vifibnet.py

 #!/usr/bin/env python
-import argparse, errno, math, os, select, sqlite3, subprocess, sys, time, xmlrpclib
+import argparse, errno, math, os, select, subprocess, sys, time
 from OpenSSL import crypto
 import traceback
 import upnpigd
 import openvpn
-import random
-import log
-
-connection_dict = {} # to remember current connections we made
-free_interface_set = set(('client1', 'client2', 'client3', 'client4', 'client5',
-                          'client6', 'client7', 'client8', 'client9', 'client10'))
-
-# TODO: flag in some way the peers that are connected to us so we don't connect to them
-# Or maybe we just don't care
-
-class PeersDB:
-    def __init__(self, dbPath):
-
-        log.log('Connectiong to peers database', 4)
-        self.db = sqlite3.connect(dbPath, isolation_level=None)
-        log.log('Preparing peers database', 4)
-        try:
-            self.db.execute("UPDATE peers SET used = 0")
-        except sqlite3.OperationalError, e:
-            if e.args[0] == 'no such table: peers':
-                raise RuntimeError
-
-    def populate(self, n):
-        # TODO: don't reconnect to server each time ?
-        log.log('Connecting to remote server', 3)
-        self.proxy = xmlrpclib.ServerProxy('http://%s:%u' % (config.server, config.server_port))
-        log.log('Updating peers database : populating', 2)
-        # TODO: determine port and proto
-        port = 1194
-        proto = 'udp'
-        new_peer_list = self.proxy.getPeerList(n, (config.internal_ip, config.external_ip, port, proto))
-        self.db.executemany("INSERT OR IGNORE INTO peers (ip, port, proto, used) VALUES (?,?,?,0)", new_peer_list)
-        self.db.execute("DELETE FROM peers WHERE ip = ?", (config.external_ip,))
-
-    def getUnusedPeers(self, nPeers):
-        return self.db.execute("SELECT id, ip, port, proto FROM peers WHERE used = 0 "
-                "ORDER BY RANDOM() LIMIT ?", (nPeers,))
-
-    def usePeer(self, id):
-        log.log('Updating peers database : using peer ' + str(id), 5)
-        self.db.execute("UPDATE peers SET used = 1 WHERE id = ?", (id,))
-
-    def unusePeer(self, id):
-        log.log('Updating peers database : unusing peer ' + str(id), 5)
-        self.db.execute("UPDATE peers SET used = 0 WHERE id = ?", (id,))
-
-def ipFromBin(prefix):
-    prefix = hex(int(prefix, 2))[2:]
-    ip = ''
-    for i in xrange(0, len(prefix) - 1, 4):
-        ip += prefix[i:i+4] + ':'
-    return ip.rstrip(':')
-
-def ipFromPrefix(prefix, prefix_len):
-    prefix = bin(int(prefix))[2:].rjust(prefix_len, '0')
-    ip_t = (config.vifibnet + prefix).ljust(128, '0')
-    return ipFromBin(ip_t)
+import utils
+import db
+import tunnelmanager
 
 def startBabel(**kw):
     args = ['babeld',
             '-C', 'redistribute local ip %s' % (config.internal_ip),
             '-C', 'redistribute local deny',
             # Route VIFIB ip adresses
-            '-C', 'in ip %s::/%u' % (ipFromBin(config.vifibnet), len(config.vifibnet)),
+            '-C', 'in ip %s::/%u' % (utils.ipFromBin(config.vifibnet), len(config.vifibnet)),
             # Route only addresse in the 'local' network,
             # or other entire networks
             #'-C', 'in ip %s' % (config.internal_ip),
@@ -79,7 +25,7 @@ def startBabel(**kw):
             ]
     if config.babel_state:
         args += '-S', config.babel_state
-    args = args + ['vifibnet'] + list(free_interface_set)
+    args = args + ['vifibnet'] + list(tunnelmanager.free_interface_set)
     if config.verbose >= 5:
         print args
     return subprocess.Popen(args, **kw)
@@ -123,7 +69,9 @@ def getConfig():
     _('openvpn_args', nargs=argparse.REMAINDER,
             help="Common OpenVPN options (e.g. certificates)")
     openvpn.config = config = parser.parse_args()
-    log.verbose = config.verbose
+    tunnelmanager.config = config
+    db.config = config
+    utils.verbose = config.verbose
 
     # Get network prefix from ca.crt
     with open(config.ca, 'r') as f:
@@ -135,8 +83,8 @@ def getConfig():
         cert = crypto.load_certificate(crypto.FILETYPE_PEM, f.read())
         subject = cert.get_subject()
         prefix, prefix_len = subject.CN.split('/')
-        config.internal_ip = ipFromPrefix(prefix, int(prefix_len))
-        log.log('Intranet ip : %s' % (config.internal_ip,), 3)
+        config.internal_ip = utils.ipFromPrefix(config.vifibnet, prefix, int(prefix_len))
+        utils.log('Intranet ip : %s' % (config.internal_ip,), 3)
 
     # Treat openvpn arguments
     if config.openvpn_args[0] == "--":
@@ -146,96 +94,42 @@ def getConfig():
     config.openvpn_args.append('--cert')
     config.openvpn_args.append(config.cert)
 
-    log.log("Configuration completed", 1)
-
-def startNewConnection(n, write_pipe):
-    try:
-        for peer_id, ip, port, proto in peers_db.getUnusedPeers(n):
-            log.log('Establishing a connection with id %s (%s:%s)' % (peer_id, ip, port), 2)
-            iface = free_interface_set.pop()
-            connection_dict[peer_id] = ( openvpn.client( ip, write_pipe, '--dev', iface, '--proto', proto, '--rport', str(port),
-                stdout=os.open(os.path.join(config.log, 'vifibnet.client.%s.log' % (peer_id,)), 
-                               os.O_WRONLY|os.O_CREAT|os.O_TRUNC) ),
-                iface)
-            peers_db.usePeer(peer_id)
-    except KeyError:
-        log.log("Can't establish connection with %s : no available interface" % ip, 2)
-    except Exception:
-        traceback.print_exc()
-
-def killConnection(peer_id):
-    try:
-        log.log('Killing the connection with id ' + str(peer_id), 2)
-        p, iface = connection_dict.pop(peer_id)
-        p.kill()
-        free_interface_set.add(iface)
-        peers_db.unusePeer(peer_id)
-    except KeyError:
-        log.log("Can't kill connection to " + peer_id + ": no existing connection", 1)
-        pass
-    except Exception:
-        log.log("Can't kill connection to " + peer_id + ": uncaught error", 1)
-        pass
-
-def checkConnections():
-    for id in connection_dict.keys():
-        p, iface = connection_dict[id]
-        if p.poll() != None:
-            log.log('Connection with %s has failed with return code %s' % (id, p.returncode), 3)
-            free_interface_set.add(iface)
-            peers_db.unusePeer(id)
-            del connection_dict[id]
-
-def refreshConnections(write_pipe):
-    checkConnections()
-    # Kill some random connections
-    try:
-        for i in range(0, max(0, len(connection_dict) - config.client_count + config.refresh_count)):
-            peer_id = random.choice(connection_dict.keys())
-            killConnection(peer_id)
-    except Exception:
-        pass
-    # Establish new connections
-    startNewConnection(config.client_count - len(connection_dict), write_pipe)
+    utils.log("Configuration completed", 1)
 
 def handle_message(msg):
     script_type, arg = msg.split()
     if script_type == 'client-connect':
-        log.log('Incomming connection from %s' % (arg,), 3)
+        utils.log('Incomming connection from %s' % (arg,), 3)
         # TODO: check if we are not already connected to it
     elif script_type == 'client-disconnect':
-        log.log('%s has disconnected' % (arg,), 3)
+        utils.log('%s has disconnected' % (arg,), 3)
     elif script_type == 'route-up':
         # TODO: save the external ip received
-        log.log('External Ip : ' + arg, 3)
+        utils.log('External Ip : ' + arg, 3)
     else:
-        log.log('Unknow message recieved from the openvpn pipe : ' + msg, 1)
+        utils.log('Unknow message recieved from the openvpn pipe : ' + msg, 1)
 
 def main():
     # Get arguments
     getConfig()
-    log.verbose = config.verbose
-    # TODO: how do we decide which protocol we use ?
-    # (externalIp, externalPort) = upnpigd.GetExternalInfo(1194)
 
     # Setup database
-    global peers_db # stop using global variables for everything ?
-    peers_db = PeersDB(config.db)
+    tunnelmanager.peers_db = db.PeersDB(config.db)
 
     # Launch babel on all interfaces. WARNING : you have to be root to start babeld
-    log.log('Starting babel', 3)
+    utils.log('Starting babel', 3)
     babel = startBabel(stdout=os.open(os.path.join(config.log, 'vifibnet.babeld.log'), os.O_WRONLY | os.O_CREAT | os.O_TRUNC), stderr=subprocess.STDOUT)
 
     # Create and open read_only pipe to get connect/disconnect events from openvpn
-    log.log('Creating pipe for openvpn events', 3)
+    utils.log('Creating pipe for openvpn events', 3)
     r_pipe, write_pipe = os.pipe()
     read_pipe = os.fdopen(r_pipe)
 
     # Establish connections
-    log.log('Starting openvpn server', 3)
+    utils.log('Starting openvpn server', 3)
     serverProcess = openvpn.server(config.internal_ip, write_pipe, '--dev', 'vifibnet',
             stdout=os.open(os.path.join(config.log, 'vifibnet.server.log'), os.O_WRONLY | os.O_CREAT | os.O_TRUNC))
-    startNewConnection(config.client_count, write_pipe)
+    tunnelmanager.startNewConnections(config.client_count, write_pipe)
 
     # Timed refresh initializing
     next_refresh = time.time() + config.refresh_time
@@ -249,7 +143,7 @@ def main():
             if ready:
                 handle_message(read_pipe.readline())
             if time.time() >= next_refresh:
-                peers_db.populate(10)
+                tunnelmanager.peers_db.populate(10)
                 refreshConnections(write_pipe)
                 next_refresh = time.time() + config.refresh_time
     except KeyboardInterrupt: