Cleaned ( a bit ) vifibnet options

TODO

 To be done :
+    Catch a more precise exception thant Exception at line 108 in vifibnet.py
+    ( UPnP forwarding )
+
+    Upgrade the logging function in order to be able to log message like
+    "Refreshing peers DB ... done", or add log messages to specify that an
+    action advertised by a previous log message has been completed
+
     Add options to start udp server, tcp server or both
 
     use the server as a bootstrap node
@@ -16,9 +23,10 @@ To be done :
     Use the server events ( client connection/deconnection ) to do something
     useful
 
-    In peers DB, flag the dead peers so we only choose them if necessary and we can remove them if we have enought peers
+    In peers DB, flag the dead peers so we only choose them if necessary and we
+    can remove them if we have enought peers
 
-    Use a timeout for the server peersDB so we can rflag unreachable peers and
+    Use a timeout for the server peersDB so we can flag unreachable peers and
     remove the peers whose certificate is no longer valid
 
     Specify a lease duration in ForwardViaUPnP
@@ -26,12 +34,6 @@ To be done :
     Handle LAN internally in order not to have catastrophic results ....
 
 To be discussed:
-    G : Database structure for bith vifibnet and registry have been changed.
-        Index is now always on the prefix ( there is no id anymore ). And
-        the (ip, port, proto) tuples have been replaced with addresses :
-        it is a list of ip, port, proto, that way a peer can announce
-        different (port, proto) combination.
-
     G, J : To get traffic stats ( bytes in/out ), you can use
           /sys/class/net/interface/statistics/rx_bytes, etc...
         or /proc/net/dev/snmp6/interface ( all in one file ). This can be enough
@@ -61,6 +63,12 @@ To be discussed:
         45% of the problems dont last more than 2 minutes, 55% no more than
         3 minutes If it takes 2 min to detect a dead connection, then we wont be
         solving many problems with our overlay network
+    G : ok, so babel hello-interval should be set to a lower value,
+        we should do some tests to pinpoint the best compromise between
+        speed and bandwith usage.
+        Btw, is there a doc ( pdf, image, file ) resuming Raphael's stats
+        on nexedi's server downtime ? it could be useful for the internship
+        rapport
 
     U : The peer DB size should depend on the number of connection and the
         refresh time
@@ -71,6 +79,7 @@ To be discussed:
         enought DB to ensure we can still choose a peer as if it was choosen
         directly from the server. The requiered db size can be calculated from
         the number of connections and the refresh time.
+    G : ok, you can erase this talk
 
     U : Why are --ip and internal-port mutually exclusive ?
         Currently upnp only forward via UDP. Should he also forward via TCP ?
@@ -78,3 +87,11 @@ To be discussed:
         No error should be raised when no upnp is detected : we should allow
         machines having public IP to do an automatic configuration using the
         discovery by an other peer
+    G : Actually, i was wrong, --ip and internal-port are no longer exclusive
+        Julien said udp might not be used by some people because of
+        restrictions imposed by the ISP ( FAI in french ), so we should
+        allow both, and act according to the options specifying which servers
+        to start (upd, tcp-server)
+
+    G : I think the number of route going through an interface should be a
+        Connection attribute, not a dict in tunnelManager

db.py

-import sqlite3, socket, xmlrpclib, time
+import sqlite3, socket, xmlrpclib, time, os
 import utils
 
 class PeerManager:
 
     # internal ip = temp arg/attribute
-    def __init__(self, db_path, server, server_port, refresh_time, address, internal_ip, prefix, manual, db_size):
+    def __init__(self, db_dir_path, server, server_port, refresh_time, address,
+                       internal_ip, prefix, manual, proto, db_size):
         self._refresh_time = refresh_time
         self._address = address
         self._internal_ip = internal_ip
@@ -12,12 +13,14 @@ class PeerManager:
         self._server = server
         self._server_port = server_port
         self._db_size = db_size
+        self._proto = proto
         self._manual = manual
 
         self._proxy = xmlrpclib.ServerProxy('http://%s:%u' % (server, server_port))
 
         utils.log('Connectiong to peers database', 4)
-        self._db = sqlite3.connect(db_path, isolation_level=None)
+        self._db = sqlite3.connect(os.path.join(db_dir_path, 'peers.db'),
+                                   isolation_level=None)
         utils.log('Preparing peers database', 4)
         try:
             self._db.execute("UPDATE peers SET used = 0")
@@ -34,7 +37,7 @@ class PeerManager:
             self._populate()
             self.next_refresh = time.time() + self._refresh_time
         except socket.error, e:
-            utils.log(str(e), 3)
+            utils.log(str(e), 4)
             utils.log('Connection to server failed, retrying in 30s', 2)
             self.next_refresh = time.time() + 30
 
@@ -73,8 +76,8 @@ class PeerManager:
         elif script_type == 'route-up':
             if not self._manual:
                 external_ip, external_port = arg.split(',')
-                new_address = [[external_ip, external_port, 'udp'],
-                                 [external_ip, external_port, 'tcp-client']]
+                new_address = list([external_ip, external_port, proto]
+                                   for proto in self._proto)
                 if self._address != new_address:
                     self._address = new_address
                     utils.log('Received new external configuration : %s:%s' % (external_ip, external_port), 3)

plib.py

@@ -43,7 +43,7 @@ def client(server_address, pipe_fd, hello_interval, *args, **kw):
     return openvpn(hello_interval, *remote, **kw)
 
 def router(network, internal_ip, interface_list,
-           wireless, hello_interval, **kw):
+           wireless, hello_interval, state_path, **kw):
     utils.log('Starting babel', 3)
     args = ['babeld',
             '-C', 'redistribute local ip %s' % (internal_ip),
@@ -59,10 +59,9 @@ def router(network, internal_ip, interface_list,
             '-d', str(verbose),
             '-h', str(hello_interval),
             '-H', str(hello_interval),
+            '-S', state_path,
             '-s',
             ]
-    #if utils.config.babel_state:
-    #    args += '-S', utils.config.babel_state
     if wireless:
         args.append('-w')
     args = args + interface_list

vifibnet.py

 #!/usr/bin/env python
-import argparse, errno, math, os, select, subprocess, sys, time, traceback
+import argparse, errno, os, select, subprocess, time
 from argparse import ArgumentParser
-from OpenSSL import crypto
 import db, plib, upnpigd, utils, tunnel
 
 class ArgParser(ArgumentParser):
@@ -27,46 +26,54 @@ def getConfig():
     parser = ArgParser(fromfile_prefix_chars='@',
             description='Resilient virtual private network application')
     _ = parser.add_argument
+
+    # General Configuration options
+    _('--ip', default=None, dest='address', action='append', nargs=3,
+            help='Ip address, port and protocol advertised to other vpn nodes')
+    _('--internal-port', default=1194,
+            help='Port on the machine to listen on for incomming connections')
+    _('--peers-db-refresh', default=3600, type=int,
+            help='the time (seconds) to wait before refreshing the peers db')
+    _('-log', '-l', default='/var/log',
+            help='Path to vifibnet logs directory')
+    _('-s', '--state', default='/var/lib/vifibnet',
+            help='Path to VPN state directory')
+    _('--verbose', '-v', default=0, type=int,
+            help='Defines the verbose level')
+    #_('--babel-state', default='/var/lib/vifibnet/babel_state',
+    #        help='Path to babeld state-file')
+    #_('--db', default='/var/lib/vifibnet/peers.db',
+    #        help='Path to peers database')
     # Server address SHOULD be a vifib address ( else requests will be denied )
     _('--server', required=True,
             help="VPN address of the discovery peer server")
     _('--server-port', required=True, type=int,
             help="VPN port of the discovery peer server")
-    _('-log', '-l', default='/var/log',
-            help='Path to vifibnet logs directory')
+
+    # Routing algorithm options
+    _('--hello', type=int, default=30,
+            help='Hello interval for babel, in seconds')
+    _('-w', '--wireless', action='store_true',
+            help='''Set all interfaces to be treated as wireless interfaces
+                    for the routing protocol''')
+
+    # Tunnel options
+    _('--proto', choices=['udp', 'tcp-server'], nargs='+', default=['udp'],
+            help='Protocol(s) to be used by other peers to connect')
     _('--tunnel-refresh', default=300, type=int,
             help='the time (seconds) to wait before changing the connections')
-    _('--peers-db-refresh', default=3600, type=int,
-            help='the time (seconds) to wait before refreshing the peers db')
-    _('--db', default='/var/lib/vifibnet/peers.db',
-            help='Path to peers database')
     _('--dh', required=True,
             help='Path to dh file')
-    _('--babel-state', default='/var/lib/vifibnet/babel_state',
-            help='Path to babeld state-file')
-    _('--hello', type=int, default=30,
-            help='Hello interval for babel, in seconds')
-    _('-w', '--wireless', action='store_true',
-            help='Set all interfaces to be treated as wireless interfaces ( in babel )')
-    _('--verbose', '-v', default=0, type=int,
-            help='Defines the verbose level')
     _('--ca', required=True,
             help='Path to the certificate authority file')
     _('--cert', required=True,
             help='Path to the certificate file')
-    ipconfig = parser.add_mutually_exclusive_group()
-    __ = ipconfig.add_argument
-    __('--ip', default=None, dest='address', action='append', nargs=3,
-            help='Ip address, port and protocol advertised to other vpn nodes')
-    __('--internal-port', default=1194,
-            help='Internal port to listen on for incomming connections')
     # args to be removed ?
-    _('--proto', default='udp',
-            help='The protocol used by other peers to connect')
     _('--connection-count', default=20, type=int,
             help='Number of tunnels')
     _('--refresh-rate', default=0.05, type=float,
-            help='The ratio of connections to drop when refreshing the connections')
+            help='''The ratio of connections to drop when refreshing the
+                    connections''')
     # Openvpn options
     _('openvpn_args', nargs=argparse.REMAINDER,
             help="Common OpenVPN options (e.g. certificates)")
@@ -95,34 +102,40 @@ def main():
     else:
         utils.log('Attempting automatic configuration via UPnP', 4)
         try:
-            external_ip, external_port = upnpigd.ForwardViaUPnP(config.internal_port)
-            config.address = [[external_ip, external_port, 'udp'],
-                              [external_ip, external_port, 'tcp-client']]
+            ext_ip, ext_port = upnpigd.ForwardViaUPnP(config.internal_port)
+            config.address = list([ext_ip, ext_port, proto]
+                                  for proto in config.proto)
         except Exception:
             utils.log('An atempt to forward a port via UPnP failed', 4)
 
-    peer_db = db.PeerManager(config.db, config.server, config.server_port,
-            config.peers_db_refresh, config.address, internal_ip, prefix, manual, 200)
-    tunnel_manager = tunnel.TunnelManager(write_pipe, peer_db, openvpn_args, config.hello,
-            config.tunnel_refresh, config.connection_count, config.refresh_rate)
+    peer_db = db.PeerManager(config.state, config.server, config.server_port,
+            config.peers_db_refresh, config.address, internal_ip, prefix,
+            manual, config.proto, 200)
+    tunnel_manager = tunnel.TunnelManager(write_pipe, peer_db, openvpn_args,
+            config.hello, config.tunnel_refresh, config.connection_count,
+            config.refresh_rate)
 
-    # Launch babel on all interfaces. WARNING : you have to be root to start babeld
+    # Launch routing protocol. WARNING : you have to be root to start babeld
     interface_list = ['vifibnet'] + list(tunnel_manager.free_interface_set)
-    router = plib.router(network, internal_ip, interface_list, config.wireless, config.hello,
+    router = plib.router(network, internal_ip, interface_list, config.wireless,
+            config.hello, os.path.join(config.state, 'vifibnet.babeld.state'),
             stdout=os.open(os.path.join(config.log, 'vifibnet.babeld.log'),
-            os.O_WRONLY | os.O_CREAT | os.O_TRUNC), stderr=subprocess.STDOUT)
+                os.O_WRONLY|os.O_CREAT|os.O_TRUNC), stderr=subprocess.STDOUT)
 
    # Establish connections
-    server_process = plib.server(internal_ip, network, config.connection_count, config.dh, write_pipe,
-            config.internal_port, config.proto, config.hello, '--dev', 'vifibnet', *openvpn_args,
-            stdout=os.open(os.path.join(config.log, 'vifibnet.server.log'), os.O_WRONLY | os.O_CREAT | os.O_TRUNC))
+    server_process = list(plib.server(internal_ip, network,
+        config.connection_count, config.dh, write_pipe, config.internal_port,
+        proto, config.hello, '--dev', 'vifibnet', *openvpn_args,
+        stdout=os.open(os.path.join(config.log, 'vifibnet.server.log'),
+            os.O_WRONLY | os.O_CREAT | os.O_TRUNC)) for proto in config.proto)
     tunnel_manager.refresh()
 
     # main loop
     try:
         while True:
             ready, tmp1, tmp2 = select.select([read_pipe], [], [],
-                    max(0, min(tunnel_manager.next_refresh, peer_db.next_refresh) - time.time()))
+                    max(0, min(tunnel_manager.next_refresh,
+                               peer_db.next_refresh) - time.time()))
             if ready:
                 peer_db.handle_message(read_pipe.readline())
             if time.time() >= peer_db.next_refresh: