Todo update

A lot of bug fix

Removing of the bandwidth calculation

When 2 tunnels are estabilshed between the same 2 peers, one of them is deleted

TODO

+Bug :
+    Peers stay connected to the bootstrap node so none can enter
+    re6stnet adress should appear on only one interface
+    Bug in the upnp refresh
+
+Check :
+    Check that boostrap nodes work as intended
+
+
 To be done :
-    Do a 'real' test, on one ( or more ) real machine(s) preferably :
+    Choose the metric to add to bootstrap peers
+    Choose peer DB size.
+    Choose the number of peer we ask to the server
+    Compare prefixes when 2 tunnels are established between the 2 same machines
+    Warn babeld about the tunnels wich are about to be deleted. Maybe we could 
+        just increase the cost.
+
+    Do a test, on one ( or more ) real machine(s) preferably :
        - package re6stnet
        - install re6stnet
        - follow the HOW TO section of the man page to setup a network
@@ -18,20 +34,6 @@ To be done :
         one have to start the registry twice, the first time without
         the --private option
 
-    Use an algorithm to choose which connections to keep and/or establish
-    instead of pure randomness
-        number of routes / tunnel
-
-    Warn babeld about the tunnels wich are about to be deleted. Maybe we could just increase the cost.
-
     Put a section about how to build the package from the sources in the README
 
----------------------------------------------------------------------------------------------
-    We should put a high latency on the servers openvpn interfaces to avoid people staying connected to it.
-
-    Check that blacklisting for peers connected to us works
-    Check that if 2 tunnels are established between 2 peers, one of them is destroyed
-
-    Check peers ID when connectiong to the server
 
-    Choose peer DB size

docs/re6stnet.rst

@@ -266,6 +266,13 @@ you can get them in the python interpreter::
     >>> print prefix
     0000000000000011
 
+
+from re6st import utils
+network = utils.networkFromCa('clients/server/ca.pem')
+re6st_ip, prefix = utils.ipFromCert(network, 'clients/server/cert.crt')
+print re6st_ip
+print prefix
+
 Now you can restart your re6st-registry with two more options:
 
 ``re6st-registry port_number --db db_path --ca path_to_ca.crt

re6st-registry

@@ -235,7 +235,7 @@ class main(object):
     def declare(self, handler, address):
         print "declaring new node"
         client_address, address = address
-        #client_address, _ = handler.client_address
+        client_address, _ = handler.client_address
         client_ip = utils.binFromIp(client_address)
         if client_ip.startswith(self.network):
             prefix = client_ip[len(self.network):]

re6st/db.py

@@ -16,6 +16,7 @@ class PeerManager:
         self._key_path = key_path
         self._pp = pp
         self._manual = manual
+        self.tunnel_manager = None
 
         logging.info('Connecting to peers database...')
         self._db = sqlite3.connect(db_path, isolation_level=None)
@@ -58,7 +59,7 @@ class PeerManager:
         logging.info('Blacklist cleared')
 
     def blacklist(self, prefix, flag):
-        logging.ninfo('Blacklisting %s' % prefix)
+        logging.info('Blacklisting %s' % prefix)
         self._db.execute("DELETE FROM peers WHERE prefix = ?", (prefix,))
         self._db.execute("INSERT OR REPLACE INTO blacklist.flag VALUES (?,?)",
                           (prefix, flag))
@@ -160,10 +161,12 @@ class PeerManager:
     def handle_message(self, msg):
         script_type, arg = msg.split()
         if script_type == 'client-connect':
-            self.blacklist(arg, 2)
             logging.info('Incomming connection from %s' % (arg,))
+            prefix = utils.binFromSubnet(arg)
+            self.tunnel_manager.checkIncommingTunnel(prefix)
+            self.blacklist(prefix, 2)
         elif script_type == 'client-disconnect':
-            self.whitelist(arg)
+            self.whitelist(utils.binFromSubnet(arg))
             logging.info('%s has disconnected' % (arg,))
         elif script_type == 'route-up':
             if not self._manual:
@@ -174,7 +177,12 @@ class PeerManager:
                     self._address = new_address
                     logging.info('Received new external ip : %s'
                               % (external_ip,))
+                    try:
                         self._declare()
+                    except socket.error, e:
+                        logging.debug('socket.error : %s' % e)
+                        logging.info('''Connection to server failed while
+                            declaring external infos''')
         else:
             logging.debug('Unknow message recieved from the openvpn pipe : %s'
                     % msg)

re6st/plib.py

-import os, subprocess, logging
+import os
+import subprocess
+import logging
 import utils
 
 verbose = 0
@@ -6,6 +8,7 @@ here = os.path.realpath(os.path.dirname(__file__))
 ovpn_server = os.path.join(here, 'ovpn-server')
 ovpn_client = os.path.join(here, 'ovpn-client')
 
+
 def openvpn(hello_interval, *args, **kw):
     args = ['openvpn',
         '--dev-type', 'tap',
@@ -43,6 +46,8 @@ def client(server_address, pipe_fd, hello_interval, *args, **kw):
               '--route-up', ovpn_client + ' ' + str(pipe_fd)]
     try:
         for ip, port, proto in utils.address_list(server_address):
+            if proto == 'tcp-server':
+                proto = 'tcp-client'
             remote += '--remote', ip, port, proto
     except ValueError, e:
         logging.warning('Error "%s" in unpacking address %s for openvpn client'
@@ -51,7 +56,7 @@ def client(server_address, pipe_fd, hello_interval, *args, **kw):
     return openvpn(hello_interval, *remote, **kw)
 
 
-def router(network, internal_ip, interface_list,
+def router(network, internal_ip, interface_list, isBootstrap,
            wireless, hello_interval, state_path, **kw):
     logging.info('Starting babel...')
     args = ['babeld',
@@ -71,6 +76,8 @@ def router(network, internal_ip, interface_list,
             '-S', state_path,
             '-s',
             ]
+    #if isBootstrap:
+    #    args.extend(['-C', 'redistribute ip %s::/%u metric 16000' % (utils.ipFromBin(network), len(network))])
     if wireless:
         args.append('-w')
     args = args + interface_list

re6st/tunnel.py

 import os, traceback, time, subprocess, math, logging
 import plib
 
-smooth = 0.3     # this is used to smooth the traffic sampling. Lower value
-                 # mean more smooth
-protected = 0.2  # ratio of the tunnels protected against kill because they are
-                 # used a lot
-
 # Be carfull the refresh interval should let the routes be established
 
+log = None
+
 
 class Connection:
 
@@ -22,8 +19,6 @@ class Connection:
         self.iface = iface
         self.routes = 0
         self._prefix = prefix
-        self.bandwidth = None
-        self._last_trafic = None
 
     def refresh(self):
         # Check that the connection is alive
@@ -31,47 +26,13 @@ class Connection:
             logging.info('Connection with %s has failed with return code %s'
                      % (self._prefix, self.process.returncode))
             return False
-
-        # self._updateBandwidth()
         return True
 
-    # Unused for now. By killing tunnels with significantly lower trafic
-    # in comparison to other tunnels, we hope to connect to nodes with
-    # better bandwith, in order to improve connectivity with destinations
-    # we are really interested in.
-    def _updateBandwidth(self):
-        try:
-            f_rx = open('/sys/class/net/%s/statistics/rx_bytes' %
-                    self.iface, 'r')
-            f_tx = open('/sys/class/net/%s/statistics/tx_bytes' %
-                    self.iface, 'r')
-
-            trafic = int(f_rx.read()) + int(f_tx.read())
-            t = time.time()
-
-            if bool(self._last_trafic):
-                bw = (trafic - self._last_trafic) / (t -
-                        self._last_trafic_update)
-                if bool(self.bandwidth):
-                    self.bandwidth = ((1 - smooth) * self.bandwidth
-                            + smooth * bw)
-                else:
-                    self.bandwidth = bw
-
-                logging.debug('New bandwidth calculated on iface %s : %s' %
-                        (self.iface, self.bandwidth))
-
-            self._last_trafic_update = t
-            self._last_trafic = trafic
-        except IOError:  # This just means that the interface is down
-            logging.debug('Unable to calculate bandwidth on iface %s' %
-                self.iface)
-
 
 class TunnelManager:
 
     def __init__(self, write_pipe, peer_db, openvpn_args, hello_interval,
-                refresh, connection_count, iface_list, network):
+                refresh, connection_count, iface_list, network, prefix):
         self._write_pipe = write_pipe
         self._peer_db = peer_db
         self._connection_dict = {}
@@ -82,6 +43,7 @@ class TunnelManager:
         self._network = network
         self._net_len = len(network)
         self._iface_list = iface_list
+        self._prefix = prefix
 
         self.next_refresh = time.time()
         self._next_tunnel_refresh = time.time()
@@ -111,9 +73,6 @@ class TunnelManager:
     def _removeSomeTunnels(self):
         # Get the candidates to killing
         candidates = sorted(self._connection_dict, key=lambda p:
-                self._connection_dict[p].bandwidth)
-        candidates = sorted(candidates[0: int(math.ceil((1 - protected)
-                * len(candidates)))], key=lambda p:
                 self._connection_dict[p].routes)
         for prefix in candidates[0: max(0, len(self._connection_dict) -
                 self._client_count + self._refresh_count)]:
@@ -175,8 +134,8 @@ class TunnelManager:
                     self._connection_dict[self._iface_to_prefix[iface]].routes += 1
                 if iface in self._iface_list and self._net_len < subnet_size < 128:
                     prefix = ip[self._net_len:subnet_size]
-                    logging.debug('A route to %s (%s) has been discovered on the LAN'
-                            % (hex(int(prefix), 2)[2:], prefix))
+                    logging.debug('A route to %s has been discovered on the LAN'
+                            % (hex(int(prefix), 2)[2:]))
                     self._peer_db.blacklist(prefix, 0)
 
         logging.debug("Routes have been counted")
@@ -188,3 +147,7 @@ class TunnelManager:
     def killAll(self):
         for prefix in self._connection_dict.keys():
             self._kill(prefix)
+
+    def checkIncommingTunnel(self, prefix):
+        if prefix in self._connection_dict and prefix >= self._prefix:
+            self._kill(prefix)

re6st/utils.py

@@ -3,6 +3,7 @@ from OpenSSL import crypto
 
 logging_levels = logging.WARNING, logging.INFO, logging.DEBUG, 5
 
+
 def setupLog(log_level):
     logging.basicConfig(level=logging_levels[log_level],
             format='%(asctime)s : %(message)s',
@@ -10,10 +11,12 @@ def setupLog(log_level):
     logging.addLevelName(5, 'TRACE')
     logging.trace = lambda *args, **kw: logging.log(5, *args, **kw)
 
+
 def binFromIp(ip):
     ip1, ip2 = struct.unpack('>QQ', socket.inet_pton(socket.AF_INET6, ip))
     return bin(ip1)[2:].rjust(64, '0') + bin(ip2)[2:].rjust(64, '0')
 
+
 def ipFromBin(prefix):
     prefix = hex(int(prefix, 2))[2:]
     ip = ''
@@ -21,17 +24,20 @@ def ipFromBin(prefix):
         ip += prefix[i:i + 4] + ':'
     return ip.rstrip(':')
 
+
 def ipFromPrefix(re6stnet, prefix, prefix_len):
     prefix = bin(int(prefix))[2:].rjust(prefix_len, '0')
     ip_t = (re6stnet + prefix).ljust(127, '0').ljust(128, '1')
     return ipFromBin(ip_t), prefix
 
+
 def networkFromCa(ca_path):
     # Get network prefix from ca.crt
     with open(ca_path, 'r') as f:
         ca = crypto.load_certificate(crypto.FILETYPE_PEM, f.read())
         return bin(ca.get_serial_number())[3:]
 
+
 def ipFromCert(network, cert_path):
     # Get ip from cert.crt
     with open(cert_path, 'r') as f:
@@ -40,9 +46,18 @@ def ipFromCert(network, cert_path):
         prefix, prefix_len = subject.CN.split('/')
         return ipFromPrefix(network, prefix, int(prefix_len))
 
+
 def address_str(address):
     return ';'.join(map(','.join, address))
 
+
 def address_list(address_list):
     return list(tuple(address.split(','))
         for address in address_list.split(';'))
+
+
+def binFromSubnet(subnet):
+    prefix, subnet_size = subnet.split('/')
+    binary = bin(int(prefix))[2:]
+    binary = ('0' * (int(subnet_size) - len(binary))) + binary
+    return binary

re6stnet

@@ -59,6 +59,9 @@ def getConfig():
     _('-w', '--wireless', action='store_true',
             help='''Set all interfaces to be treated as wireless interfaces
                     for the routing protocol''')
+    _('--isbootstrap', default=False, action='store_true',
+            help="""Notify that the peer is a bootstrap peer and that other
+            peers shouldn't stay connected to it for too long""")
 
     # Tunnel options
     _('--pp', nargs=2, action='append',
@@ -139,13 +142,14 @@ def main():
             manual, config.pp, 200)
     tunnel_manager = tunnel.TunnelManager(write_pipe, peer_db, openvpn_args,
             config.hello, config.tunnel_refresh, config.connection_count,
-            config.iface_list, network)
+            config.iface_list, network, prefix)
+    peer_db.tunnel_manager = tunnel_manager
 
     # Launch routing protocol. WARNING : you have to be root to start babeld
     interface_list = list(tunnel_manager.free_interface_set) \
                      + config.iface_list + list(iface
                              for _, _, iface in config.pp)
-    router = plib.router(network, internal_ip, interface_list, config.wireless,
+    router = plib.router(network, internal_ip, interface_list, config.isbootstrap, config.wireless,
             config.hello, os.path.join(config.state, 'babeld.state'),
             stdout=os.open(os.path.join(config.log, 'babeld.log'),
             os.O_WRONLY | os.O_CREAT | os.O_TRUNC), stderr=subprocess.STDOUT)
@@ -159,7 +163,6 @@ def main():
             os.O_WRONLY | os.O_CREAT | os.O_TRUNC),
         stderr=subprocess.STDOUT)
         for port, proto, iface in config.pp)
-    tunnel_manager.refresh()
 
     # main loop
     try: