Add communities to re6st

See https://www.erp5.com/P-VIFIB-Re6stnet.Roadmap and https://www.erp5.com/P-VIFIB-Re6stnet.Communities

See merge request nexedi/re6stnet!36

demo/demo

@@ -277,7 +277,7 @@ def new_network(registry, reg_addr, serial, ca):
 
 with new_network(registry, REGISTRY, REGISTRY_SERIAL, 'ca.crt') as new_node:
     new_node(machine1, 'm1', '-I%s' % m1_if_0.name)
-    new_node(machine2, 'm2', '--remote-gateway 10.1.1.1', prefix_len=80)
+    new_node(machine2, 'm2', '--remote-gateway 10.1.1.1', prefix_len=77)
     new_node(machine3, 'm3', '-i%s' % m3_if_0.name)
     new_node(machine4, 'm4', '-i%s' % m4_if_0.name)
     new_node(machine5, 'm5', '-i%s' % m5_if_0.name)

demo/registry/community.conf

+# Community example config
+000 *
+001 @AS AU
+010 FR DE IT

demo/registry/re6st-registry.conf

@@ -7,3 +7,5 @@ hello 4
 client-count 2
 tunnel-refresh 100
 ipv4 10.42.0.0/16 8
+prefix-length 13
+community registry/community.conf

docs/re6stnet.rst

@@ -167,6 +167,12 @@ For bootstrapping, you may have to explicitly set an IP in the configuration
 of the first node, via the ``--ip`` option. Otherwise, additional nodes won't
 be able to connect to it.
 
+You can use communities to group prefixes in different subprefixes based on
+their location. Each line in the community configuration is a mapping
+from a subprefix (in binary) to a list of locations. Each location is either
+"*" (default assignment), a country (Alpha-2 code), or a continent (Alpha-2
+code) preceded by "@". See demo/registry/community.conf for an example.
+
 TROUBLESHOOTING
 ===============
 

re6st/cli/conf.py

@@ -45,6 +45,10 @@ def main():
     _('--anonymous', action='store_true',
         help="Request an anonymous certificate. No email is required but the"
              " registry may deliver a longer prefix.")
+    _('--location',
+            help="Alpha-2 codes of country and continent separated by a comma."
+                 " Will be used for the community assignment (default: location"
+                 " is automatically detected). Example: FR,EU")
     config = parser.parse_args()
     if config.dir:
         os.chdir(config.dir)
@@ -141,7 +145,10 @@ def main():
         # to avoid using our token for nothing.
         cert_fd = os.open(cert_path, os.O_CREAT | os.O_WRONLY, 0666)
         print "Requesting certificate ..."
-        cert = s.requestCertificate(token, req)
+        if config.location:
+            cert = s.requestCertificate(token, req, location=config.location)
+        else:
+            cert = s.requestCertificate(token, req)
         if not cert:
             token_advice = None
             sys.exit("Error: invalid or expired token")

re6st/cli/registry.py

@@ -110,6 +110,9 @@ def main():
         help="Reject nodes that are too old. Current is %s." % version.protocol)
     _('--authorized-origin', action='append', default=['127.0.0.1', '::1'],
         help="Authorized IPs to access origin-restricted RPC.")
+    _('--community',
+        help="File containing community configuration. This file cannot be"
+             " empty and must contain the default location ('*').")
 
     _ = parser.add_argument_group('routing').add_argument
     _('--hello', type=int, default=15,

re6st/registry.py

@@ -68,6 +68,20 @@ class RegistryServer(object):
         self.sessions = {}
         self.sock = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM)
 
+        # Parse community file
+        self.community_map = {}
+	if config.community:
+            with open(config.community) as x:
+                for x in x:
+                    x = x.strip()
+                    if x and not x.startswith('#'):
+                        x = x.split()
+                        self.community_map[x.pop(0)] = x
+            if sum('*' in x for x in self.community_map.itervalues()) != 1:
+                sys.exit("Invalid community configuration: missing or multiple default location ('*')")
+        else:
+            self.community_map[''] = '*'
+
         # Database initializing
         db_dir = os.path.dirname(self.config.db)
         db_dir and utils.makedirs(db_dir)
@@ -88,7 +102,23 @@ class RegistryServer(object):
                 "prefix TEXT PRIMARY KEY NOT NULL",
                 "email TEXT",
                 "cert TEXT")
-        self.db.execute("INSERT OR IGNORE INTO cert VALUES ('',null,null)")
+        if not self.db.execute("SELECT 1 FROM cert LIMIT 1").fetchone():
+            self.db.execute("INSERT INTO cert VALUES ('',null,null)")
+
+        prev = '-'
+        for community in sorted(self.community_map):
+            if community.startswith(prev):
+                err = "communities %s and %s overlap" % (prev, community)
+            else:
+                x = self.db.execute("SELECT prefix, cert FROM cert"
+                                    " WHERE substr(?,1,length(prefix)) = prefix",
+                                    (community,)).fetchone()
+                if not x or x[1] is None:
+                    prev = community
+                    continue
+                err = "prefix %s contains community %s" % (x[0], community)
+            sys.exit("Invalid community configuration: " + err)
+
         utils.sqliteCreateTable(self.db, "crl",
                 "serial INTEGER PRIMARY KEY NOT NULL",
                 # Expiration date of revoked certificate.
@@ -106,15 +136,16 @@ class RegistryServer(object):
         self.ctl = ctl.Babel(os.path.join(config.run, 'babeld.sock'),
             weakref.proxy(self), self.network)
 
-        db = os.getenv('GEOIP2_MMDB')
-        if db:
+        self.geoip_db = os.getenv('GEOIP2_MMDB')
+        if self.geoip_db:
             from geoip2 import database, errors
-            country = database.Reader(db).country
+            country = database.Reader(self.geoip_db).country
             def geoiplookup(ip):
                 try:
-                    return country(ip).country.iso_code.encode()
-                except errors.AddressNotFoundError:
-                    return
+                    req = country(ip)
+                    return req.country.iso_code.encode(), req.continent.code.encode()
+                except (errors.AddressNotFoundError, ValueError):
+                    return '*', '*'
             self._geoiplookup = geoiplookup
         elif self.config.same_country:
             sys.exit("Can not respect 'same_country' network configuration"
@@ -246,8 +277,7 @@ class RegistryServer(object):
                       (prefix,))
                 elif not_after is None or x < not_after:
                     not_after = x
-            # TODO: reduce 'cert' table by merging free slots
-            #       (IOW, do the contrary of newPrefix)
+            self.mergePrefixes()
             self.timeout = not_after and not_after + GRACE_PERIOD
 
     def handle_request(self, request, method, kw):
@@ -279,6 +309,9 @@ class RegistryServer(object):
                 request.headers.get("X-Forwarded-For") or
                 request.headers.get("host"),
                 request.headers.get("user-agent"))
+        if not kw.get('ip', True):
+            kw['ip'] = (request.headers.get("X-Forwarded-For", "").split(',',1)[0].strip() or
+                        request.headers.get("host"))
         try:
             result = m(**kw)
         except HTTPError, e:
@@ -385,26 +418,80 @@ class RegistryServer(object):
             s.sendmail(self.email, email, msg.as_string())
             s.quit()
 
-    def newPrefix(self, prefix_len):
+    def getCommunity(self, country, continent):
+        for prefix, location_list in self.community_map.iteritems():
+            if country in location_list:
+                return prefix
+        default = ''
+        for prefix, location_list in self.community_map.iteritems():
+            if continent in location_list:
+                return prefix
+            if '*' in location_list:
+                default = prefix
+        return default
+
+    def mergePrefixes(self):
+        q = self.db.execute
+        prev_prefix = None
+        max_len = 128,
+        while True:
+            max_len = q("SELECT max(length(prefix)) FROM cert"
+                        " WHERE cert is null AND length(prefix) < ?",
+                        max_len).next()
+            if not max_len[0]:
+                break
+            for prefix, in q("SELECT prefix FROM cert"
+                             " WHERE cert is null AND length(prefix) = ?"
+                             " ORDER BY prefix",
+                             max_len):
+                if prev_prefix and prefix[:-1] == prev_prefix[:-1]:
+                    q("UPDATE cert SET prefix = ? WHERE prefix = ?",
+                      (prefix[:-1], prev_prefix))
+                    q("DELETE FROM cert WHERE prefix = ?", (prefix,))
+                    prev_prefix = None
+                else:
+                    prev_prefix = prefix
+
+    def newPrefix(self, prefix_len, community):
+        community_len = len(community)
+        prefix_len += community_len
         max_len = 128 - len(self.network)
         assert 0 < prefix_len <= max_len
-        try:
-            prefix, = self.db.execute("""SELECT prefix FROM cert WHERE length(prefix) <= ? AND cert is null
-                                         ORDER BY length(prefix) DESC""", (prefix_len,)).next()
-        except StopIteration:
-            logging.error('No more free /%u prefix available', prefix_len)
-            raise
-        while len(prefix) < prefix_len:
-            self.db.execute("UPDATE cert SET prefix = ? WHERE prefix = ?", (prefix + '1', prefix))
-            prefix += '0'
-            self.db.execute("INSERT INTO cert VALUES (?,null,null)", (prefix,))
-        if len(prefix) < max_len or '1' in prefix:
-            return prefix
-        self.db.execute("UPDATE cert SET cert = 'reserved' WHERE prefix = ?", (prefix,))
-        return self.newPrefix(prefix_len)
+        q = self.db.execute
+        while True:
+            try:
+                # Find longest free prefix whithin community.
+                prefix, = q(
+                    "SELECT prefix FROM cert"
+                    " WHERE prefix LIKE ?"
+                    "   AND length(prefix) <= ? AND cert is null"
+                    " ORDER BY length(prefix) DESC",
+                    (community + '%', prefix_len)).next()
+            except StopIteration:
+                # Community not yet allocated?
+                # There should be exactly 1 row whose
+                # prefix is the beginning of community.
+                prefix, x = q("SELECT prefix, cert FROM cert"
+                              " WHERE substr(?,1,length(prefix)) = prefix",
+                              (community,)).next()
+                if x is not None:
+                    logging.error('No more free /%u prefix available',
+                                  prefix_len)
+                    raise
+            # Split the tree until prefix has wanted length.
+            for x in xrange(len(prefix), prefix_len):
+                # Prefix starts with community, then we complete with 0.
+                x = community[x] if x < community_len else '0'
+                q("UPDATE cert SET prefix = ? WHERE prefix = ?",
+                  (prefix + str(1-int(x)), prefix))
+                prefix += x
+                q("INSERT INTO cert VALUES (?,null,null)", (prefix,))
+            if len(prefix) < max_len or '1' in prefix[community_len:]:
+                return prefix
+            q("UPDATE cert SET cert = 'reserved' WHERE prefix = ?", (prefix,))
 
     @rpc
-    def requestCertificate(self, token, req):
+    def requestCertificate(self, token, req, location='', ip=''):
         req = crypto.load_certificate_request(crypto.FILETYPE_PEM, req)
         with self.lock:
             with self.db:
@@ -424,7 +511,12 @@ class RegistryServer(object):
                     if not prefix_len:
                         raise HTTPError(httplib.FORBIDDEN)
                     email = None
-                prefix = self.newPrefix(prefix_len)
+                country, continent = '*', '*'
+                if self.geoip_db:
+                    country, continent = location.split(',') if location else self._geoiplookup(ip)
+                    if continent != '*':
+                        continent = '@' + continent
+                prefix = self.newPrefix(prefix_len, self.getCommunity(country, continent))
                 self.db.execute("UPDATE cert SET email = ? WHERE prefix = ?",
                                 (email, prefix))
                 if self.prefix is None:
@@ -523,7 +615,8 @@ class RegistryServer(object):
 
     @rpc
     def getCountry(self, cn, address):
-        return self._geoiplookup(address)
+        country = self._geoiplookup(address)[0]
+        return None if country == '*' else country
 
     @rpc
     def getBootstrapPeer(self, cn):