From 3b9c0e5d49cbf094356b795b85784e2e1aefbf54 Mon Sep 17 00:00:00 2001
From: Rafael Monnerat <rafael@nexedi.com>
Date: Fri, 27 Jan 2017 19:01:13 +0100
Subject: [PATCH] re6st-registry: Refactor integration with re6st registry

  * Follow up methods changes on re6stnet registry API.
  * getIPv4Information was simplifed on registry, so reimplement it.
  * Remove scripts generated and keep a single script to orchestate the token management.
  * Drop some code duplication
  * Remove babeld, openvpn and miniupnp as they are not required.
  * Remove useless scripts.
  * Remove re6stnet section, extra-eggs is more them enough and prevent duplications.
---
 slapos/recipe/re6stnet/__init__.py         |  18 --
 slapos/recipe/re6stnet/re6stnet.py         | 302 ++++++---------------
 software/re6stnet/instance-re6stnet.cfg.in |  25 +-
 software/re6stnet/software.cfg             |   6 +-
 4 files changed, 92 insertions(+), 259 deletions(-)

diff --git a/slapos/recipe/re6stnet/__init__.py b/slapos/recipe/re6stnet/__init__.py
index 5a8c5dda4..3516e52c5 100644
--- a/slapos/recipe/re6stnet/__init__.py
+++ b/slapos/recipe/re6stnet/__init__.py
@@ -218,24 +218,6 @@ class Recipe(GenericBaseRecipe):
       )
     path_list.append(request_add)
 
-    request_drop = self.createPythonScript(
-        self.options['drop-service-wrapper'].strip(),
-        '%s.re6stnet.requestRemoveToken' % __name__, service_dict
-      )
-    path_list.append(request_drop)
-
-    request_check = self.createPythonScript(
-        self.options['check-service-wrapper'].strip(),
-        '%s.re6stnet.checkService' % __name__, service_dict
-      )
-    path_list.append(request_check)
-
-    revoke_check = self.createPythonScript(
-        self.options['revoke-service-wrapper'].strip(),
-        '%s.re6stnet.requestRevoqueCertificate' % __name__, service_dict
-      )
-    path_list.append(revoke_check)
-
     # Send connection parameters of slave instances
     if token_dict:
       self.slap.initializeConnection(self.server_url, self.key_file,
diff --git a/slapos/recipe/re6stnet/re6stnet.py b/slapos/recipe/re6stnet/re6stnet.py
index 3990ec3cd..c71c109b5 100644
--- a/slapos/recipe/re6stnet/re6stnet.py
+++ b/slapos/recipe/re6stnet/re6stnet.py
@@ -1,42 +1,19 @@
 # -*- coding: utf-8 -*-
+import httplib
 import logging
 import json
 import os
 import time
-import sqlite3
 import slapos
 import traceback
 import logging
-import socket
-import select
-from re6st import tunnel, ctl, registry, utils, x509
-from OpenSSL import crypto
+from re6st import  registry
 
 
 log = logging.getLogger('SLAPOS-RE6STNET')
 logging.basicConfig(level=logging.INFO)
 
-logging.trace = logging.debug 
-
-class iterRoutes(object):
-
-    _waiting = True
-
-    def __new__(cls, control_socket, network):
-        self = object.__new__(cls)
-        c = ctl.Babel(control_socket, self, network)
-        c.request_dump()
-        while self._waiting:
-            args = {}, {}, ()
-            c.select(*args)
-            utils.select(*args)
-        return (prefix
-            for neigh_routes in c.neighbours.itervalues()
-            for prefix in neigh_routes[1]
-            if prefix)
-
-    def babel_dump(self):
-        self._waiting = False
+logging.trace = logging.debug
 
 def loadJsonFile(path):
   if os.path.exists(path):
@@ -57,12 +34,11 @@ def readFile(path):
     return content
   return ''
 
-def getDb(db_path):
-  db = sqlite3.connect(db_path, isolation_level=None,
-                                                  check_same_thread=False)
-  db.text_factory = str
-
-  return db.cursor()
+def updateFile(file_path, value):
+  if readFile(file_path) != value:
+    writeFile(file_path, value)
+    return True
+  return False
 
 def bang(args):
   computer_guid = args['computer_id']
@@ -77,64 +53,62 @@ def bang(args):
   partition.bang(message='Published parameters changed!')
   log.info("Bang with message 'parameters changed'...")
 
-
-def requestAddToken(args, can_bang=True):
-
+def requestAddToken(client, base_token_path):
   time.sleep(3)
-  registry_url = args['registry_url']
-  base_token_path = args['token_base_path']
   path_list = [x for x in os.listdir(base_token_path) if x.endswith('.add')]
 
+  updated = False
+  log.info("Searching tokens to add at %s and found %s." % (base_token_path, path_list))
+
   if not path_list:
     log.info("No new token to add. Exiting...")
     return
 
-  client = registry.RegistryClient(registry_url)
-  call_bang = False
-
   for reference_key in path_list:
     request_file = os.path.join(base_token_path, reference_key)
     token = readFile(request_file)
+    log.info("Including token %s for %s" % (token, reference_key))
     if token :
       reference = reference_key.split('.')[0]
       # email is unique as reference is also unique
       email = '%s@slapos' % reference.lower()
       try:
-        result = client.requestAddToken(token, email)
+        result = client.addToken(email, token)
       except Exception:
-        log.debug('Request add token fail for %s... \n %s' % (request_file,
+        log.info('Request add token fail for %s... \n %s' % (request_file,
                     traceback.format_exc()))
         continue
+
       if result and result == token:
         # update information
         log.info("New token added for slave instance %s. Updating file status..." %
                             reference)
-        writeFile(os.path.join(base_token_path, '%s.status' % reference),
-                    'TOKEN_ADDED')
+        status_file = os.path.join(base_token_path, '%s.status' % reference)
+        updateFile(status_file, 'TOKEN_ADDED')
         os.unlink(request_file)
-        call_bang = True
+        updated = True
     else:
       log.debug('Bad token. Request add token fail for %s...' % request_file)
 
-  if can_bang and call_bang:
-    bang(args)
+  return updated
 
-def requestRemoveToken(args):
-  base_token_path = args['token_base_path']
+def requestRemoveToken(client, base_token_path):
   path_list = [x for x in os.listdir(base_token_path) if x.endswith('.remove')]
 
   if not path_list:
     log.info("No token to delete. Exiting...")
     return
 
-  client = registry.RegistryClient(args['registry_url'])
   for reference_key in path_list:
     request_file = os.path.join(base_token_path, reference_key)
     token = readFile(request_file)
     if token :
       reference = reference_key.split('.')[0]
       try:
-        result = client.requestDeleteToken(token)
+        result = client.deleteToken(token)
+      except httplib.NOTFOUND:
+        # Token is alread removed.
+        result = True
       except Exception:
         log.debug('Request delete token fail for %s... \n %s' % (request_file,
                     traceback.format_exc()))
@@ -142,10 +116,12 @@ def requestRemoveToken(args):
       else:
         # certificate is invalidated, it will be revoked
         writeFile(os.path.join(base_token_path, '%s.revoke' % reference), '')
-      if result == 'True':
+
+      if result in (True, 'True'):
         # update information
         log.info("Token deleted for slave instance %s. Clean up file status..." %
                             reference)
+
       if result in ['True', 'False']:
         os.unlink(request_file)
         status_file = os.path.join(base_token_path, '%s.status' % reference)
@@ -159,198 +135,96 @@ def requestRemoveToken(args):
       log.debug('Bad token. Request add token fail for %s...' % request_file)
 
 def requestRevoqueCertificate(args):
-
   base_token_path = args['token_base_path']
-  db = getDb(args['db'])
   path_list = [x for x in os.listdir(base_token_path) if x.endswith('.revoke')]
-  client = registry.RegistryClient(args['registry_url'])
 
   for reference_key in path_list:
     reference = reference_key.split('.')[0]
-    # XXX - email is always unique
-    email = '%s@slapos' % reference.lower()
-    cert_string = ''
-    try:
-      cert_string, = db.execute("SELECT cert FROM cert WHERE email = ?",
-          (email,)).next()
-    except StopIteration:
-      # Certificate was not generated yet !!!
-      pass
-
-    try:
-      if cert_string:
-        cert = crypto.load_certificate(crypto.FILETYPE_PEM, cert_string)
-        cn = x509.subnetFromCert(cert)
-        result = client.revoke(str(cn))
-        time.sleep(2)
-    except Exception:
-      log.debug('Request revoke certificate fail for %s... \n %s' % (reference,
-                  traceback.format_exc()))
-      continue
-    else:
+
+    if revokeByMail(args['registry_url'],
+                   '%s@slapos' % reference.lower(),
+                   args['db']):
       os.unlink(os.path.join(base_token_path, reference_key))
       log.info("Certificate revoked for slave instance %s." % reference)
+      return
 
+    log.info("Failed to revoke email for %s" % reference)
 
-def dumpIPv6Network(slave_reference, db, network, ipv6_file):
-  email = '%s@slapos' % slave_reference.lower()
-
-  try:
-    cert_string, = db.execute("SELECT cert FROM cert WHERE email = ?",
-        (email,)).next()
-  except StopIteration:
-    # Certificate was not generated yet !!!
-    pass
-
-  try:
-    if cert_string:
-      cert = crypto.load_certificate(crypto.FILETYPE_PEM, cert_string)
-      cn = x509.subnetFromCert(cert)
-      subnet = network + utils.binFromSubnet(cn)
-      ipv6 = utils.ipFromBin(subnet)
-      changed = readFile(ipv6_file) != ipv6
-      writeFile(ipv6_file, ipv6)
-      return ipv6, utils.binFromSubnet(cn), changed
-  except Exception:
-    log.debug('XXX for %s... \n %s' % (slave_reference,
-              traceback.format_exc()))
-
-def sendto(sock, prefix, code):
-  return sock.sendto("%s\0%c" % (prefix, code), ('::1', tunnel.PORT))      
-
-def recv(sock, code):
-  try:
-    prefix, msg = sock.recv(1<<16).split('\0', 1)
-    int(prefix, 2)
-  except ValueError:
-    pass
-  else:
-    if msg and ord(msg[0]) == code:
-      return prefix, msg[1:]
-  return None, None
-
-def dumpIPv4Network(ipv6_prefix, network, ipv4_file, sock, peer_prefix_list):
-  try:
-
-    if ipv6_prefix == "00000000000000000000000000000000":
-      # workarround to ignore the first node
-      ipv4 = "0.0.0.0"
-      changed = readFile(ipv4_file) != ipv4
-      writeFile(ipv4_file, ipv4)
-      return ipv4, changed
-
-    peers = []
-
-    peer_list = [prefix for prefix in peer_prefix_list if prefix == ipv6_prefix ]
-
-    if len(peer_list) == 0:
-      raise ValueError("Unable to find such prefix on database")
-
-    peer = peer_list[0]
-
-    sendto(sock, peer, 1)
-    s = sock,
-    timeout = 15 
-    end = timeout + time.time()
-
-    while select.select(s, (), (), timeout)[0]:
-      prefix, msg = recv(sock, 1)
-      if prefix == peer:
-        break
-
-      timeout = max(0, end - time.time())
-    else:
-     logging.info("Timeout while querying address for %s/%s", int(peer, 2), len(peer))
-     msg = ""
-
-    if "," in msg:
-      ipv4 = msg.split(',')[0]
-    else:
-      ipv4 = "0.0.0.0"
-    changed = readFile(ipv4_file) != ipv4
-    writeFile(ipv4_file, ipv4)
-    return ipv4, changed
-  except Exception:
-    log.info('XXX for %s... \n %s' % (ipv6_prefix,
-              traceback.format_exc()))
-    return "0.0.0.0", False
-
-def checkService(args, can_bang=True):
-  base_token_path = args['token_base_path']
-  token_dict = loadJsonFile(args['token_json'])
-
+def checkService(client, base_token_path, token_json):
+  token_dict = loadJsonFile(token_json)
+  updated = False
   if not token_dict:
     return
 
-  db = getDb(args['db'])
-  call_bang = False
-
-  computer_guid = args['computer_id']
-  partition_id = args['partition_id']
-  slap = slapos.slap.slap()
-  client = registry.RegistryClient(args['registry_url'])
-  ca = client.getCa()
-  network = x509.networkFromCa(crypto.load_certificate(crypto.FILETYPE_PEM, ca))
-
-  sock = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM)
-
-  peer_prefix_list = [prefix for prefix in
-    iterRoutes("/var/run/re6stnet/babeld.sock", network)]
-
-
   # Check token status
   for slave_reference, token in token_dict.iteritems():
+    log.info("%s %s" % (slave_reference, token))
     status_file = os.path.join(base_token_path, '%s.status' % slave_reference)
-    ipv6_file = os.path.join(base_token_path, '%s.ipv6' % slave_reference)
-    ipv4_file = os.path.join(base_token_path, '%s.ipv4' % slave_reference)
     if not os.path.exists(status_file):
       # This token is not added yet!
       log.info("Token %s dont exist yet." % status_file)
       continue
 
+    if not client.isToken(str(token)):
+      # Token is used to register client
+      updated = True
+      updateFile(status_file, 'TOKEN_USED')
+      log.info("Token status of %s updated to 'used'." % slave_reference)
+
     msg = readFile(status_file)
     log.info("Token %s has %s State." % (status_file, msg))
-    if msg == 'TOKEN_USED':
-      log.info("Dumping ipv6...")
-      ipv6, ipv6_prefix, ipv6_changed = dumpIPv6Network(slave_reference, db, network, ipv6_file)
-      log.info("%s, IPV6 = %s, IPV6_PREFIX = %s" % (slave_reference, ipv6, ipv6_prefix))
-      _, ipv4_changed = dumpIPv4Network(ipv6_prefix, network, ipv4_file, sock, peer_prefix_list)
-      if ipv4_changed or ipv6_changed:
-        call_bang = True
-      continue
 
-    # Check if token is not in the database
-    status = False
-    try:
-        token_found, = db.execute("SELECT token FROM token WHERE token = ?",
-            (token,)).next()
-        if token_found == token:
-          status = True
-    except StopIteration:
-        pass
-    if not status:
-      # Token is used to register client
-      call_bang = True
+    if msg == 'TOKEN_USED':
       try:
-        writeFile(status_file, 'TOKEN_USED')
-        dumpIPv6Network(slave_reference, db, network, ipv6_file)
-        dumpIPv4Network(ipv6_prefix, network, ipv4_file, sock, peer_prefix_list)
-        log.info("Token status of %s updated to 'used'." % slave_reference)
+        log.info("Dumping ipv6...")
+        email = '%s@slapos' % slave_reference.lower()
+        try:
+          ipv6 = client.getIPv6Address(str(email))
+          ipv6_file = os.path.join(base_token_path, '%s.ipv6' % slave_reference)
+          ipv6_changed = updateFile(ipv6_file, ipv6)
+        except Exception:
+          log.info('Error for dump ipv6 for %s... \n %s' % (slave_reference,
+                                          traceback.format_exc()))
+          continue
+
+        log.info("%s, IPV6 = %s" % (slave_reference, ipv6))
+        log.info("Dumping ipv4...")
+        try:
+          ipv4 = client.getIPv4Information(str(email)) or "0.0.0.0"
+          ipv4_file = os.path.join(base_token_path, '%s.ipv4' % slave_reference)
+          ipv4_changed = updateFile(ipv4_file, ipv4)
+        except Exception:
+          log.info('Error for dump ipv4 for %s... \n %s' % (slave_reference,
+                                          traceback.format_exc()))
+          continue
+
+        log.info("%s, IPV4 = %s" % (slave_reference, ipv4))
+
       except IOError:
-        # XXX- this file should always exists
-        log.debug('Error when writing in file %s. Clould not update status of %s...' %
-                              (status_file, slave_reference))
+        log.debug('Error when writing in file %s. Could not update status of %s...' %
+          (status_file, slave_reference))
 
-  if call_bang and can_bang:
-    bang(args)
+      if not updated or ipv4_changed or ipv6_changed:
+        updated = True
+
+  return updated
+
+def manage(args, can_bang=True):
+
+  client = registry.RegistryClient(args['registry_url']) 
+  base_token_path = args['token_base_path']
+  token_json = args['token_json']
 
-def manage(args):
   # Request Add new tokens
-  requestAddToken(args)
+  has_new_token = requestAddToken(client, base_token_path)
 
   # Request delete removed token
-  requestRemoveToken(args)
+  requestRemoveToken(client, base_token_path)
 
   # check status of all token
-  checkService(args)
+  changed = checkService(client, base_token_path, token_json)
+
+  if (has_new_token or changed) and can_bang:
+    bang(args)
+
 
diff --git a/software/re6stnet/instance-re6stnet.cfg.in b/software/re6stnet/instance-re6stnet.cfg.in
index 2f16e8d47..674a4822a 100644
--- a/software/re6stnet/instance-re6stnet.cfg.in
+++ b/software/re6stnet/instance-re6stnet.cfg.in
@@ -138,9 +138,6 @@ command = {{ re6st_registry }}
 wrapper = ${directory:services}/re6st-registry
 pid-file = ${directory:run}/registry.pid
 manager-wrapper = ${directory:bin}/re6stManageToken
-check-service-wrapper = ${directory:bin}/re6stCheckService
-drop-service-wrapper = ${directory:bin}/re6stManageDeleteToken
-revoke-service-wrapper = ${directory:bin}/re6stRevokeCertificate
 openssl-bin = {{ openssl_bin }}/openssl
 python-bin = {{ python_bin }}
 ipv6-prefix = {{ slapparameter_dict.get('ipv6-prefix', '2001:db8:24::/48') }}
@@ -167,26 +164,12 @@ recipe = slapos.cookbook:wrapper
 wrapper-path = ${directory:script}/re6st-token-manager
 command-line = "{{ python_bin }}" ${re6st-registry:manager-wrapper}
 
-[cron-entry-re6st-check]
+[cron-entry-re6st-manage]
 recipe = slapos.cookbook:cron.d
 cron-entries = ${cron:cron-entries}
 name = re6stnet-check-token
 frequency = */5 * * * *
-command = {{ python_bin }} ${re6st-registry:check-service-wrapper}
-
-[cron-entry-re6st-revoke]
-recipe = slapos.cookbook:cron.d
-cron-entries = ${cron:cron-entries}
-name = re6stnet-revoke-cert
-frequency = */5 * * * *
-command = {{ python_bin }} ${re6st-registry:revoke-service-wrapper}
-
-[cron-entry-re6st-drop]
-recipe = slapos.cookbook:cron.d
-cron-entries = ${cron:cron-entries}
-name = re6stnet-drop-token
-frequency = */5 * * * *
-command = {{ python_bin }} ${re6st-registry:drop-service-wrapper}
+command = {{ python_bin }} ${re6st-registry:manager-wrapper}
 
 [logrotate-entry-re6stnet]
 < = logrotate-entry-base
@@ -227,9 +210,7 @@ parts =
   logrotate-entry-re6stnet
   re6stnet-manage
   cron-entry-logrotate
-  cron-entry-re6st-check
-  cron-entry-re6st-drop
-  cron-entry-re6st-revoke
+  cron-entry-re6st-manage
   apache-httpd
   apache-httpd-graceful
   publish
diff --git a/software/re6stnet/software.cfg b/software/re6stnet/software.cfg
index aaf2b1689..d1f403396 100644
--- a/software/re6stnet/software.cfg
+++ b/software/re6stnet/software.cfg
@@ -1,7 +1,6 @@
 [buildout]
 
 extends =
-  ../../component/re6stnet/buildout.cfg
   ../../component/dash/buildout.cfg
   ../../component/dcron/buildout.cfg
   ../../component/gzip/buildout.cfg
@@ -23,8 +22,6 @@ parts +=
   slapos-cookbook
   eggs
   dash
-  babeld
-  re6stnet
   template
 
 [eggs]
@@ -45,7 +42,6 @@ eggs =
   ${python-cffi:egg}
   ${python-cryptography:egg}
   pyOpenSSL
-  miniupnpc
   re6stnet
 
 [download-base]
@@ -87,7 +83,7 @@ extra-context =
 [template-re6stnet]
 < = download-base
 filename = instance-re6stnet.cfg.in
-md5sum = 6e9452d283e82e2f512a9f9edb17fe3a
+md5sum = 4596d91cef4184b97d257a68478b6330
 
 [template-apache-conf]
 < = download-base
-- 
2.30.9