WIP

slapos/format.py

@@ -25,31 +25,43 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 #
 ##############################################################################
+import configparser
+import ipaddress
+import logging
+import netifaces
+
+from netifaces import AF_INET, AF_INET6
+
+from typing import List, Union
 
 
 def do_format(conf):
+  # load configuration
   computer = Computer(conf)
-  computer.check()
+  # sanity checks
+  computer.validate()
+  # format
   computer.format()
+  # ??
   computer.update()
+  # send to master
   computer.send()
 
 
-class FormatConfig(object):
-  class Parameters(object):
+class Parameters(object):
   master_url: str
   computer_id: str
   instance_root: str
   software_root: str
   partition_amount: int
 
-  class FileOptions(object):
+class Options(object):
+  input_definition_file: str = None
+  computer_cache_file: str = None
+
   key_file: str = None
   cert_file: str = None
-    master_ca_file: str = None
-    input_definition_file: str = None
 
-  class Options(object):
   log_file: str = None
   dry_run: bool = False
 
@@ -72,6 +84,18 @@ class FormatConfig(object):
   tun_ipv6: bool = True
   tun_ipv4_network: str = '172.16.0.0/12'
 
+
+class FormatConfig(Parameters, Options):
+  DEPRECATED = [
+    'bridge_name',
+    'no_bridge_',
+    'master_ca_file',
+    'alter_network',
+    'alter_user',
+    'computer_json',
+    'computer_xml',
+  ]
+  CHECK_FILES = ['key_file', 'cert_file', 'input_definition_file']
   logger : logging.Logger
 
   def __init__(self, logger):
@@ -85,54 +109,64 @@ class FormatConfig(object):
   def parse(self, name, value, t):
     if not isinstance(value, str):
       if not isinstance(value, t):
-        self.error("Option %s is not of type %s", name, t.__name__)
+        self.error("Option %s takes type %s, not %r", name, t.__name__, value)
       return value
     if t in (int,):
       try:
         return t(value)
       except ValueError:
-        self.error("Option %s must be an integer, not %r", name, value)
+        self.error("Option %s takes type %s, not %r", name, t.__name__, value)
     if t is bool:
       try:
         return {'true': True, 'false': False}[value.lower()]
       except KeyError:
-        self.error("Option %r must be 'True' or 'False', not %r", name, value)
+        self.error("Option %r must be 'true' or 'false', not %r", name, value)
+
+  def get(self, option):
+    try:
+      return gettatr(self, option)
+    except AttributeError:
+      self.error("Parameter %r is not defined", option)
 
   def mergeConfig(self, args, configp):
-    for cls in (self.FileOptions, self.Options):
-      self.__dict__.update({k: cls.__dict__[k] for k in cls.__annotations__})
     # args (from command line) override configp (from cfg) options
     for section in ('slapformat', 'slapos'):
       self.__dict__.update(configp.items(section))
     self.__dict__.update(args.__dict__)
 
   def setConfig(self):
-    for parameter, t in self.Parameters.__annotations__.items():
-      try:
-        value = gettatr(self, parameter)
-      except AttributeError:
-        raise UsageError("Parameter %r is not defined" % option)
-      setattr(self, parameter, self.parse(parameters, value, t))
+    for option in self.DEPRECATED:
+      if option in self.__dict__:
+        if option == 'computer_xml':
+          self.error(
+            "Option %r is no longer supported\n"
+            "Use --output_definition_file to migrate existing computer_xml"
+            ", then use the generated file as input_definition_file",
+            option
+          )
+        else:
+          self.error("Option %r is no longer supported" % option)
+
+    for option, t in Parameters.__annotations__.items():
+      setattr(self, option, self.parse(option, self.get(option), t))
   
-    for option, t in self.Options.__annotations__.items():
+    for option, t in self.__annotations__.items():
       setattr(self, option, self.parse(option, getattr(self, option), t))
 
-    for option in self.FileOptions.__annotations__:
+    for option in self.CHECK_FILES:
       path = getattr(self, option)
       if path is not None and not os.path.exists(path):
         self.error("File %r does not exist or is not readable", path)
       setattr(self, option, os.path.abspath(path))
 
-    # XXX
-    # Check command line tools
-    # Logs  
+    # XXX Check command line tools + Logs
 
 
 class Computer(object):
   reference : str
   interface : Interface
-  partitions : [Partitions]
-  address : ipaddress.IPv4Interface, ipaddress.IPv6Interface
+  partitions : List[Partition]
+  address : Union[ipaddress.IPv4Interface, ipaddress.IPv6Interface]
   user : User
   conf : FormatConfig
 
@@ -141,64 +175,96 @@ class Computer(object):
     self.reference = conf.computer_id
     self.interface = Interface(conf)
     if conf.input_definition_file:
-      computer_definition = configparser.ConfigParser(interpolation=None)
-      computer_definition.read(conf.input_definition_file)
-      self.from_definition(conf, computer_definition)
+      definition = configparser.ConfigParser(interpolation=None)
+      definition.read(conf.input_definition_file)
+      self.from_definition(conf, definition)
     else:
       self.from_conf(conf)
 
-  def from_definition(self, conf, computer_definition)
-    self.partitions = [
-      Partition.from_definition(index, conf, computer_definition)
-      for index in conf.partition_amount
-    ]
-    address = computer_definition.get('computer', 'address', fallback=None) # XXX
+  def from_definition(self, conf, definition)
+    amount = conf.partition_amount
+    self.partitions = [Partition(i, conf, definition) for i in range(amount)]
+    address = definition.get('computer', 'address', fallback=None) # XXX fallback
     self.address = ipaddress.ip_interface(address)
-    username = computer_definition.get(
-      'computer', 'software_user', fallback=conf.software_user)
-    self.user = User(conf, username)
+    username = definition.get('computer', 'software_user', fallback=conf.software_user)
+    self.user = User(username)
 
   def from_conf(self, conf):
-    self.partitions = [
-      Partition.from_conf(conf)
-      for index in conf.partition_amount
-    ]
+    amount = conf.partition_amount
+    self.partitions = [Partition(i, conf) for i in range(amount)]
     self.address = None # XXX
-    self.user = User(conf, conf.software_user)
+    self.user = User(conf.software_user)
 
-  def check(self):
+  def validate(self):
     pass
 
   def format(self):
     pass
 
+  def update(self):
+    pass
+
+  def send(self):
+    pass
+
 
 class Interface(object):
   ipv4_interface : str
   ipv6_interface : str
-  ipv4_local_network : ipaddress.IPv4Network
+  ipv4_network : ipaddress.IPv4Network
   ipv6_network : ipaddress.IPv6Network
   conf : FormatConfig
 
   def __init__(self, conf):
-    self.conf
+    self.conf = conf
     self.ipv4_interface = conf.interface_name
     self.ipv6_interface = conf.ipv6_interface or conf.interface_name
-    self.ipv4_local_network = conf.ipv4_local_network
-    self.ipv6_interface_address = self.getIPv6Network()
+    self.ipv4_network = self.getIPv4Network(conf.ipv4_local_network)
+    self.ipv6_network = self.getIPv6Network()
+
+  def getIPv4Network(self, cidr):
+    if cidr:
+      # XXX allow ipv4_local_network to be None ?
+      return ipaddress.IPv4Network(cidr, strict=False)
 
-  def getIPv6InterfaceAddress(self):
+  def getPartitionIPv4(self, index):
+    return self.ipv4_network[index + 2]
+
+  def getIPv6Network(self):
     try:
-      if_addresses = netifaces.ifaddresses(self.ipv6_interface)[socket.AF_INET6]
+      addresses = netifaces.ifaddresses(self.ipv6_interface)[AF_INET6]
     except KeyError:
-      self.conf.error("%s must have at least one IPv6 address assigned", self.ipv6_interface)
-    addr = None
-    for a in if_addresses:
-      ip = ipaddress.ip_interface((q['addr']split('%')[0], q['netmask']))
-      if isGlobalScopeAddress(ip):
-        if not addr or ip.network.num_addresses > addr.network.num_addresses:
-          addr = ip
-    return addr
+      self.conf.error(
+        "%s must have at least one IPv6 address assigned",
+        self.ipv6_interface
+      )
+    result = None
+    for a in addresses:
+      address = a['addr'].split('%')[0]
+      netmask = a['netmask'].split('/')[-1]
+      ip = ipaddress.IPv6Interface('%s/%s' % (address, netmask))
+      network = ip.network
+      if network.is_global:
+        if not result or network.prefixlen < result.prefixlen:
+          result = network
+    return result
+
+  def getComputerIPv6Addr(self):
+    network = self.ipv6_network
+    return ipaddress.ip_interface((network[1], network.prefixlen))
+
+  def getPartitionIPv6Addr(self, index):
+    network = self.ipv6_network
+    return ipaddress.ip_interface((network[index + 2], network.prefixlen))
+
+  def getPartitionIPv6Range(self, index):
+    network = self.ipv6_network
+    prefixlen = network.prefixlen + 16
+    if prefixlen > 128:
+      self.conf.error("IPv6 network %s is too small for IPv6 ranges", network)
+    bits = 128 - network.prefixlen
+    addr = network[(1 << (bits - 2)) + (i << 14)]
+    return ipaddress.ip_interface((addr, prefixlen))
 
 
 class Partition(object):
@@ -206,78 +272,76 @@ class Partition(object):
   index: int
   path : str
   user : User
-  address_list : list
+  ipv4_list: List[ipaddress.IPv4Interface]
+  ipv6_list: List[ipaddress.IPv6Interface]
+  ipv6_range: ipaddress.IPv6Interface
   tap : Tap
   tun : Tun
-  external_storage_list: list
 
-  def __init__(self, **kwargs):
-    self.__dict__.update(kwargs)
+  def __init__(self, index, computer, definition=None):
+    self.from_conf(index, computer)
+    if definition:
+      self.from_definition(index, computer.conf, definition)
 
-  @classmethod
-  def from_definition(cls, index, conf, computer_definition):
+  def from_definition(cls, index, conf, definition):
     section = 'partition_%d' % index
-    if computer_definition.has_section(section):
-      reference = computer_definition.get(section, 'pathname')
-      path = os.path.join(conf.instance_root, reference)
-      user = User(conf, computer_definition.get(section, 'user'), path)
-      address_list = [
-        ipaddress.ip_interface(addr)
-        for add in computer_definition.get(section, 'address')
-      ] # XXX fallback
+    options = {}
+    if definition.has_section('default'):
+      options.update(definition.items('default'))
+    if definition.has_section(section):
+      options.update(definition.items(section))
+    if 'pathname' in options:
+      self.reference = options['pathname']
+      self.path = os.path.join(conf.instance_root, self.reference)
+    if 'user' in options:
+      self.user = User(options['user'], self.path)
+    if 'address' in options:
+      address_list = [ipaddress.ip_interface(a) for a in options['address']]
+      for v in (4, 6):
+        ip_list = [ip for ip in ip_addresses if ip.version == v]
+        if ip_list:
+          setattr(self, 'ipv%d_list' % v, ip_list)
     # tap = Tap(computer_definition.get(section, 'network_interface'))
     # tun = Tun.load(conf, index)
-      return Partition(
-        reference = reference,
-        index = index,
-        path = path,
-        user = user,
-        address_list = address_list,
-        # tap = tap,
-        # tun = tun
-      )
-    # XXX [default] ??
-    else:
-      return cls.from_conf(index, conf)
 
-  @classmethod
-  def from_conf(cls, conf):
-    reference = '%s%d' % (conf.partition_base_name, i)
-    path = os.path.join(conf.instance_root, reference)
-    user = User(conf, '%s%d' % (conf.user_base_name, i), path)
-    address_list = None # XXX
-    return Partition(
-      reference = reference,
-      path = path,
-      user = user,
-      address_list = address_list,
-      # tap = tap,
-      # tun = tun,
-    )
+  def from_conf(self, index, computer):
+    conf = computer.conf
+    self.reference = '%s%d' % (conf.partition_base_name, index)
+    self.path = os.path.join(conf.instance_root, self.reference)
+    self.user = User('%s%d' % (conf.user_base_name, index), self.path)
+    self.ipv4_list = [computer.interface.getPartitionIPv4(index)]
+    self.ipv6_list = [computer.interface.getPartitionIPv6(index)]
+    # XXX Tap & tun
+
+  def createPath(self):
+    self.path = os.path.abspath(self.path)
+    owner = self.user if self.user else User('root')
+    if not os.path.exists(self.path):
+      os.mkdir(self.path, 0o750)
+    owner_pw = pwd.getpwnam(owner.name)
+    os.chown(self.path, owner_pw.pw_uid, owner_pw.pw_gid)
+    os.chmod(self.path, 0o750)
 
 
 class User(object):
   name: str
   path: str
-  additional_group_list: list
+  groups: List[str]
 
   SHELL = '/bin/sh'
 
-  def __init__(self, conf, name, path=None, additional_group_list=None):
-    self.alter_user = conf.alter_user
+  def __init__(self, name, path, groups=None):
     self.name = name
     self.path = path
-    self.additional_group_list = additional_group_list
+    self.groups = groups
 
-  def apply(self):
-    if not self.alter_user:
-      return
+  def create(self):
     grpname = 'grp_' + self.name if sys.platform == 'cygwin' else self.name
     if not self.isGroupAvailable(grpname):
       callAndRead(['groupadd', grpname])
     user_parameter_list = ['-d', self.path, '-g', self.name, '-s', self.SHELL]
-    if self.additional_group_list:
-      user_parameter_list.extend(['-G', ','.join(self.additional_group_list)])
+    if self.groups:
+      user_parameter_list.extend(['-G', ','.join(self.groups), '-a'])
     user_parameter_list.append(self.name)
     if self.isUserAvailable(self.name):
       # if the user is already created and used we should not fail
@@ -305,10 +369,17 @@ class User(object):
       return False
 
 
-# Utils
+# Utilities
 
-def isGlobalScopeAddress(ip):
-  return (
-    not ip.is_link_local and not ip.is_loopback
-    and not ip.is_reserved and not ip.is_multicast
+def callAndRead(argument_list, raise_on_error=True):
+  popen = subprocess.Popen(
+    argument_list,
+    stdout=subprocess.PIPE,
+    stderr=subprocess.STDOUT,
+    universal_newlines=True,
   )
+  result = popen.communicate()[0]
+  if raise_on_error and popen.returncode != 0:
+    raise ValueError('Issue while invoking %r, result was:\n%s' % (
+                     argument_list, result))
+  return popen.returncode, result