vm.install-debian: fixes/improvements

- No more limit on the number of preseed parameters, by placing a preseed.cfg
  file inside the initrd, instead of passing them all via the command line.
  The kernel is usually limited to 32 parameters and it panics when there are
  too many.

- Dist-specific options.

- Recognize preseed aliases.

- late-command is run with '/bin/sh -e' and it must exit with EX_OK (0),
  otherwise the installer stops.

README.rst

@@ -641,6 +641,7 @@ arch
 dists
     List of VMs to build: each token refers to a buildout section name that
     describes the ISOs to use. See `ISO sections`_ below.
+    Tokens can't contain `'.'` characters.
 
 size
     Size of the VM image. This must be an integer, optionally followed by a
@@ -649,28 +650,31 @@ size
 mem
     Default: 256
 
-preseed.<preseed>
+[<dist>/]preseed.<preseed>
     Set the <preseed> value for the installation. The recipe has many default
     preseed values: you can see the list in the ``InstallDebianRecipe.preseed``
-    class attribute (file ``slapos/recipe/vm.py``). Note however that:
+    class attribute (file ``slapos/recipe/vm.py``). Aliases are recognized
+    (but the recipe includes a mapping that may be out-of-date.).
+    Any value except ``passwd/*`` can optionally be prefixed so that they only
+    apply for a particular VM.
 
-    - aliases are not recognized so you must check this list if you want to
-      override any default value;
-    - all values are passed via the kernel command line, so beware complex
-      values (double-quotes are automatically added when there are spaces).
-
-debconf.<owner>
+[<dist>/]debconf.<owner>
     List of debconf value for <owner> (usually a package name),
     each line with 2 whitespace-separated parts: <key> <value>.
+    Like for preseed.* values, they can be specific to <dist>.
 
 late-command
     Shell commands to execute at the end of the installation. They are run
     inside the target system. This is a reliable alternative to the
-    ``preseed.preseed/late_command`` option.
+    ``preseed.preseed/late_command`` option. The ``DIST`` shell variable is
+    set to the VM being built.
 
 packages
     Extra packages to install.
     Like for `late-command`, do not use ``preseed.pkgsel/include``.
+    If you want to install packages only for some specific <dist>, you can do
+    it in ``late-command``, by testing ``$DIST`` and using
+    ``apt-get install -y``.
 
 vm.run
     Boolean value that is `true` by default, to configure the VM for use with
@@ -730,9 +734,9 @@ Example
     debconf/frontend noninteractive
     debconf/priority critical
   # minimal size
+  preseed.apt-setup/enable-source-repositories = false
   preseed.recommends = false
   preseed.tasks =
-  packages = localepurge
 
   [debian-jessie]
   x86_64.iso = debian-amd64-netinst.iso
@@ -754,6 +758,12 @@ modified).
 
 ``${buildout:directory}`` is always mounted as `/mnt/buildout` inside the VM.
 
+Mount points use the 9p file-system. Make sure that:
+
+- QEMU is built with --enable-virtfs;
+- the VM runs a kernel that is recent enough (Debian Squeeze kernel 2.6.32 is
+  known to fail, and you'd have to use the one from squeeze-backports).
+
 Options
 ~~~~~~~
 

slapos/recipe/vm.py

@@ -25,8 +25,10 @@
 #
 ##############################################################################
 
-import base64, os, select, shutil, socket
+import base64, gzip, os, select, shutil, socket, stat
 import subprocess, sys, tempfile, threading, time
+from cStringIO import StringIO
+from collections import defaultdict
 from contextlib import contextmanager
 from os.path import join
 from slapos.recipe import EnvironMixin, generatePassword, logger, rmtree
@@ -61,6 +63,38 @@ class Popen(subprocess.Popen):
       return r
 
 
+class CPIO(object):
+
+  ino = 1
+
+  def __init__(self):
+    self.buf = StringIO()
+    self.mtime = int(time.time())
+
+  def _add(self, path, mode, data='',
+          _header="070701" + "%08x" * 13):
+    size = len(data)
+    path_len = len(path)
+    write = self.buf.write
+    write(_header % (self.ino, mode, 0, 0, 1,
+      self.mtime, size, 0, 0, 0, 0, path_len + 1, 0))
+    write(path + '\0' * (4 - (path_len + 2) % 4))
+    write(data + '\0' * ((4 - size) % 4))
+    self.ino += 1
+
+  def add_file(self, path, data, mode=0644):
+    self._add(path, mode | stat.S_IFREG, data)
+
+  def close(self):
+    self.ino = self.mtime = 0
+    try:
+      self._add("TRAILER!!!", 0)
+      self.buf.write('\0' * ((512 - self.buf.tell()) % 512))
+      return self.buf
+    finally:
+      self.__dict__.clear()
+
+
 class BaseRecipe(EnvironMixin):
 
   def __init__(self, buildout, name, options, allow_none=True):
@@ -114,7 +148,6 @@ class InstallDebianRecipe(BaseRecipe):
     partman/confirm_nooverwrite = true
     grub-installer/bootdev = default
     finish-install/reboot_in_progress = note
-    preseed/url = file:///dev/null
 
     clock-setup/ntp = false
     time/zone = UTC
@@ -127,6 +160,29 @@ class InstallDebianRecipe(BaseRecipe):
     partman-auto/expert_recipe = : 1 1 -1 ext4 $primary{ } $bootable{ } method{ format } format{ } use_filesystem{ } filesystem{ ext4 } mountpoint{ / } options/discard{ } options/noatime{ } .
     """
 
+  # XXX: The mapping should be automatically computed from the
+  #      /etc/preseed_aliases in the initrd.
+  _alias = (lambda alias: staticmethod(lambda x: alias(x, x)))({
+      'auto':       'auto-install/enable',
+      'classes':    'auto-install/classes',
+      'country':    'debian-installer/country',
+      'desktop':    ('tasksel', 'tasksel/desktop'),
+      'dmraid':     'disk-detect/dmraid/enable',
+      'domain':     'netcfg/get_domain',
+      'fb':         'debian-installer/framebuffer',
+      'hostname':   'netcfg/get_hostname',
+      'interface':  'netcfg/choose_interface',
+      'keymap':     'keyboard-configuration/xkb-keymap',
+      'language':   'debian-installer/language',
+      'locale':     'debian-installer/locale',
+      'modules':    'anna/choose_modules',
+      'priority':   'debconf/priority',
+      'protocol':   'mirror/protocol',
+      'recommends': 'base-installer/install-recommends',
+      'suite':      'mirror/suite',
+      'tasks':      ('tasksel', 'tasksel/first'),
+    }.get)
+
   def __init__(self, buildout, name, options):
     BaseRecipe.__init__(self, buildout, name, options)
     self.vm = options['location']
@@ -140,8 +196,6 @@ class InstallDebianRecipe(BaseRecipe):
         dist[arch + '.kernel'],
         dist[arch + '.initrd']))
 
-    if 'preseed.preseed/late_command' in options:
-      raise UserError("use 'late-command' recipe option instead")
     late_command = (options.get('late-command') or '').strip()
     self.late_command = [late_command] if late_command else []
 
@@ -156,44 +210,60 @@ class InstallDebianRecipe(BaseRecipe):
 
   def install(self):
     options = self.options
-    cmdline = {}
-    for preseed in self.preseed.splitlines():
-      preseed = preseed.strip()
-      if preseed and preseed[0] != '#':
-        k, v = preseed.split('=', 1)
-        cmdline[k.strip()] = v.strip()
+    preseed = defaultdict(dict)
+    common = preseed[None]
+    for p in self.preseed.splitlines():
+      p = p.strip()
+      if p and p[0] != '#':
+        k, v = p.split('=', 1)
+        common[self._alias(k.strip())] = v.strip()
     for k, v in options.iteritems():
-      if k.startswith('preseed.'):
-        cmdline[k[8:]] = v.strip()
-      elif k.startswith('debconf.'):
-        owner = k[8:] + ':'
-        for k in v.splitlines():
       try:
-            k, v = k.split(None, 1)
+        p, k = k.split('.', 1)
+        p = p.rsplit('/', 1)
+        x = ('preseed', 'debconf').index(p.pop())
+      except ValueError:
+        continue
+      p = preseed[p[0]] if p else common
+      if x:
+        for x in v.splitlines():
+          try:
+            x, v = x.split(None, 1)
           except ValueError:
-            if not k:
+            if not x:
               continue
             v = ''
-          cmdline[owner + k] = v
+          p[(k, x)] = v
+      else:
+        k = self._alias(k)
+        if isinstance(k, str):
+          if k in ('preseed/late_command', 'pkgsel/include'):
+            raise UserError('Use the recipe-specific option instead of %s.' % k)
+          if k in ('preseed/url', 'preseed/file', 'preseed/file/checksum'):
+            # We could extend a provided preseed file.
+            raise NotImplementedError
+          if k.startswith('passwd/') and p is not common:
+            raise NotImplementedError
+        p[k] = v.strip()
 
     vm_run = is_true(options.get('vm.run', 'true'))
     packages = ['ssh', 'sudo'] if vm_run else []
     packages += options.get('packages', '').split()
     if packages:
-      cmdline['pkgsel/include'] = ','.join(packages)
+      common['pkgsel/include'] = ','.join(packages)
 
     generated = []
     for x, p in (('root', 'passwd/root-login'),
                  ('user', 'passwd/make-user')):
-      if is_true(cmdline[p]):
+      if is_true(common[p]):
         p = 'passwd/%s-password' % x
         if x == 'user':
-          x = cmdline.get('passwd/username')
+          x = common.get('passwd/username')
           if not x:
             raise UserError('passwd/username is empty')
-          cmdline.setdefault('passwd/user-fullname', '')
-        if not (cmdline.get(p) or cmdline.get(p + '-crypted')):
-          cmdline[p] = cmdline[p + '-again'] = passwd = generatePassword()
+          common.setdefault('passwd/user-fullname', '')
+        if not (common.get(p) or common.get(p + '-crypted')):
+          common[p] = common[p + '-again'] = passwd = generatePassword()
           generated.append((x, passwd))
 
     env = self.environ
@@ -208,14 +278,23 @@ class InstallDebianRecipe(BaseRecipe):
           os.remove(key)
           key = f.read().strip()
         self.late_command.append(
-          "mkdir -m 0700 ~/.ssh && echo %s > ~/.ssh/authorized_keys" % key)
+          "mkdir -m 0700 ~/.ssh\necho %s > ~/.ssh/authorized_keys" % key)
+      for dist, iso, kernel, initrd in self.dists:
+        cpio = CPIO()
+        p = common.copy()
+        p.update(preseed.get(dist, ()))
         if self.late_command:
-        cmdline['preseed/late_command'] = (
-          'cd /target && echo %s|usr/bin/base64 -d|chroot . sh'
-          % base64.standard_b64encode(
-            'export HOME=/root\n' + '\n'.join(self.late_command)))
+          p['preseed/late_command'] = ('set -e; unset DEBCONF_REDIR'
+            ' DEBIAN_FRONTEND DEBIAN_HAS_FRONTEND DEBCONF_OLD_FD_BASE;'
+            ' cd /target; cp /late-command .; exec chroot . /late-command')
+          cpio.add_file('late-command', '#!/bin/sh -e\nrm $0\n'
+            'export HOME=/root; DIST=%s\n%s\n'
+            % (dist, '\n'.join(self.late_command)), 0755)
+        cpio.add_file('preseed.cfg', ''.join(sorted(
+          "%s string %s\n" % ('%s %s' % k if type(k) is tuple else
+                              'd-i ' + k, v)
+          for k, v in p.iteritems())))
 
-      for dist, iso, kernel, initrd in self.dists:
         vm = join(location, dist + '.img')
         args = self.getQemuBasicArgs(dist, 256, unsafe=True)
         open_flags = os.O_CREAT | os.O_EXCL | os.O_WRONLY
@@ -228,13 +307,14 @@ class InstallDebianRecipe(BaseRecipe):
         try:
           subprocess.check_call(('7z', 'x', iso, kernel, initrd),
                                 cwd=tmp, env=env)
+          initrd = join(tmp, initrd)
+          with gzip.open(initrd, 'ab') as f:
+            f.write(cpio.close().getvalue())
           args += (
             '-vnc', join('unix:' + tmp, 'vnc.sock'), # for debugging purpose
             '-cdrom', iso, '-no-reboot',
             '-kernel', join(tmp, kernel),
-            '-initrd', join(tmp, initrd),
-            '-append', ' '.join(k + '=' + ('"' + v + '"' if ' ' in v else v)
-                                for k, v in cmdline.iteritems()))
+            '-initrd', initrd)
           subprocess.check_call(args, env=env)
         finally:
           shutil.rmtree(tmp)