re6st-node: reimplement in Python the part to build files to send to OBS

Makefile was so horrible and unreliable.
I tried https://waf.io/ which is nice but not suitable for this
(see the example 'wscript' at the end).

Functional improvements:
- better detection of what needs to be rebuilt or not
- reproducible tarballs, except for the re6stnet egg
  (and the main tarball if the egg is rebuilt)
- fewer temporary files

And support for OSC is back.

_______________________________________________________________________________

import os, shutil, subprocess, urllib
from waflib import Node, Utils

PREFIX = "opt/re6st"
BOOTSTRAP_URL = "http://downloads.buildout.org/1/bootstrap.py"
repo_dict = dict(
    re6stnet="http://git.erp5.org/repos/re6stnet.git",
    slapos="http://git.erp5.org/repos/slapos.git",
)

def configure(ctx):
    for name, url in repo_dict.iteritems():
        if ctx.path.find_node(name) is None:
            ctx.exec_command(("git", "clone", url), stdout=None, stderr=None)

def cfg(task):
    o, = task.outputs
    bld = task.generator.bld
    o.write(task.inputs[0].read() % dict(
        SLAPOS=bld.srcnode.search_node("slapos").abspath(),
        ROOT="${buildout:directory}/" + bld.bldnode.path_from(o.parent),
        TARGET="/"+PREFIX))

def bootstrap(task):
    b = task.outputs[0]
    d = b.parent.parent
    bootstrap = urllib.urlopen(BOOTSTRAP_URL).read()
    for cache in "download-cache", "extends-cache":
        cache = d.make_node(cache).abspath()
        if os.path.exists(cache):
            shutil.rmtree(cache)
        os.mkdir(cache)
    cwd = d.abspath()
    task.exec_command(("python2", "-S"), input=bootstrap, cwd=cwd)
    task.exec_command((b.abspath(), "buildout:parts=python"), cwd=cwd)

def sdist(task):
    r, p = task.inputs
    d = p.find_node("../../download-cache/dist")
    for x in d.ant_glob("re6stnet-*", quiet=True):
        x.delete()
    task.exec_command((p.abspath(), "setup.py", "sdist", "-d", d.abspath()),
                      cwd=r.abspath())

def build(bld):
    b = bld.bldnode.make_node(PREFIX)
    buildout_cfg = b.make_node("buildout.cfg")
    bld(source="buildout.cfg.in", target=buildout_cfg, rule=cfg)
    tg = bld(source=(buildout_cfg, "slapos"), rule=bootstrap,
             target=map(b.make_node, ("bin/buildout", "bin/python")))
    buildout, python = tg.target
    r = bld.path.find_node("re6stnet")
    tg = bld(source=(r, python), rule=sdist, update_outputs=True,
             target=r.make_node("re6stnet.egg-info/PKG-INFO"))
    bld(name="buildout", source=(buildout_cfg, tg.target),
        target=b.make_node(".installed.cfg"),
        rule=lambda task: task.exec_command((buildout.abspath(),),
            stdout=None, stderr=None, cwd=b.abspath()))

 ###

def h_file(fname, h_file=Utils.h_file):
    if os.path.isdir(fname):
        m = Utils.md5(fname)
        n = len(fname) + 1
        for dirpath, dirs, files in os.walk(fname):
            if dirpath.endswith(("/.git", ".egg-info")):
                del dirs[:]
                continue
            dirs.sort()
            files.sort()
            m.update(dirpath[n:] + '\0')
            for fname in files:
                m.update("%s\0%s\0" % (h_file(os.path.join(dirpath, fname)),
                                       fname))
        return m.digest()
    return h_file(fname)
Utils.h_file = h_file

def find_resource(self, lst):
    if isinstance(lst, str):
        lst = [x for x in Node.split_path(lst) if x and x != '.']
    return self.get_bld().search_node(lst) \
        or self.get_src().find_node(lst)
Node.Node.find_resource = find_resource

obs/re6st/.gitignore

+/dist/
 /re6stnet/
 /slapos/
-/upstream.mk
-/re6st-node_*.tar.gz
-/re6st-node_*.dsc
-/debian/control
+/osc
 /debian/changelog
-/re6stnet.spec
-/PKGBUILD

obs/re6st/README

-preparing the package
----------------------
+Building the files to be sent to OBS
+------------------------------------
 
 First make sure all files are ready and you have all necessary packages installed.
-You need in particular an OBS directory (for example, the vifib test directory)::
 
-  $ cd <directory_to_contain_prepare_script>
-  $ osc checkout home:VIFIBnexedi:branches:home:VIFIBnexedi/Re6stnet
-  $ cd home:VIFIBnexedi:branches:home:VIFIBnexedi/Re6stnet
-  $ osc up
+  $ ./make
 
+All output files are in the 'dist' folder, which is created automatically.
+
+Upload to OBS
+-------------
+
+For this, you need a checkout of the OBS repository, and make a 'osc' symlink
+pointing to it. For example, the vifib test directory::
+
+  A$ cd <where_you_want>
+  B$ osc checkout home:VIFIBnexedi:branches:home:VIFIBnexedi/Re6stnet
+  B$ cd home:VIFIBnexedi:branches:home:VIFIBnexedi/Re6stnet
+  B$ osc up
+  B$ cd A
+  A$ ln -s B/home:VIFIBnexedi:branches:home:VIFIBnexedi/Re6stnet osc
+
+And whenever you want to push updates::
+
+  A$ ./make osc
+  A$ (cd osc; osc commit)
+
+Warning about SlapOS updates
+----------------------------
+
+When a SlapOS update would add new files to download-cache or extends-cache,
+everything should be rebuilt by deleting the 'build' folder, in order to remove
+unused files from the caches.

obs/re6st/make

+#!/usr/bin/env python
+#
+# Copyright (C) 2016 Julien Muchembled <jm@nexedi.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+"""
+  Very basic Python implementation of a build system similar to the well-known
+  'make'. The main concept is the same:
+    input files --( recipe )-> output files
+  with comparison of timestamps to trigger recipes
+  (i.e. no temporary files generated to track the status of the build).
+  The main differences are:
+  - no parallelism
+  - no implicit rule, no predefined rule
+  + inputs can be whole directory
+  + change time of inputs is also taken into account
+  + multiple list of outputs for a recipe
+  + dynamic list of outputs
+  + no need to care about wrongly/partially written files in case of failure
+  + Python...
+"""
+import argparse, atexit, errno, gzip, os, shutil, subprocess, sys, tarfile
+from contextlib import contextmanager
+
+INF = float("inf")
+_INF = - INF
+
+class _task(object):
+
+    def __init__(self, *src):
+        self.outputs = src
+
+    def __str__(self):
+        o = self.outputs
+        return repr(o[0]) if len(o) == 1 else str(o)
+
+    @property
+    def output(self):
+        o, = self.outputs
+        return o
+
+    def __call__(self, dry_run):
+        run = self._run
+        if callable(run):
+            self._run = run = run(dry_run)
+        return run
+
+    @staticmethod
+    def _time(path):
+        return os.stat(path).st_mtime
+
+    def _run(self, dry_run=None):
+        x = self.outputs
+        return max(map(self._time, x)) if x else _INF
+
+class files(_task):
+
+    def __init__(self, *src, **kw):
+        if not kw.pop("ctime", True):
+            self._time = _task._time
+        _task.__init__(self, *src, **kw)
+
+    @staticmethod
+    def _time(path):
+        s = os.stat(path)
+        return max(s.st_ctime, s.st_mtime)
+
+class tree(files):
+    """
+    e.g. @task(tree("some/folder"), "some/output")
+    """
+
+    def __init__(self, root, ignore=None, **kw):
+        files.__init__(self, root, **kw)
+        self.ignore = ignore
+
+    def __iter__(self):
+        root, = self.outputs
+        yield root
+        ignore = self.ignore
+        n = len(root) + 1
+        for dirpath, dirs, files in os.walk(root):
+            dirpath += os.sep
+            if ignore:
+                x = dirpath[n:]
+                dirs[:] = (name for name in dirs if not ignore(x + name))
+            for name in dirs:
+                yield dirpath + name
+            for name in files:
+                x = dirpath + name
+                if not (ignore and ignore(x[n:])):
+                    yield x
+
+    def _run(self, dry_run=None):
+        return max(map(self._time, self))
+
+class task(_task):
+    """
+    @task(depends, provides)
+    def mytask(task):
+        # task.inputs
+        # task.outputs
+
+    'depends' is a sequence of other tasks or file paths. The order is important
+    because it defines the order of task.inputs: not however that paths are
+    automatically moved after tasks.
+
+    'provides' is a sequence of callables or file paths. Callables are always
+    called to complete task.outputs. Recipe is always called if task.outputs
+    is empty.
+
+    For both 'depends' and 'provides', if you have only 1 element, you can pass
+    it directly instead of making a 1-size sequence.
+
+    'input' and 'output' properties are shortcut to get the only path.
+    'why' is a list of tasks explaining why the recipe is called.
+    """
+
+    def __new__(cls, *args, **kw):
+        def task_gen(func):
+            self = _task.__new__(cls)
+            self.__init__(func, *args, **kw)
+            return self
+        return task_gen
+
+    def __init__(self, run, depends, provides=(),
+                 __str_or_task = (basestring, _task)):
+        self.run = run
+        self.depends = []
+        f = []
+        for x in (depends,) if isinstance(depends, __str_or_task) else depends:
+            (f if isinstance(x, basestring) else self.depends).append(x)
+        f and self.depends.append(files(*f))
+        self.provides = ((provides,)
+            if isinstance(provides, basestring) or callable(provides)
+            else provides)
+        self.why = self,
+
+    def __str__(self):
+        return self.run.__name__
+
+    @property
+    def input(self):
+        i, = self.inputs
+        return i
+
+    def _run(self, dry_run, _otime=INF):
+        self.inputs = x = []
+        deps = []
+        for dep in self.depends:
+            deps.append((dep, dep(dry_run)))
+            x += dep.outputs
+        self.outputs = x = []
+        for p in self.provides:
+            if callable(p):
+                x += p(self)
+            else:
+                x.append(p)
+        if None not in x:
+            try:
+                _otime = _task._run(self)
+            except OSError as e:
+                if e.errno != errno.ENOENT:
+                    raise
+            else:
+                if deps:
+                    self.why = [dep for dep, itime in deps if _otime < itime]
+        if self.why:
+            print "# Processing %s: %s -> %s" % (self,
+                ", ".join(map(str, self.depends)),
+                ", ".join("<%s>" % x.__name__ if callable(x) else x
+                          for x in self.provides or "?"))
+            if not dry_run:
+                try:
+                    self.run(self)
+                except:
+                    if _otime is INF:
+                        _otime = max(x[1] for x in deps) - 1
+                    # Files may still be open due to references in tracebacks.
+                    # Make sure they're all closed before reverting mtimes.
+                    atexit.register(self._revert, self.outputs, _otime)
+                    raise
+                return _task._run(self) if self.outputs else INF
+        return _otime
+
+    @classmethod
+    def _revert(cls, outputs, mtime):
+        try:
+            for x in outputs:
+                if mtime < cls._time(x):
+                    os.utime(x, (mtime, mtime))
+        except Exception:
+            pass
+
+def main():
+    parser = argparse.ArgumentParser()
+    _ = parser.add_argument
+    _("-f", "--file", type=argparse.FileType("r"), default="make.py",
+      help="Python script describing how to build the project"
+           " (default: make.py).")
+    _("-l", "--list", action="store_true",
+      help="List defined tasks.")
+    _("-n", "--dry-run", action="store_true",
+      help="Print the tasks that would be executed.")
+    _("task", nargs="*", default=("build",),
+      help="Tasks to process (default: build).")
+    args = parser.parse_args()
+
+    sys.modules["make"] = sys.modules.pop(__name__)
+    f = args.file
+    tasks = {"__file__": f.name}
+    exec(compile(f.read(), f.name, "exec"), tasks)
+    if args.list:
+        print " ".join(sorted(k for k, v in tasks.iteritems()
+                                if isinstance(v, _task)))
+        return
+    for t in args.task:
+        if not isinstance(tasks.get(t), _task):
+            sys.exit("%s is not a valid task." % t)
+    for t in args.task:
+        tasks[t](args.dry_run)
+
+##
+# Helpers
+#
+
+class git(tree):
+
+    def __init__(self, root, url=None, ignore=None, **kw):
+        _ignore = lambda x: x == ".git" or ignore and ignore(x)
+        tree.__init__(self, root, _ignore, **kw)
+        self.url = url
+
+    def _run(self, dry_run):
+        root, = self.outputs
+        if os.path.isdir(root):
+            return tree._run(self)
+        dry_run or subprocess.check_call(("git", "clone", self.url, root))
+        return _INF
+
+def check_output(*args, **kw):
+    # BBB: 'input' arg is a backport from Python 3.4
+    input = kw.pop("input", None)
+    if input is not None:
+        kw["stdin"] = subprocess.PIPE
+    p = subprocess.Popen(stdout=subprocess.PIPE, *args, **kw)
+    out = p.communicate(input)[0]
+    if p.returncode:
+        raise subprocess.CalledProcessError(p.returncode,
+            args[0] if args else kw["args"])
+    return out
+
+@contextmanager
+def cwd(path):
+    p = os.getcwd()
+    try:
+        os.chdir(path)
+        yield p
+    finally:
+        os.chdir(p)
+
+def mkdir(path):
+    try:
+        os.makedirs(path)
+    except OSError as e:
+        if e.errno != errno.EEXIST or not os.path.isdir(path):
+            raise
+
+def remove(path):
+    try:
+        os.remove(path)
+    except OSError as e:
+        if e.errno != errno.ENOENT:
+            raise
+
+def rmtree(path):
+    if os.path.exists(path):
+        shutil.rmtree(path)
+
+@contextmanager
+def make_tar_gz(path, mtime, xform=(lambda x: x), **kw):
+    # Make reproducible tarball. Otherwise, it's really annoying that we can't
+    # rely on 'osc status' to know whether there are real changes or not.
+    # BBB: Results differ between Python 2.6 and 2.7 because of tarfile.
+    __init__ = gzip.GzipFile.__init__
+    listdir = os.listdir
+    try:
+        if sys.version_info >= (2, 7):
+            gzip.GzipFile.__init__ = lambda *args: __init__(mtime=mtime, *args)
+        os.listdir = lambda path: sorted(listdir(path))
+        t = tarfile.open(path, "w:gz", **kw)
+        _gettarinfo = t.gettarinfo
+        def gettarinfo(name=None, arcname=None, fileobj=None):
+            tarinfo = _gettarinfo(name, xform(arcname or name), fileobj)
+            tarinfo.mtime = mtime
+            return tarinfo
+        t.gettarinfo = gettarinfo
+        yield t
+    except:
+        remove(path)
+        raise
+    finally:
+        gzip.GzipFile.__init__ = __init__
+        os.listdir = listdir
+        t.close()
+    if sys.version_info < (2, 7):
+        with open(path, "r+") as t:
+            t.seek(4)
+            gzip.write32u(t, long(mtime))
+
+if __name__ == "__main__":
+    sys.exit(main())

obs/re6st/make.py

@@ -2,17 +2,17 @@
 # It automatically clones missing repositories but doesn't automatically pull.
 # You have to choose by using git manually.
 
-# Run with SLAPOS_EPOCH=<N> (where <N> is an integer > 1)
+# Run with SLAPOS_EPOCH=<N> environment variable (where <N> is an integer > 1)
 # if rebuilding for new SlapOS version but same re6stnet.
 
 # Non-obvious dependencies:
 # - Debian: python-debian, python-docutils | python3-docutils
 # We could avoid them by doing like for setuptools, but I'd rather go the
-# opposite way: simplify the upload part by using the system setuptools and
-# recent features from Make 4.
+# opposite way: simplify the upload part by using the system setuptools.
 
-# This Makefile tries to be smart by only rebuilding the necessary parts after
-# some change. But as always, it does not handle all cases, so once everything
+# This "makefile" is quite smart at only rebuilding the necessary parts after
+# some change. The main exception concerns the download-cache & extends-cache,
+# because the 'buildout' step is really long. In doubt, and once everything
 # works, you should clean up everything before the final prepare+upload.
 
 # TODO:
@@ -33,17 +33,6 @@
 #   - re6stnet sdist
 #   - a tarball of remaining download-cache
 #   - 1 tarball with everything else
-# - Make tarballs "reproducible". If the contents does not change, and we don't
-#   really care about modification times, the result should always be the same.
-#   It's really annoying that we can't rely on 'osc status' to know whether
-#   there are real changes or not. 2 ways:
-#   - This is doable with a very recent of tar (not even in Jessie), in order
-#     to sort files by name (--sort option). Then there are timestamps:
-#     see -n option of gzip, and --mtime option of tar.
-#   - But the best one is probably to use Python instead of tar+gzip.
-#     And in fact, such Python script should not be limited to that, since many
-#     intermediate files like re6stnet.spec are so quick to generate that they
-#     should be produced in RAM.
 #
 # Note that package don't contain *.py[co] files and they're not generated
 # at installation. For this package, it's better like this because it minimizes
@@ -51,124 +40,216 @@
 # If this way of packaging is reused for other software, postinst scripts
 # should be implemented.
 
-BOOTSTRAP_URL = http://downloads.buildout.org/1/bootstrap.py
-RE6STNET_URL = http://git.erp5.org/repos/re6stnet.git
-SLAPOS_URL = http://git.erp5.org/repos/slapos.git
-
-PACKAGE = re6st-node
-BIN = re6st-conf re6st-registry re6stnet
-NOPART = chrpath flex glib lunzip m4 patch perl popt site_perl xz-utils
-TARGET = opt/re6st
-
-ROOT = build
-BUILD = $(ROOT)/$(TARGET)
-
-BUILD_KEEP = buildout.cfg extends-cache download-cache
-
-all: tarball debian.tar.gz re6stnet.spec PKGBUILD re6stnet.install
-
-re6stnet:
-	git clone $(RE6STNET_URL) $@
-
-slapos:
-	git clone $(SLAPOS_URL) $@
-
-$(BUILD)/buildout.cfg: buildout.cfg.in
-	mkdir -p $(@D)
-	python2 -c 'import os; cfg = open("$<").read() % dict( \
-		SLAPOS="$(CURDIR)/slapos", ROOT="$${buildout:directory}/" \
-			+ os.path.relpath("$(ROOT)", "$(BUILD)"), \
-		TARGET="/$(TARGET)"); open("$@", "w").write(cfg)'
-
-$(BUILD)/bin/python: $(BUILD)/buildout.cfg slapos
-	if [ -e $@ ]; then touch $@; else cd $(BUILD) \
-	&& rm -rf extends-cache && mkdir -p download-cache extends-cache \
-	&& wget -qO - $(BOOTSTRAP_URL) | python2 -S \
-	&& bin/buildout buildout:parts=$(@F); fi
-
-re6stnet/re6stnet.egg-info: $(BUILD)/bin/python re6stnet
-	rm -f $(BUILD)/download-cache/dist/re6stnet-*
-	cd re6stnet && ../$< setup.py sdist -d ../$(BUILD)/download-cache/dist
-	# Touch target because the current directory is used as temporary
-	# storage, and it is cleaned up after that setup.py runs egg_info.
-	touch $@
-
-$(BUILD)/.installed.cfg: re6stnet/re6stnet.egg-info
-	cd $(BUILD) && bin/buildout
-	# Touch target in case that buildout had nothing to do.
-	touch $@
-
-$(ROOT)/Makefile: Makefile.in Makefile
-	($(foreach x,BIN NOPART BUILD_KEEP TARGET, \
-	echo $(x) = $($(x)) &&) cat $<) > $@
-
-prepare: $(BUILD)/.installed.cfg $(ROOT)/Makefile
-	$(eval VERSION = $(shell cd $(BUILD)/download-cache/dist \
-	&& set re6stnet-* && set $${1#*-} \
-	&& echo -n $${1%.tar.*}+slapos$(SLAPOS_EPOCH).g \
-	&& cd $(CURDIR)/slapos && git rev-parse --short HEAD))
-	make -C re6stnet
-
-upstream.mk: re6stnet Makefile
-	(echo 'override PYTHON = /$(TARGET)/parts/python2.7/bin/python' \
-	&& cat $</Makefile) > $@
-
-tarball: upstream.mk prepare
-	tar -caf $(PACKAGE)_$(VERSION).tar.gz \
-		--xform s,^re6stnet/,, \
-		--xform s,^,$(PACKAGE)-$(VERSION)/, \
-		cleanup install-eggs rebootstrap $< \
-		re6stnet/daemon re6stnet/docs/*.1 re6stnet/docs/*.8 \
-		-C $(ROOT) Makefile $(patsubst %,$(TARGET)/%,$(BUILD_KEEP))
-
-debian/changelog: prepare
-	cd re6stnet && sed s,$@,../$@, debian/common.mk | \
-	make -f - PACKAGE=$(PACKAGE) VERSION=$(VERSION) ../$@
-
-debian/control: debian/source/format prepare Makefile
-	$(eval DSC = $(PACKAGE)_$(VERSION).dsc)
-	python2 -c 'from debian.deb822 import Deb822; d = Deb822(); \
-		b = open("re6stnet/$@"); s = Deb822(b); b = Deb822(b); \
-		d["Format"] = open("$<").read().strip(); \
-		d["Source"] = s["Source"] = b["Package"] = "$(PACKAGE)"; \
-		d["Version"] = "$(VERSION)"; \
-		d["Architecture"] = b["Architecture"] = "any"; \
-		d["Build-Depends"] = s["Build-Depends"] = \
-		"python (>= 2.6), debhelper (>= 8)"; \
-		b["Depends"] = "$${shlibs:Depends}, iproute2 | iproute"; \
-		b["Conflicts"] = b["Provides"] = b["Replaces"] = "re6stnet"; \
-		open("$@", "w").write("%s\n%s" % (s, b)); \
-		open("$(DSC)", "w").write(str(d))'
-
-debian.tar.gz: $(patsubst %,debian/%,changelog control prerm postinst rules source/*)
-# Unfortunately, OBS does not support symlinks.
-	set -e; cd re6stnet; [ ! -e debian/postinst ]; \
-	x=`find debian ! -type d $(patsubst %,! -path %,$^))`; \
-	tar -chaf ../$@ $$x -C .. $^
-
-define SED_SPEC
-	# https://fedoraproject.org/wiki/Packaging:Python_Appendix#Manual_byte_compilation
-	1i%global __os_install_post %(echo '%{__os_install_post}' |grep -v brp-python-bytecompile)
-	/^%define (_builddir|ver)/d
-	s/^(Name:\s*).*/\1$(PACKAGE)/
-	s/^(Version:\s*).*/\1$(VERSION)/
-	s/^(Release:\s*).*/\11/
-	/^BuildArch:/cAutoReqProv: no\nBuildRequires: gcc-c++, make, python\n#!BuildIgnore: rpmlint-Factory\nSource: %{name}_%{version}.tar.gz
-	/^Requires:/{/iproute/!d}
-	/^Recommends:/d
-	s/^(Conflicts:\s*).*/\1re6stnet/
-	/^%description$$/a%prep\n%setup -q
-	/^%preun$$/,/^$$/{/^$$/ifind /$(TARGET) -type f -name '*.py[co]' -delete
-	}
-endef
-
-re6stnet.spec: prepare
-	$(eval export SED_SPEC)
-	sed -r "$$SED_SPEC" re6stnet/$@ > $@
-
-PKGBUILD: PKGBUILD.in prepare
-	sed 's/%VERSION%/$(VERSION)/' $< > $@
-
-clean:
-	rm -rf "$(ROOT)" *.dsc *.tar.gz re6stnet.spec \
-		upstream.mk debian/control debian/changelog
+import os, rfc822, shutil, time, urllib
+from glob import glob
+from cStringIO import StringIO
+from subprocess import check_call
+from make import *
+from debian.changelog import Changelog
+from debian.deb822 import Deb822
+
+BOOTSTRAP_URL = "http://downloads.buildout.org/1/bootstrap.py"
+PACKAGE = "re6st-node"
+
+BIN = "re6st-conf re6st-registry re6stnet".split()
+BUILD_KEEP = "buildout.cfg", "extends-cache", "download-cache"
+NOPART = "chrpath flex glib lunzip m4 patch perl popt site_perl xz-utils".split()
+TARGET = "opt/re6st"
+
+ROOT = "build"
+BUILD = ROOT + "/" + TARGET
+DIST = "dist"
+OSC = "osc" # usually a symlink to the destination osc folder
+
+re6stnet = git("re6stnet", "http://git.erp5.org/repos/re6stnet.git",
+               "docs".__eq__)
+slapos = git("slapos", "http://git.erp5.org/repos/slapos.git",
+             ctime=False) # ignore ctime due to hardlinks to *-cache
+
+os.environ["TZ"] = "UTC"; time.tzset()
+
+@task("buildout.cfg.in", BUILD + "/buildout.cfg")
+def cfg(task):
+    cfg = open(task.input).read() % dict(
+        SLAPOS=os.path.abspath("slapos"),
+        ROOT="${buildout:directory}/" + os.path.relpath(ROOT, BUILD),
+        TARGET="/"+TARGET)
+    mkdir(BUILD)
+    open(task.output, "w").write(cfg)
+
+@task((cfg, slapos), (BUILD + "/bin/buildout", BUILD + "/bin/python"))
+def bootstrap(task):
+    try:
+        os.utime(task.outputs[1], None)
+    except OSError:
+        bootstrap = urllib.urlopen(BOOTSTRAP_URL).read()
+        mkdir(BUILD + "/download-cache")
+        with cwd(BUILD):
+            rmtree("extends-cache")
+            os.mkdir("extends-cache")
+            check_output((sys.executable, "-S"), input=bootstrap)
+            check_call(("bin/buildout", "buildout:parts=python"))
+
+def sdist_version(egg):
+    global MTIME, VERSION
+    MTIME = os.stat(egg).st_mtime
+    VERSION = "%s+slapos%s.g%s" % (
+        egg.rsplit("-", 1)[1].split(".tar.")[0],
+        os.getenv("SLAPOS_EPOCH", ""),
+        check_output(("git", "rev-parse", "--short", "HEAD"),
+                     cwd="slapos").strip())
+    tarball.provides = "%s/%s_%s.tar.gz" % (DIST, PACKAGE, VERSION),
+    deb.provides = deb.provides[0], "%s/%s_%s.dsc" % (DIST, PACKAGE, VERSION)
+    mkdir(DIST)
+    return egg
+
+def sdist(task):
+    o = glob(BUILD + "/download-cache/dist/re6stnet-*")
+    try:
+        return sdist_version(*o),
+    except TypeError:
+        return None,
+
+@task((bootstrap, re6stnet), ("re6stnet/re6stnet.egg-info", sdist))
+def sdist(task):
+    # XXX: We'd like to produce a reproducible tarball, so that 'make_tar_gz'
+    #      is really useful for the main tarball.
+    d = BUILD + "/download-cache/dist"
+    g = d + "/re6stnet-*"
+    map(os.remove, glob(g))
+    check_call((os.path.abspath(task.inputs[1]), "setup.py", "sdist",
+                "-d", os.path.abspath(d)), cwd="re6stnet")
+    task.outputs[1] = sdist_version(*glob(g))
+    # Touch target because the current directory is used as temporary
+    # storage, and it is cleaned up after that setup.py runs egg_info.
+    os.utime(task.outputs[0], None)
+
+@task(sdist, BUILD + "/.installed.cfg")
+def buildout(task):
+    check_call(("bin/buildout",), cwd=BUILD)
+    # Touch target in case that buildout had nothing to do.
+    os.utime(task.output, None)
+
+def tarfile_addfileobj(tarobj, name, dataobj, statobj):
+    tarinfo = tarobj.gettarinfo(arcname=name, fileobj=statobj)
+    dataobj.seek(0, 2)
+    tarinfo.size = dataobj.tell()
+    dataobj.reset()
+    tarobj.addfile(tarinfo, dataobj)
+
+@task(re6stnet)
+def upstream(task):
+    check_call(("make", "-C", "re6stnet"))
+    task.outputs = glob("re6stnet/docs/*.[1-9]")
+
+@task((upstream, buildout, __file__,
+       "Makefile.in", "cleanup", "install-eggs", "rebootstrap"))
+def tarball(task):
+    prefix = "%s-%s/" % (PACKAGE, VERSION)
+    def xform(path):
+        for p in "re6stnet/", "build/", "":
+            if path.startswith(p):
+                return prefix + path[len(p):]
+    with make_tar_gz(task.output, MTIME, xform) as t:
+        s = StringIO()
+        for k in "BIN", "NOPART", "BUILD_KEEP", "TARGET":
+            v = globals()[k]
+            s.write("%s = %s\n" % (k, v if type(v) is str else " ".join(v)))
+        with open(task.inputs[-4]) as x:
+            s.write(x.read())
+            tarfile_addfileobj(t, "Makefile", s, x)
+        s.truncate(0)
+        s.write("override PYTHON = /%s/parts/python2.7/bin/python\n" % TARGET)
+        with open("re6stnet/Makefile") as x:
+            s.write(x.read())
+            tarfile_addfileobj(t, "upstream.mk", s, x)
+        for x in task.inputs[-3:]:
+            t.add(x)
+        t.add("re6stnet/daemon")
+        for x in upstream.outputs:
+            t.add(x)
+        for x in BUILD_KEEP:
+            t.add(BUILD + "/" + x)
+
+@task(sdist, "debian/changelog")
+def dch(task):
+    with cwd("re6stnet") as p:
+        p += "/" + task.output
+        check_output(("make", "-f", "-", p,
+                      "PACKAGE=" + PACKAGE, "VERSION=" + VERSION),
+            input=open("debian/common.mk").read().replace(task.output, p))
+
+@task((dch, tree("debian")), DIST + "/debian.tar.gz")
+def deb(task):
+    control = open("re6stnet/debian/control")
+    d = Deb822(); s = Deb822(control); b = Deb822(control)
+    d["Format"] = open("debian/source/format").read().strip()
+    d["Source"] = s["Source"] = b["Package"] = PACKAGE
+    d["Version"] = VERSION
+    d["Architecture"] = b["Architecture"] = "any"
+    d["Build-Depends"] = s["Build-Depends"] = \
+        "python (>= 2.6), debhelper (>= 8)"
+    b["Depends"] = "${shlibs:Depends}, iproute2 | iproute"
+    b["Conflicts"] = b["Provides"] = b["Replaces"] = "re6stnet"
+    patched_control = StringIO(str("%s\n%s" % (s, b))) # BBB: cast to str for Python 2.6
+    open(task.outputs[1], "w").write(str(d))
+    date = rfc822.parsedate_tz(Changelog(open(dch.output)).date)
+    mtime = time.mktime(date[:9]) - date[9]
+    # Unfortunately, OBS does not support symlinks.
+    with make_tar_gz(task.outputs[0], mtime, dereference=True) as t:
+        added = glob("debian/*")
+        t.add("debian")
+        x = "debian/control"
+        tarfile_addfileobj(t, x, patched_control, control)
+        added.append(x)
+        with cwd("re6stnet"):
+            upstream = set(glob("debian/*"))
+            upstream.difference_update((x, "debian/rules", "debian/source"))
+            # check we are aware of any upstream file we override
+            assert upstream.isdisjoint(added), upstream.intersection(added)
+            map(t.add, sorted(upstream))
+
+@task((sdist, __file__), DIST + "/re6stnet.spec")
+def rpm(task):
+    check_call(("sed", "-r", r"""
+# https://fedoraproject.org/wiki/Packaging:Python_Appendix#Manual_byte_compilation
+1i%%global __os_install_post %%(echo '%%{__os_install_post}' |grep -v brp-python-bytecompile)
+/^%%define (_builddir|ver)/d
+s/^(Name:\s*).*/\1%s/
+s/^(Version:\s*).*/\1%s/
+s/^(Release:\s*).*/\11/
+/^BuildArch:/cAutoReqProv: no\
+BuildRequires: gcc-c++, make, python\
+#!BuildIgnore: rpmlint-Factory\
+Source: %%{name}_%%{version}.tar.gz
+/^Requires:/{
+    /iproute/!d
+}
+/^Recommends:/d
+s/^(Conflicts:\s*).*/\1re6stnet/
+/^%%description$/a%%prep\n%%setup -q
+/^%%preun$/,/^$/{
+    /^$/ifind /%s -type f -name '*.py[co]' -delete
+}
+""" % (PACKAGE, VERSION, TARGET), "re6stnet/re6stnet.spec"),
+    stdout=open(task.output, "w"))
+
+@task((sdist, "PKGBUILD.in"), DIST + "/PKGBUILD")
+def arch(task):
+    pkgbuild = open(task.inputs[-1]).read().replace("%VERSION%", VERSION)
+    open(task.output, "w").write(pkgbuild)
+
+@task((tarball, deb, rpm, arch, "re6stnet.install"))
+def build(task):
+    pass
+
+@task(build)
+def osc(task):
+    check_call(("osc", "up"), cwd=OSC)
+    old = set(glob(OSC + "/re6st-node_*"))
+    for path in build.inputs:
+        shutil.copy2(path, OSC)
+        old.discard(OSC + "/" + os.path.basename(path))
+    for path in old:
+        os.remove(path)
+    check_call(("osc", "addremove"), cwd=OSC)