Merge remote-tracking branch 'upstream/master' into jupyter_add_jupyterlab

component/file/buildout.cfg

@@ -8,7 +8,7 @@ extends =
 
 [file]
 recipe = slapos.recipe.cmmi
-url = ftp://ftp.astron.com/pub/file/file-5.23.tar.gz
+url = http://ftp.icm.edu.pl/packages/file/file-5.23.tar.gz
 md5sum = 61db35209ce71a6d576392ce6e1d2f80
 configure-options =
   --disable-static

component/percona-toolkit/buildout.cfg

@@ -8,9 +8,9 @@ parts =
 
 [percona-toolkit]
 recipe = slapos.recipe.cmmi
-version = 2.2.15
-url = http://www.percona.com/redir/downloads/percona-toolkit/${:version}/tarball/percona-toolkit-${:version}.tar.gz
-md5sum = 022f40dadaea9025820530ea1f986192
+version = 3.0.3
+url = https://www.percona.com/downloads/percona-toolkit/${:version}/source/tarball/percona-toolkit-${:version}.tar.gz
+md5sum = 8af181994fdf9aa984475637861098e9
 depends =
   ${perl-DBI:location}
   ${perl-DBD-mariadb:location}

software/jupyter/ERP5kernel.py.jinja → software/jupyter/ERP5kernel.py

-#!{{ python_executable }}
-
 from ipykernel.kernelbase import Kernel
 from ipykernel.kernelapp import IPKernelApp
 from IPython.core.display import HTML
 import requests
 import json
+import sys
 
-# erp5_url from buildout
-erp5_url = "{{ erp5_url }}"
-if not erp5_url:
-    erp5_url = None
-else:
-    erp5_url = "%s/erp5/Base_executeJupyter" % erp5_url
+erp5_url = None
+if len(sys.argv) > 1:
+    erp5_url = "%s/erp5/Base_executeJupyter" % (sys.argv[1],)
 
 class MagicInfo:
   """

software/jupyter/buildout.hash.cfg

@@ -16,7 +16,7 @@
 
 [instance-jupyter-notebook]
 filename = instance.cfg.in
-md5sum = fe5f82427cad1a24c2396c1684b8c411
+md5sum = 9b611c1bf176ac3d088b6387b5591678
 
 [jupyter-notebook-config]
 filename = jupyter_notebook_config.py.jinja
@@ -27,13 +27,9 @@ filename = jupyter_set_password.cgi.jinja
 md5sum = b8d31441780b524a7e52d1710dd78385
 
 [erp5-kernel]
-filename = ERP5kernel.py.jinja
-md5sum = 8bd16cc9f35b00172e8266f1cde5956f
+filename = ERP5kernel.py
+md5sum = 7d5309fe79afbcb455c0d8181b42e56c
 
 [kernel-json]
 filename = kernel.json.jinja
-md5sum = ab6e78ea20855e07d388b5b86d1770fe
-
-[custom-js]
-filename = custom.js.jinja
-md5sum = 0bf9e2eb1718b14307265fe05a167018
+md5sum = 33547be93a67530165e079dc3ecfdac3

software/jupyter/instance.cfg.in

@@ -138,13 +138,9 @@ recipe = slapos.cookbook:publish.serialised
 url = https://[${instance-parameter:host}]:${instance-parameter:port}
 
 [erp5-kernel]
-<= dynamic-jinja2-template-base
-template = {{ erp5_kernel_location }}/{{ erp5_kernel_filename }}
-rendered = ${directory:erp5_kernel_dir}/ERP5kernel.py
-# Use ipython as executable python as we'll be needing requests library in kernel
-context =
-  raw python_executable {{ bin_directory }}/ipython
-  key erp5_url slapconfiguration:configuration.erp5-url
+recipe = slapos.cookbook:symbolic.link
+link-binary = {{ erp5_kernel_location }}/{{ erp5_kernel_filename }}
+target-directory = ${directory:erp5_kernel_dir}
 
 [kernel-json]
 <= dynamic-jinja2-template-base
@@ -153,12 +149,13 @@ rendered = ${directory:erp5_kernel_dir}/kernel.json
 # Use python2.7 executable bin file for kernel config
 context =
   raw python_executable {{ python_executable }}
-  key kernel_dir erp5-kernel:rendered
+  raw kernel_dir ${erp5-kernel:target-directory}/{{ erp5_kernel_filename }}
+  key erp5_url slapconfiguration:configuration.erp5-url 
   raw display_name ERP5
   raw language_name python
 
 [custom-js]
-<= dynamic-jinja2-template-base
-template = {{ custom_js_location }}/{{ custom_js_filename }}
-rendered = ${directory:jupyter_custom_dir}/custom.js
-mode = 0744
+recipe = slapos.cookbook:symbolic.link
+target-directory = ${directory:jupyter_custom_dir}
+link-binary = {{ custom_js_location }}/custom.js
+

software/jupyter/kernel.json.jinja

@@ -2,6 +2,7 @@
   "argv": [
     "{{python_executable}}",
     "{{kernel_dir}}",
+    "{{erp5_url}}",
     "-f",
     "{connection_file}"
   ],

software/nayuos/README.md

 # NayuOS
 
-This is a SlapOS recipe to build NayuOS. It needs to be put in the <code>/srv/slapgrid/\<part\>/srv/runner/project/slapos/software/</code> directory. The created directory is called <code>\<nayuos_build_dirname\></code> in this documentation.
+This is a SlapOS recipe to build NayuOS.
 
 ## License
 
@@ -8,42 +8,61 @@ GPL v2 or later
 
 ## Requirements
 
-*   sudo on the host (for now)
-*   some environment variables need to be authorized to be propagated when cros_sdk calls sudo:
+*   sudo on the host
+*   environment variables need to be authorized to be propagated when cros_sdk calls sudo: `: Defaults        env_keep += "CROS_CACHEDIR DEPOT_TOOLS"`
+*   slapuser with sudo rights to execute the cros_sdk scripts (needed to access the chroot environment provided by Chromium OS)
+in /etc/sudoers (replace slapuser9 by your user, and release-R48-7647.B by the release you have chosen): `: slapuser9    ALL=NOPASSWD: /srv/slapgrid/slappart9/srv/runner/instance/slappart0/parts/chromiumos/release-R48-7647.B/chromite/bin/cros_sdk, /srv/slapgrid/slappart9/srv/runner/instance/slappart0/wrapper_bin/wrapper_cros_sdk, /bin/kill`
 
-: Defaults        env_keep += "CROS_CACHEDIR DEPOT_TOOLS"
+It's useful to have the right to kill cros_sdk processes, when needed. ;)
 
-*   slapuser with sudo rights to execute the cros_sdk scripts (needed to access the chroot environment provided by Chromium OS)
-in /etc/sudoers (replace slapuser9 by your user, and release-R48-7647.B by the release you have chosen):
+## Technical notes
 
-: slapuser9    ALL=NOPASSWD: /srv/slapgrid/slappart9/srv/runner/instance/slappart0/parts/chromiumos/release-R48-7647.B/chromite/bin/cros_sdk, /srv/slapgrid/slappart9/srv/runner/instance/slappart0/wrapper_bin/wrapper_cros_sdk, /bin/kill
+After any change to the build process it is necessary to delete (using sudo)
+the building environment at `~/srv/runner/instance/slappart0/parts/chromiumos/<TAG>`.
+
+BEWARE that the web runner is serving images for the [official website](https://nayuos.nexedi.com).
+
+NayuOS and ChromiumOS is "just" a version of Gentoo. Thus it uses `ebuild` packages
+and anything installable in Gentoo can be installed to NayuOS too. Of course only
+during OS build phase and one has to count with limited space.
+
+### Upgrading (building new image)
+
+Please read **Requirements** section carefully. After selecting your desired 
+`release` from the list <https://chromium.googlesource.com/chromiumos/manifest/+refs>
+it is **necessary** to add `sudo` rules for that release as shown there.
 
-It's useful to have the right to kill cros_sdk processes, when needed. ;)
 
 ## Input
+
 In the vifib parameters (softinst\<nb\>.host.vifib.net \> Services \> Parameters):
 
-*   board / ex: peppy, swanky, ... (choosing daisy will accept all licenses for the daisy board build only, in order to use Mali drivers, see [chromium mailing list](https://groups.google.com/a/chromium.org/forum/#!topic/chromium-os-dev/Pf9ZG2itxWM))
-*   branch / ex: release-R46-7390.B (you can find the release in the [Chromium OS source tree](https://chromium.googlesource.com/chromiumos/manifest/+refs))
-*   keep_cache / yes|no (choosing "no" saves about 15Go of disk space per board, choosing "yes" will makes next build faster and less expensive in term of needed ressources because of not rebuilding everything)
+*   **board** / ex: peppy, swanky, ... (choosing daisy will accept all licenses for the daisy board build only, in order to use Mali drivers, see [chromium mailing list](https://groups.google.com/a/chromium.org/forum/#!topic/chromium-os-dev/Pf9ZG2itxWM))
+*   **branch** / ex: release-R46-7390.B (you can find the release in the [Chromium OS source tree](https://chromium.googlesource.com/chromiumos/manifest/+refs))
+*   **keep_cache** / yes|no (choosing "no" saves about 15Go of disk space per board, choosing "yes" will makes next build faster and less expensive in term of needed ressources because of not rebuilding everything)
 
 
 ## Output
-The image will be produced in:
-<code>/srv/slapgrid/\<part\>/srv/runner/instance/\<inst_part\>/parts/chromiumos/images/</code>
-and the logs are in:
-<code>/srv/slapgrid/\<part\>/srv/runner/instance/\<inst_part\>/var/log/cros_sources_dl.log</code> and <code>/srv/slapgrid/\<part\>/srv/runner/instance/\<inst_part\>/var/log/cros_build.log</code>
 
-The script that download the sources and build is located in
-<code>/srv/slapgrid/\<part\>/srv/runner/instance/\<inst_part\>/etc/run</code>
+Software release produces a build script `<instance_partition>/etc/run/cros_full_build`.
+
+Build produces
+
+*  Image: `<instance_partition>/parts/chromiumos/images/`
+*  Compilation logs: `<instance_partition>/var/log/cros_sources_dl.log`
+*  Build logs: `<instance_partition>/var/log/cros_build.log`
+
 
 ## External documents
 
-*   [ <code>repo</code> command reference ](https://source.android.com/source/using-repo.html)
-*   [ NayuOS official website ](https://www.nayuos.com)
+*   [ `repo` command reference ](https://source.android.com/source/using-repo.html)
+*   [ NayuOS official website ](https://nayuos.nexedi.com)
 *   [ crouton for chroot ](https://github.com/dnschneid/crouton) ([warning about verified boot](https://github.com/dnschneid/crouton/blob/2a1fc9da380650f47e2bcf37d00962bfb68c4830/installer/main.sh#L517-L536))
 
 ## Notes for possible improvements
+
 *   [ Running virtual machines on your chromebook ](https://www.chromium.org/chromium-os/developer-information-for-chrome-os-devices/running-virtual-machines-on-your-chromebook)
 *   to have a more common User Agent (the one of ChromiumOS/NayuOS is quite rare and identifies the user, see [studies of the EFF](https://panopticlick.eff.org/static/browser-uniqueness.pdf)), it seems possible to change the User-Agent flag for guest mode in the getOffTheRecord function, and adding a line (key "kUserAgent" , value "some common user agent" string). Then rebuild Chromium and [add it to NayuOS](https://www.chromium.org/chromium-os/developer-guide#TOC-Making-changes-to-the-Chromium-web-).
 *   remove need of root priviledge for entering the chroot, maybe by using fakeroot in 'scripts/wrapper_sudo.in'?
+*   change more options on Chromium OS "Privacy" part by default: there are [a few options](https://support.google.com/chromebook/answer/114836) which still use Google services
+*   provide ChromiumOS package manager [ chromebrew ](https://skycocker.github.io/chromebrew/) by default

software/nayuos/instance.cfg

+# Jinja2 template of a buildout file to create runnable scripts
+#
+# Received variables
+# -  software_dir: location where this repository was cloned to
+# -  instance_dir: location where this 'instance.cfg' is gettin rendered (same as ${buildout:directory})
+
 [buildout]
 parts =
   parameters
+  promise-sudo-on-host
   template-full-build-script
   template-sudo-wrapper
   template-cros-sdk-wrapper
 
-# eggs given by software.cfg
+# standard declaration of eggs directories
 eggs-directory = {{ eggs_directory }}
 develop-eggs-directory = {{ develop_eggs_directory }}
 
@@ -16,15 +23,18 @@ develop-eggs-directory = {{ develop_eggs_directory }}
 recipe = slapos.recipe.build:gitclone
 repository = https://chromium.googlesource.com/chromium/tools/depot_tools.git
 branch = master
+git-executable = {{ git_path }}/bin/git
 
 [customize-path]
 # add depot tools directory (for cros_sdk binary among others) and git directory to the path
 command =
-  export PATH="${directory:wrapper_dir}":"{{ git_path }}":"{{ curl_path }}":"${depot-tools:location}":"$PATH";
+  export PATH="${directory:wrapper_dir}":"{{ git_path }}/bin":"{{ curl_path }}/bin":"${depot-tools:location}":"$PATH";
 
 [nayuos-ebuilds]
 recipe = slapos.recipe.build:gitclone
 repository = https://lab.nexedi.com/nexedi/nayuos-ebuilds.git
+branch = master
+git-executable = {{ git_path }}/bin/git
 
 ############################################################################################
 
@@ -44,20 +54,25 @@ cert = ${slap_connection:cert_file}
 recipe = slapos.cookbook:mkdirectory
 log = ${buildout:directory}/var/log
 run = ${buildout:directory}/etc/run
+promise = ${buildout:directory}/etc/promise
 wrapper_dir = ${buildout:directory}/wrapper_bin
 cros_location = ${buildout:directory}/parts/chromiumos
-ebuilds_dir = ${nayuos-ebuilds:location}
-scripts_dir = {{ scripts_dir }}
-logo_dir = {{ logo_dir }}
 
 [bin]
+# dummy section to hold references to frequently used binaries
 wrapper_cros_sdk=${directory:wrapper_dir}/wrapper_cros_sdk
 bash=/bin/bash
 sudo=/usr/bin/sudo
 
+[promise-sudo-on-host]
+# assert sudo is installed, as it is required to enter the chroot 'cros_sdk'
+recipe  = slapos.cookbook:wrapper
+wrapper-path = ${directory:promise}/${:_buildout_section_name_}
+command-line = sudo -V
+
 [template-sudo-wrapper]
 recipe = slapos.recipe.template:jinja2
-template = {{ scripts_dir }}/wrapper_sudo.in
+template = {{ software_dir }}/scripts/wrapper_sudo.in
 rendered = ${directory:wrapper_dir}/sudo
 md5sum = ded5a92be4e37ec32eb9d3087d3e19bd
 mode = 0700
@@ -68,23 +83,23 @@ context =
 
 [template-cros-sdk-wrapper]
 recipe = slapos.recipe.template:jinja2
-template = {{ scripts_dir }}/wrapper_cros_sdk.in
+template = {{ software_dir }}/scripts/wrapper_cros_sdk.in
 rendered = ${bin:wrapper_cros_sdk}
 md5sum = 7159fe3d5b85a283733cf686c4ee0a74
 mode = 0700
 context =
 # XXX bash path is the one from the host
   key bash_path bin:bash
-  raw git_path {{ git_path }}
-  raw curl_path {{ curl_path }}
+  raw git_path  {{ git_path }}/bin
+  raw curl_path {{ curl_path }}/bin
 
 [template-full-build-script]
 # create the rendered script in the buildout-directory/etc/run
 # (use jinja for templating)
 recipe = slapos.recipe.template:jinja2
-template = {{ scripts_dir }}/cros_full_build.in
+template = {{ software_dir }}/scripts/cros_full_build.in
 rendered = ${directory:run}/cros_full_build
-md5sum = 75599e6b8418a5f3756c7c7b26600399
+md5sum = 2bb9dd83260ea96dd6a6602f6faa9794
 mode = 0700
 context =
   key bash_path                         bin:bash
@@ -94,7 +109,11 @@ context =
   key branch                            parameters:configuration.branch
   key boards_list                       parameters:configuration.boards
   key keep_cache                        parameters:configuration.keep_cache
-  key ebuilds_dir directory:ebuilds_dir
-  key scripts_dir directory:scripts_dir
-  key logo_dir directory:logo_dir
-  raw nayu_dev_packages net-libs/nodejs net-misc/re6stnet dev-vcs/git dev-python/flask dev-python/virtualenv sys-fs/cryptsetup
+  key ebuilds_dir                       nayuos-ebuilds:location
+  raw scripts_dir                       {{ software_dir }}/scripts
+  raw logo_dir                          {{ software_dir }}/logo
+# packages to be copied from our overlay into ChromiuOS' overlay
+# those packages will be marked as dependency of root filesystem
+  raw nayu_dev_rootfs_packages          app-misc/nayuos-chromium-policy
+# those packages will be marked as dependency of the chromium build itself
+  raw nayu_dev_packages                 net-misc/re6stnet dev-vcs/git dev-python/flask dev-python/virtualenv sys-fs/cryptsetup

software/nayuos/scripts/cros_full_build.in

 #! {{ bash_path }}
 
+function increase_revision_number() {
+  # this increase the revision number an ebuild
+  # (except for revision number 9999 which is a value for "live ebuild", for dev revisions)
+  # see https://devmanual.gentoo.org/general-concepts/ebuild-revisions/
+
+  if [ -z "$1" ] ; then
+    echo "ERROR: No parameter given to function: increase_revision_number."
+    echo "Usage: increase_revision_number /path/to/dir/containing/ebuild"
+    exit 1
+  elif [ -d "$1" ]; then
+    DIR="$1"
+    CURRENT_FILE=""
+    CURRENT_REVISION=0
+    # get the biggest revision number
+    # XXX: may break if there are many version of an ebuild
+    # (eg: bash-4.3_p42-r4.ebuild and bash-3.7-r8.ebuild)
+    for FILENAME in $(find "${DIR}" -maxdepth 1 -regextype sed -regex .*-r[0-9]*\.ebuild); do
+      echo $FILENAME
+      REVISION="$(echo ${FILENAME} | rev | cut -d- -f1 | rev | tr -d [:alpha:] | tr -d [:punct:])"
+      echo "${REVISION}"
+      if [ "$CURRENT_REVISION" -lt "$REVISION" ] ; then
+          CURRENT_REVISION=${REVISION}
+          CURRENT_FILE=${FILENAME}
+      fi
+    done
+    if [ "$CURRENT_REVISION" -ne "9999" ]; then
+      # increase REVISION
+      NEW_REVISION=$((CURRENT_REVISION+1))
+      NEW_FILE="$(echo ${CURRENT_FILE} | sed -r 's/-r'${CURRENT_REVISION}'/-r'${NEW_REVISION}'/')"
+      echo "changing revision number: ${CURRENT_FILE} -> ${NEW_FILE}"
+      mv "${CURRENT_FILE}" "${NEW_FILE}"
+    fi
+  fi
+}
+
+function insert_packages_into_ebuild() {
+  # Insert packages (in form of "category/package") into RDEPEND part of ebuild.
+  # Optionally increasing version of given ebuild.
+  EBUILD=$1
+  changed=0
+  shift
+  while [ ! -z "$1" ]
+  do
+    package=$1
+    if [[ $( grep "${package}" "${EBUILD}" ) ]] ; then
+      echo "no need to change ${EBUILD} file to add ${package}..." >> "${BUILD_LOG}"
+    else
+      printf "\n\nRDEPEND=\"\${RDEPEND}\n          ${package}\"\n">> ${EBUILD}
+      changed=1
+    fi
+    shift
+  done;
+
+  if [ $changed -eq 1 ]
+  then
+    # if we inserted at least one package then increase the version to be sure
+    # it gets selected in case of more overlays with the same package
+    increase_revision_number $(dirname $EBUILD)
+  fi
+}
+
 ######################## Download of sources using the "repo" command ########################
 DL_LOG="{{ instance_log_dir }}/cros_sources_dl.log"
 BRANCH="{{ branch }}"
 CHROMIUM_OVERLAY="{{ cros_location }}/{{ branch }}/src/third_party/chromiumos-overlay"
+PORTAGE_STABLE="{{ cros_location }}/{{ branch }}/src/third_party/portage-stable"
 
 {{ export_path_cmd }}
 echo "getting Chromium OS sources..." >> $DL_LOG
@@ -22,6 +84,8 @@ repo init -u https://chromium.googlesource.com/chromiumos/manifest.git -b {{ bra
 # in case changes were already made, stash changes to be able to pull
 cd ${CHROMIUM_OVERLAY}
 git stash
+cd ${PORTAGE_STABLE}
+git stash
 
 repo sync >> $DL_LOG || { echo "Problem while downloading Chromium OS sources (repo sync). Exiting." ; exit 1 ; }
 
@@ -38,10 +102,10 @@ cros_sdk --download || { echo "Problem while entering chroot or downloading chro
 # it is needed by re6st
 # just need to change -sqlite by sqlite on the right line
 # same for IPv6 support
-PACKAGE_USE_FILE=${CHROMIUM_OVERLAY}/profiles/targets/chromeos/package.use
+PACKAGE_USE_FILE="${CHROMIUM_OVERLAY}/profiles/targets/chromeos/package.use"
 line_number=$(sed -n '/dev-lang\/python/=' ${PACKAGE_USE_FILE})
 if [ $line_number ] ; then
-    echo $line_number
+    echo "$line_number"
     sed -i "${line_number}s/-sqlite/sqlite/" ${PACKAGE_USE_FILE}
     if [[ ! $(sed -n ${line_number}p ${PACKAGE_USE_FILE} | grep ipv6) ]]; then
         sed -i "${line_number}s/$/ ipv6/" ${PACKAGE_USE_FILE}
@@ -51,57 +115,167 @@ fi
 # add some ebuilds, specific to NayuOS
 for category in $( ls {{ ebuilds_dir }} ); do
   echo ${category}
-  cp -R {{ ebuilds_dir }}/${category}/* ${CHROMIUM_OVERLAY}/${category}/
+  cp -R "{{ ebuilds_dir }}/${category}/"* "${CHROMIUM_OVERLAY}/${category}/"
 done
 
 install -m 770 "{{ scripts_dir }}/test_nayuos_image" "{{ cros_location }}/{{ branch }}/src/scripts/test_nayuos_image"
 
 ### packages management ###
 
-BASE_CHROMEOS_DEV_ROOT_EBUILD=${CHROMIUM_OVERLAY}/chromeos-base/chromeos-dev-root/chromeos-dev-root-0.0.1.ebuild
-VIRTUAL_CHROMEOS_OS_DEV_EBUILD=${CHROMIUM_OVERLAY}/virtual/target-chromium-os-dev/target-chromium-os-dev-1.ebuild
+# make virtual ebuild responsible for installing all packages dependent on NayuOS own packages
+insert_packages_into_ebuild \
+  ${CHROMIUM_OVERLAY}/virtual/target-chromium-os-dev/target-chromium-os-dev-1.ebuild \
+  {{ nayu_dev_packages }}
+
+# insert policies into /etc/chromium/policies/recommended with NayuOS customizations
+insert_packages_into_ebuild \
+  ${CHROMIUM_OVERLAY}/chromeos-base/chromeos-dev-root/chromeos-dev-root-0.0.1.ebuild \
+  {{ nayu_dev_rootfs_packages }}
 
-# change the virtual ebuild responsible for installing all packages to add the ones
-# needed for NayuOS
-for package in {{ nayu_dev_packages }} ; do
-  echo $package
-  if [[ $( grep "${package}" "${VIRTUAL_CHROMEOS_OS_DEV_EBUILD}" ) ]] ; then
-    echo "no need to change ${VIRTUAL_CHROMEOS_OS_DEV_EBUILD} file to add ${package}..." >> "${BUILD_LOG}"
-  else
-    printf "\n\nRDEPEND=\"\${RDEPEND}\n          ${package}\"\n">> ${VIRTUAL_CHROMEOS_OS_DEV_EBUILD}
-  fi
-done
 
 # do not install the Upstart init script that starts ssh daemon at boot time
 rm ${CHROMIUM_OVERLAY}/chromeos-base/chromeos-sshd-init/files/openssh-server.conf
+# strip away the src_install part because it only sets up booting scripts
 sed -i -n '/src_install/q;p' ${CHROMIUM_OVERLAY}/chromeos-base/openssh-server-init/openssh-server-init-0.0.1.ebuild
 sed -i -n '/src_install/q;p' ${CHROMIUM_OVERLAY}/chromeos-base/chromeos-sshd-init/chromeos-sshd-init-0.0.1.ebuild
+# increase version in case of other overlays with the same packages
+increase_revision_number ${CHROMIUM_OVERLAY}/chromeos-base/openssh-server-init/
+increase_revision_number ${CHROMIUM_OVERLAY}/chromeos-base/chromeos-sshd-init/
 
 # bashrc modifications
-BASH_EBUILD_DIR={{ cros_location }}/{{ branch }}/src/third_party/portage-stable/app-shells/bash
-if [[ $(grep "git --exec-path" "${BASH_EBUILD_DIR}/files/dot-bashrc") ]] ; then
-  echo "git alias already set"
-else
-  echo "# git quickfix for finding right git executables
-if [ -d /usr/local/libexec/git-core/ ] ; then
-    alias git='git --exec-path=/usr/local/libexec/git-core/'
+BASH_EBUILD_DIR="${PORTAGE_STABLE}/app-shells/bash"
+if ! grep "BEGIN NayuOS configuration" "${BASH_EBUILD_DIR}/files/dot-bashrc" > /dev/null ; then
+  cat >> "${BASH_EBUILD_DIR}/files/dot-bashrc" <<EOF
+
+# ----- BEGIN NayuOS configuration -----
+
+# use vim as default editor if nano does not exist
+which nano &> /dev/null || export EDITOR=vim
+
+# git quickfix for finding right git executables
+export GIT_EXEC_PATH=/usr/local/libexec/git-core
+
+# configure .gitconfig once
+if [ ! -e ~/.gitconfig ] ; then
+  which less &> /dev/null && git config --global core.pager less
 fi
 
-# git quickfix for finding 'less' and using it as pager
-less_path=\$(which less 2>/dev/null)
-if [ "\${less_path}" ] ; then
-    git config --global core.pager \${less_path}
-fi" >> ${BASH_EBUILD_DIR}/files/dot-bashrc
+# go to ~/Download
+[ "\$(pwd)" = / ] && cd ~/Downloads
+
+alias ls='ls -CF --color=auto'
+alias l='ls -lF'
+alias lh='ls -hAl'
+alias la='ls -la'
+alias ll='ls -l'
+
+tree() {
+  python -Sc 'import sys, os
+aa = sys.argv[1:] or ["."]
+for a in aa:
+  o = len(a.split(os.sep))
+  for r, dd, ff in os.walk(a):
+    dp = len(r.split(os.sep)) - o + 1
+    print "%s- %s" % ((dp - 1) * "  ", os.path.basename(r))
+    for f in ff:
+      print "%s- %s" % (dp * "  ", f)' "\$@"
+}
+
+showzip() {
+  [ \$# = 1 ] || { echo "Usage: showzip ZIPFILE" >&2 ; return 1 ; }
+  python -Sc 'import sys, zipfile
+zfile = zipfile.ZipFile(sys.argv[1])
+zfile.printdir()
+zfile.close()' "\$@"
+}
+
+zip() {
+  [ \$# -gt 1 ] || { echo "Usage: zip ZIPFILE FILE..." >&2 ; return 1 ; }
+  python -Sc 'import sys, zipfile, os
+if os.path.lexists(sys.argv[1]):
+  sys.stderr.write("zip: file %r already exists\n" % sys.argv[1])
+  sys.exit(1)
+zfile = zipfile.ZipFile(sys.argv[1], "w", zipfile.ZIP_DEFLATED)
+def arc(arg):
+  if os.path.islink(arg):
+    sys.stderr.write("zip: symlink %r ignored\n" % arg)
+  elif os.path.isdir(arg):
+    for r, _, ff in os.walk(arg):
+      zfile.write(r)
+      for f in ff: arc(os.path.join(r, f))
+  else:
+    zfile.write(arg)
+for arg in sys.argv[2:]: arc(arg)
+zfile.close()' "\$@"
+}
+
+unzip() {
+  [ \$# != 0 ] || { echo "Usage: unzip ZIPFILE [TARGET]" >&2 ; return 1 ; }
+  python -Sc 'import sys, zipfile
+zfile = zipfile.ZipFile(sys.argv[1])
+zfile.extractall(sys.argv[2] if len(sys.argv) > 2 else ".")
+zfile.close()' "\$@"
+}
+
+encmount() {
+  if [ -z \$1 ] ; then
+      echo "Please, give a device as parameter, ex: /dev/sda"
+      return 1
+  fi
+  if  [ ! -d /media/removable/key ] ; then
+      echo "New dir at /media/removable/key"
+      sudo mkdir /media/removable/key
+  fi
+  echo "Mounting \$1 on /media/removable/key"
+  sudo cryptsetup luksOpen \$1 key
+  sudo mount -o noexec /dev/mapper/key /media/removable/key
+}
+
+__safewrite() {
+  local cmd=safewrite
+  local usage="Usage: \$cmd DST < stdin
+
+Where DST is the destination file
+
+Copy stdin to DST. Data is flushed regularly to avoid some eventual crash during cache sync.
+
+Example:
+  gunzip myimage.img.gz -c | sudosafewrite /dev/sda"
+
+  local KiB=1024
+  local MiB=\$((1024 * \$KiB))
+  local bs=4096
+  local count=\$((50 * \$MiB / \$bs))
+
+  local infostep=50
+  local infounit=MiB
+
+  local safewrite_seek=0
+  local safewrite_out=
+  local safewrite_length=\$infostep
+
+  [ \$# = 2 ] || { echo "\$usage" >&2 ; return 2 ; }
+  [ \$1 != - ] || { echo "DST should not be '-'" >&2 ; return 2 ; }
+
+  while [ \$? = 0 ] ; do
+    echo "+ copying until \$safewrite_length \$infounit"
+    safewrite_out=\$(\$1 bs=\$bs count=\$count seek=\$safewrite_seek of="\$2" 2>&1) || { echo "\$safewrite_out" ; return 1 ; }
+    safewrite_length=\$((\$safewrite_length + \$infostep))
+    safewrite_seek=\$((\$safewrite_seek + \$count))
+    echo "\$safewrite_out"
+    ! echo "\$safewrite_out" | grep '(0 B) copied' > /dev/null
+  done
+  return 0
+}
+safewrite() { __safewrite dd "\$@" ; }
+sudodd() { sudo dd "\$@" ; }
+sudosafewrite() { __safewrite sudodd "\$@" ; }
+
+# ----- END NayuOS configuration -----
+EOF
 fi
 
-# increase the revision number
-# XXX: remove hardcoded revision numbers
-cd ${CHROMIUM_OVERLAY}/chromeos-base/openssh-server-init/
-ln -s openssh-server-init-0.0.1-r24.ebuild openssh-server-init-0.0.1-r25.ebuild
-cd ${CHROMIUM_OVERLAY}/chromeos-base/chromeos-sshd-init/
-ln -s chromeos-sshd-init-0.0.1-r5.ebuild chromeos-sshd-init-0.0.1-r6.ebuild
-cd ${BASH_EBUILD_DIR}
-ln -s bash-4.2_p48-r1.ebuild bash-4.2_p48-r2.ebuild
+increase_revision_number "${BASH_EBUILD_DIR}"
 
 ######################################## Build ##############################################
 BOARDS="{{ boards_list }}"
@@ -139,12 +313,14 @@ for board in ${BOARDS} ; do
   cros_sdk -- ./build_image --board=${board} dev >> "${BUILD_LOG}" \
     && cros_sdk -- rm -f $NAYU_IMAGE_LOCATION && cros_sdk -- touch $NAYU_IMAGE_LOCATION \
     && cros_sdk -- cros flash --board=${board} file://$NAYU_IMAGE_LOCATION >> "${BUILD_LOG}" \
+    && cros_sdk -- ./test_nayuos_image ${board} > "${TEST_LOG}" \
     || { echo "An error occured while building ${board} NayuOS image. Exiting." ; exit 1 ;}
-#     && cros_sdk -- ./test_nayuos_image ${board} > "${TEST_LOG}" \
 
-  # save ~15Go/device but delete cache (next build will be as long)
+  # save a lot of space on the server but delete cache and build files
+  # (it means that the next build will be as long and use as much resources as this one)
   if [ ${KEEP_CACHE,,} == "no" ] ; then
-    cros_sdk -- sudo rm -R /var/cache/chromeos-chrome/chrome-src/src/out_${board}
+    cros_sdk -- sudo rm -R "/var/cache/chromeos-chrome/chrome-src/src/out_${board}"
+    cros_sdk -- sudo rm -R "/build/${board}"
   fi
 
   if [ ${board} == daisy ]; then

software/nayuos/scripts/test_nayuos_image

@@ -14,8 +14,10 @@ ORIGINAL_GRANDENET_SCRIPT=~/trunk/src/third_party/chromiumos-overlay/net-misc/re
 GRANDENET_SCRIPT="usr/local/bin/grandenet"
 INIT_SSH_SERVER="etc/init/openssh-server.conf"
 BASHRC="etc/skel/.bashrc"
-EXPECTED_ALIAS="alias git='git --exec-path=/usr/local/libexec/git-core/'"
+GIT_CORE="usr/local/libexec/git-core/"
+GIT_EXPECTED_EXPORT="export GIT_EXEC_PATH=/usr/local/libexec/git-core"
 VIRTUALENV_BIN="usr/local/bin/virtualenv"
+CHROMIUM_POLICY="etc/chromium/policies/recommended/nayuos_policy.json"
 
 HAS_FAILED=0
 
@@ -32,14 +34,17 @@ function print_result() {
 
 # MOUNT IMAGE AND GET INFO
 
-install -d ${MOUNTPOINT}
+install -d "${MOUNTPOINT}"
 ./mount_gpt_image.sh --safe -f $( ./get_latest_image.sh --board=${BOARD} ) -r ${MOUNTPOINT}
 echo $(ls "${MOUNTPOINT}/usr/local")
 if [[ $(ls "${MOUNTPOINT}/usr/local") ]] ; then
-  my_diff=$(diff ${ORIGINAL_GRANDENET_SCRIPT} "${MOUNTPOINT}/${GRANDENET_SCRIPT}")
+  my_diff=$(diff "${ORIGINAL_GRANDENET_SCRIPT}" "${MOUNTPOINT}/${GRANDENET_SCRIPT}")
   opensshd_config=$(ls "${MOUNTPOINT}/${INIT_SSH_SERVER}")
-  gitalias=$(grep "${EXPECTED_ALIAS}" "${MOUNTPOINT}/${BASHRC}")
+  gitcore=$(ls "${MOUNTPOINT}/${GIT_CORE}")
+  gitexport=$(grep "${GIT_EXPECTED_EXPORT}" "${MOUNTPOINT}/${BASHRC}")
   virtualenvbin=$(ls "${MOUNTPOINT}/${VIRTUALENV_BIN}")
+  chromium_policy=$(ls "${MOUNTPOINT}/${CHROMIUM_POLICY}")
+
 else
   is_empty=1
 fi
@@ -50,7 +55,7 @@ rmdir ${MOUNTPOINT}
 # PRINT RESULTS
 
 echo "* test if /usr/local exists"
-if [[ ${is_empty} == 1 ]] ; then
+if [[ "${is_empty}" == "1" ]] ; then
   print_result ${FAILURE} "/usr/local is empty."
 else
   print_result ${SUCCESS} "/usr/local is not empty."
@@ -71,10 +76,18 @@ else
     print_result ${SUCCESS} "opensshd config removed."
   fi
 
-  echo "* test git quick fix for option --exec-path"
+  echo "* test if git core directory is not empty"
+
+  if [[ "${gitcore}" == "" ]] ; then
+    print_result ${FAILURE} "No git file in ${MOUNTPOINT}/${GIT_CORE}."
+  else
+    print_result ${SUCCESS} "git core directory exists and is not empty."
+  fi
+
+  echo "* test bashrc changes for git paths quick fix"
 
-  if [[ ${gitalias} == ""  ]] ; then
-    print_result ${FAILURE} "Expected alias for git command not in ${MOUNTPOINT}/${BASHRC}. Should be: ${EXPECTED_ALIAS}"
+  if [[ "${gitexport}" == ""  ]] ; then
+    print_result ${FAILURE} "Expected alias for git command not in ${MOUNTPOINT}/${BASHRC}. There should be: ${GIT_EXPECTED_EXPORT}"
   else
     print_result ${SUCCESS} "git alias is correct."
   fi
@@ -86,6 +99,14 @@ else
   else
     print_result ${FAILURE} "Expected virtualenv binary not in ${MOUNTPOINT}/${VIRTUALENV_BIN}."
   fi
+
+  echo "* test if Chromium policies are installed"
+
+  if [[ ${chromium_policy} ]] ; then
+    print_result ${SUCCESS} "Chromium policies are installed."
+  else
+    print_result ${FAILURE} "Expected Chromium policies file not in ${MOUNTPOINT}/${CHROMIUM_POLICY}."
+  fi
 fi
 
 exit ${HAS_FAILED}

software/nayuos/software.cfg

@@ -4,46 +4,30 @@ extends =
   ../../stack/slapos.cfg
 
 parts +=
-# use stack/slapos.cfg
   slapos-cookbook
-  verify-packages-exist-on-host
   template-instance
-  install-eggs-for-the-instance
+  git
+  curl
+
+versions = versions
+
 
 [versions]
 slapos.recipe.template = 3.0
 
-[verify-packages-exist-on-host]
-recipe = plone.recipe.command
-stop-on-error = true
-# run the same command when installing and on updates
-update-command = ${:command}
-command = sudo -V
 
 [template-instance]
-# create the instance.cfg file in the buildout directory
-# (use jinja for templating)
+# jinja2 render instance.cfg file in the buildout directory
 recipe = slapos.recipe.template:jinja2
 template = ${:_profile_base_location_}/instance.cfg
 rendered = ${buildout:directory}/instance.cfg
-md5sum = be6bd665c0b356a84090d200ea14b33c
 mode = 0644
-scripts_dir = ${:_profile_base_location_}/scripts
-logo_dir = ${:_profile_base_location_}/logo
-curl_path = ${curl:location}/bin
-git_path = ${git:location}/bin
+md5sum = 154882944c6a58642629c17d47b4f6fc
 context =
-# for access to the eggs from the instance
+  key software_dir            :_profile_base_location_
+  key instance_dir            buildout:directory
+  key curl_path               curl:location
+  key git_path                git:location
   key eggs_directory          buildout:eggs-directory
   key develop_eggs_directory  buildout:develop-eggs-directory
-  key scripts_dir :scripts_dir
-  key logo_dir :logo_dir
-  key curl_path :curl_path
-  key git_path :git_path
 
-[install-eggs-for-the-instance]
-# after installation of the eggs, the recipes
-# will be available (added to sys.path)
-recipe = zc.recipe.egg
-eggs = slapos.recipe.build
-       plone.recipe.command

stack/erp5/buildout.hash.cfg

@@ -83,7 +83,7 @@ md5sum = d41d8cd98f00b204e9800998ecf8427e
 
 [template-erp5]
 filename = instance-erp5.cfg.in
-md5sum = 64fe4800dc9430aac8e325ba4b3d97ae
+md5sum = 52ac560be07f254618631b84d2d58f41
 
 [template-zeo]
 filename = instance-zeo.cfg.in
@@ -95,7 +95,7 @@ md5sum = a2377d5c53fd2a441ea713b428e4844b
 
 [template-balancer]
 filename = instance-balancer.cfg.in
-md5sum = d2a339d0bc2f05f3f1f78defffae6ab6
+md5sum = 25d0ebecda78d1440dacd3495d53c928
 
 [template-haproxy-cfg]
 filename = haproxy.cfg.in

stack/erp5/instance-balancer.cfg.in

@@ -219,7 +219,7 @@ path = ${directory:promise}/apache
 hostname = {{ ipv4 }}
 port = {{ apache_dict.values()[0][0] }}
 
-[publish]
+[{{ section('publish') }}]
 recipe = slapos.cookbook:publish.serialised
 {% for family_name, (apache_port, scheme, _, _) in apache_dict.items() -%}
 {{   family_name ~ '-v6' }} = {% if ipv6_set %}{{ scheme ~ '://[' ~ ipv6 ~ ']:' ~ apache_port }}{% endif %}
@@ -260,7 +260,7 @@ cert =
 crl =
 {%- endif %}
 
-[logrotate-apache]
+[{{ section('logrotate-apache') }}]
 < = logrotate-entry-base
 name = apache
 log = ${apache-conf-parameter-dict:error-log} ${apache-conf-parameter-dict:access-log}
@@ -271,7 +271,7 @@ recipe = slapos.cookbook:mkdirectory
 apache-conf = ${:etc}/apache
 bin = ${buildout:directory}/bin
 etc = ${buildout:directory}/etc
-promise = ${directory:etc}/promise
+promise = ${:etc}/promise
 services = ${:etc}/run
 var = ${buildout:directory}/var
 run = ${:var}/run
@@ -284,7 +284,7 @@ newcerts = ${:ca-dir}/newcerts
 crl = ${:ca-dir}/crl
 apachedex = ${monitor-directory:private}/apachedex
 
-[monitor-generate-apachedex-report]
+[{{ section('monitor-generate-apachedex-report') }}]
 recipe = slapos.cookbook:wrapper
 wrapper-path = ${monitor-directory:reports}/${:command}
 command-line = "{{ parameter_dict['run-apachedex-location'] }}" "{{ parameter_dict['apachedex-location'] }}" "${directory:apachedex}" --default "${apachedex-parameters:default}" --apache-log-list "${apachedex-parameters:apache-log-list}" --base-list "${apachedex-parameters:base-list}" --skip-base-list "${apachedex-parameters:skip-base-list}" --erp5-base-list "${apachedex-parameters:erp5-base-list}"
@@ -316,7 +316,4 @@ extends =
   {{ logrotate_cfg }}
   {{ parameter_dict['template-monitor'] }}
 parts +=
-  publish
-  logrotate-apache
-  monitor-generate-apachedex-report
   {{ part_list | join('\n  ') }}

stack/erp5/instance-erp5.cfg.in

-{% import "root_common" as root_common with context %}
+{% import "root_common" as root_common with context -%}
 {% set frontend_dict = slapparameter_dict.get('frontend', {}) -%}
 {% set has_frontend = frontend_dict.get('software-url', '') != '' -%}
 {% set site_id = slapparameter_dict.get('site-id', 'erp5') -%}