software/theia: Select free ports automatically

Changes:
- Add comments and reorganise instance.cfg.in for clarity
- Select free ports instead of hardcoded ports
- Upgrade to slapos.core 1.6.5 to prefix forwarded requests

This makes it possible to recursively nest theias into theias.

See merge request nexedi/slapos!919

software/theia/buildout.hash.cfg

@@ -15,7 +15,7 @@
 
 [instance]
 filename = instance.cfg.in
-md5sum = 397a2e0ac0d7f8813e0b594af6785300
+md5sum = 4f119df2b2d71e5ec1fd04051fd8fff8
 
 [yarn.lock]
 filename = yarn.lock

software/theia/instance.cfg.in

 [buildout]
-parts =
-  monitor-base
-  promises
-  frontend-reload
+extends =
+  ${monitor-template:rendered}
+
+theia-environment-parts =
   tasks.json
-  publish-connection-parameter
   slapos-repository
   runner-link
   settings.json
 
-extends = ${monitor-template:rendered}
+theia-parts =
+  frontend-reload
+  promises
+
+parts =
+  monitor-base
+  $${:theia-parts}
+  $${:theia-environment-parts}
+  publish-connection-parameter
 
 eggs-directory = ${buildout:eggs-directory}
 develop-eggs-directory = ${buildout:develop-eggs-directory}
 offline = true
 
 
+[instance-parameter]
+recipe = slapos.cookbook:slapconfiguration
+computer = $${slap-connection:computer-id}
+partition = $${slap-connection:partition-id}
+url = $${slap-connection:server-url}
+key = $${slap-connection:key-file}
+cert = $${slap-connection:cert-file}
+configuration.autorun = running
+configuration.embedded-sr =
+configuration.embedded-sr-type =
+
+
+[publish-connection-parameter]
+<= monitor-publish
+recipe = slapos.cookbook:publish
+url = $${apache-frontend:connection-secure_access}
+username = $${frontend-instance-password:username}
+password = $${frontend-instance-password:passwd}
+
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+etc = $${buildout:directory}/etc
+var = $${buildout:directory}/var
+srv = $${buildout:directory}/srv
+bin = $${buildout:directory}/bin
+tmp = $${buildout:directory}/tmp
+dot-theia = $${buildout:directory}/.theia/
+pidfiles = $${:var}/run
+
+services = $${:etc}/service
+runner = $${:srv}/runner
+project = $${:srv}/project
+frontend-static = $${:srv}/frontend-static
+frontend-static-public = $${:frontend-static}/public
+frontend-static-css = $${:frontend-static}/css
+
+bash-completions = $${buildout:directory}/.local/share/bash-completion/completions/
+fish-completions = $${buildout:directory}/.config/fish/completions/
+
+
+# Promises
+# --------
+
+[promises]
+recipe =
+instance-promises =
+  $${theia-listen-promise:name}
+  $${frontend-listen-promise:name}
+  $${apache-frontend-url-available-promise:name}
+  $${slapos-standalone-listen-promise:name}
+  $${slapos-autorun-promise:name}
+
+[theia-listen-promise]
+<= monitor-promise-base
+module = check_port_listening
+name = $${:_buildout_section_name_}.py
+config-hostname = $${theia-instance:ip}
+config-port = $${theia-instance:port}
+
+[frontend-listen-promise]
+<= monitor-promise-base
+module = check_port_listening
+name = $${:_buildout_section_name_}.py
+config-hostname = $${frontend-instance:ip}
+config-port = $${frontend-instance:port}
+
+[apache-frontend-url-available-promise]
+<= monitor-promise-base
+module = check_url_available
+name = $${:_buildout_section_name_}.py
+config-url = $${apache-frontend:connection-secure_access}
+config-check-secure = 1
+
+[slapos-standalone-listen-promise]
+<= monitor-promise-base
+module = check_port_listening
+# XXX promise plugins can not contain "slapos" in their names
+name = standalone-listen-promise.py
+config-hostname = $${slapos-standalone-instance:hostname}
+config-port = $${slapos-standalone-instance:port}
+
+[slapos-autorun-promise]
+<= monitor-promise-base
+module = check_service_state
+# XXX promise plugins can not contain "slapos" in their names
+name = autorun-state-promise.py
+config-service = $${slapos-autorun:service-name}
+config-expect = $${slapos-autorun:autorun}
+
+
+# Remote Apache Frontend
+# ----------------------
+
+[apache-frontend]
+<= slap-connection
+recipe = slapos.cookbook:requestoptional
+name = Theia Frontend
+# XXX We have hardcoded SR URL here.
+software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
+slave = true
+config-url = $${frontend-instance:url}
+config-https-only = true
+config-type = websocket
+config-websocket-path-list = /services /file-upload
+return = domain secure_access
+
+
+# Local Caddy Frontend
+# --------------------
+
 [frontend-instance-password]
 recipe = slapos.cookbook:generate.password
 username = node
 bytes = 12
 
+[frontend-instance-port]
+recipe = slapos.cookbook:free_port
+minimum = 3000
+maximum = 3100
+ip = $${instance-parameter:ipv6-random}
+
 [frontend-instance-certificate]
 recipe = plone.recipe.command
 command =
@@ -70,9 +194,9 @@ template = inline:
       /
     }
   }
-ip = $${instance-parameter:ipv6-random}
+ip = $${frontend-instance-port:ip}
 hostname = [$${:ip}]
-port = 3001
+port = $${frontend-instance-port:port}
 
 [frontend-instance]
 recipe = slapos.cookbook:wrapper
@@ -152,67 +276,15 @@ install =
 location = $${directory:frontend-static}/$${:filename}
 filename = $${:_buildout_section_name_}
 
-[tasks.json]
-recipe = slapos.recipe.template:jinja2
-rendered = $${directory:dot-theia}/tasks.json
-template =
-  inline:
-  {
-    // See https://go.microsoft.com/fwlink/?LinkId=733558
-    // for the documentation about the tasks.json format
-    "version": "2.0.0",
-    "tasks": [
-      {
-        "label": "slapos node software",
-        "detail": "Build all software supplied to the node",
-        "type": "shell",
-        "command": "${buildout:bin-directory}/slapos",
-        "args": [
-          "node",
-          "software",
-          // debug mode can be enabled by commenting out this line:
-          // "--buildout-debug",
-          "--all"
-        ],
-        "options": {
-          "env": {
-            "SLAPOS_CONFIGURATION": "$${slapos-standalone-config:slapos-configuration}",
-            "GIT_EXEC_PATH": ""
-          }
-        },
-        "group": {
-          "kind": "build",
-          "isDefault": true
-        },
-        "problemMatcher": []
-      },
-      {
-        "label": "slapos node instance",
-        "detail": "Create all instances requested on the node",
-        "type": "shell",
-        "command": "${buildout:bin-directory}/slapos",
-        "args": [
-          "node",
-          "instance",
-          // debug mode can be enabled by commenting out this line:
-          // "--buildout-debug",
-          "--all"
-        ],
-        "options": {
-          "env": {
-            "SLAPOS_CONFIGURATION": "$${slapos-standalone-config:slapos-configuration}",
-            "GIT_EXEC_PATH": ""
-          }
-        },
-        "problemMatcher": [],
-        "group": {
-          "kind": "build",
-          "isDefault": true
-        }
-      }
-    ]
-  }
 
+# Theia Backend
+# -------------
+
+[theia-service-port]
+recipe = slapos.cookbook:free_port
+minimum = 3500
+maximum = 3600
+ip = $${instance-parameter:ipv4-random}
 
 [theia-service]
 recipe = slapos.recipe.template:jinja2
@@ -233,8 +305,8 @@ template =
   export TERMINFO=${ncurses:location}/lib/terminfo/
   export EDITOR="${python-language-server:location}/bin/python -m theia_open --wait"
   exec ${theia-wrapper:rendered} $@
-ip =  $${instance-parameter:ipv4-random}
-port = 3000
+ip =  $${theia-service-port:ip}
+port = $${theia-service-port:port}
 base-url = http://$${:ip}:$${:port}/
 
 [theia-instance]
@@ -285,10 +357,19 @@ command =
   ${buildout:bin-directory}/slapos complete --shell fish > $${directory:fish-completions}/slapos.fish
 
 
+# SlapOS Standalone
+# -----------------
+
+[slapos-standalone-port]
+recipe = slapos.cookbook:free_port
+minimum = 4000
+maximum = 4100
+ip = $${instance-parameter:ipv4-random}
+
 [slapos-standalone-config]
 ipv4 = $${instance-parameter:ipv4-random}
 ipv6 = $${instance-parameter:ipv6-random}
-port = 4000
+port = $${slapos-standalone-port:port}
 slapos-configuration = $${directory:runner}/etc/slapos.cfg
 computer-id = slaprunner
 
@@ -333,82 +414,9 @@ hash-files =
 hostname = $${slapos-standalone-config:ipv4}
 port = $${slapos-standalone-config:port}
 
-[promises]
-recipe =
-instance-promises =
-  $${theia-listen-promise:name}
-  $${frontend-listen-promise:name}
-  $${apache-frontend-url-available-promise:name}
-  $${slapos-standalone-listen-promise:name}
-  $${slapos-autorun-promise:name}
-
-[theia-listen-promise]
-<= monitor-promise-base
-module = check_port_listening
-name = $${:_buildout_section_name_}.py
-config-hostname = $${theia-instance:ip}
-config-port = $${theia-instance:port}
-
-[frontend-listen-promise]
-<= monitor-promise-base
-module = check_port_listening
-name = $${:_buildout_section_name_}.py
-config-hostname = $${frontend-instance:ip}
-config-port = $${frontend-instance:port}
-
-[apache-frontend-url-available-promise]
-<= monitor-promise-base
-module = check_url_available
-name = $${:_buildout_section_name_}.py
-config-url = $${apache-frontend:connection-secure_access}
-config-check-secure = 1
-
-[slapos-standalone-listen-promise]
-<= monitor-promise-base
-module = check_port_listening
-# XXX promise plugins can not contain "slapos" in their names
-name = standalone-listen-promise.py
-config-hostname = $${slapos-standalone-instance:hostname}
-config-port = $${slapos-standalone-instance:port}
 
-[slapos-autorun-promise]
-<= monitor-promise-base
-module = check_service_state
-# XXX promise plugins can not contain "slapos" in their names
-name = autorun-state-promise.py
-config-service = $${slapos-autorun:service-name}
-config-expect = $${slapos-autorun:autorun}
-
-[apache-frontend]
-<= slap-connection
-recipe = slapos.cookbook:requestoptional
-name = Theia Frontend
-# XXX We have hardcoded SR URL here.
-software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
-slave = true
-config-url = $${frontend-instance:url}
-config-https-only = true
-config-type = websocket
-config-websocket-path-list = /services /file-upload
-return = domain secure_access
-
-[publish-connection-parameter]
-<= monitor-publish
-recipe = slapos.cookbook:publish
-url = $${apache-frontend:connection-secure_access}
-username = $${frontend-instance-password:username}
-password = $${frontend-instance-password:passwd}
-
-[instance-parameter]
-recipe = slapos.cookbook:slapconfiguration
-computer = $${slap-connection:computer-id}
-partition = $${slap-connection:partition-id}
-url = $${slap-connection:server-url}
-key = $${slap-connection:key-file}
-cert = $${slap-connection:cert-file}
-configuration.autorun = running
-configuration.embedded-sr =
-configuration.embedded-sr-type =
+# Slapos Standalone Autoprocessing
+# --------------------------------
 
 [slapos-autorun]
 recipe = plone.recipe.command
@@ -422,6 +430,71 @@ service-name = slapos-node-auto
 supervisor-conf = $${directory:runner}/etc/supervisord.conf
 autorun = $${instance-parameter:configuration.autorun}
 
+
+# Theia Local Environment Setup
+# -----------------------------
+
+[tasks.json]
+recipe = slapos.recipe.template:jinja2
+rendered = $${directory:dot-theia}/tasks.json
+template =
+  inline:
+  {
+    // See https://go.microsoft.com/fwlink/?LinkId=733558
+    // for the documentation about the tasks.json format
+    "version": "2.0.0",
+    "tasks": [
+      {
+        "label": "slapos node software",
+        "detail": "Build all software supplied to the node",
+        "type": "shell",
+        "command": "${buildout:bin-directory}/slapos",
+        "args": [
+          "node",
+          "software",
+          // debug mode can be enabled by commenting out this line:
+          // "--buildout-debug",
+          "--all"
+        ],
+        "options": {
+          "env": {
+            "SLAPOS_CONFIGURATION": "$${slapos-standalone-config:slapos-configuration}",
+            "GIT_EXEC_PATH": ""
+          }
+        },
+        "group": {
+          "kind": "build",
+          "isDefault": true
+        },
+        "problemMatcher": []
+      },
+      {
+        "label": "slapos node instance",
+        "detail": "Create all instances requested on the node",
+        "type": "shell",
+        "command": "${buildout:bin-directory}/slapos",
+        "args": [
+          "node",
+          "instance",
+          // debug mode can be enabled by commenting out this line:
+          // "--buildout-debug",
+          "--all"
+        ],
+        "options": {
+          "env": {
+            "SLAPOS_CONFIGURATION": "$${slapos-standalone-config:slapos-configuration}",
+            "GIT_EXEC_PATH": ""
+          }
+        },
+        "problemMatcher": [],
+        "group": {
+          "kind": "build",
+          "isDefault": true
+        }
+      }
+    ]
+  }
+
 [slapos-repository]
 recipe = slapos.recipe.build:gitclone
 repository = https://lab.nexedi.com/nexedi/slapos.git
@@ -451,23 +524,3 @@ template =
 recipe = slapos.cookbook:symbolic.link
 target-directory = $${directory:project}
 link-binary = $${directory:runner}
-
-[directory]
-recipe = slapos.cookbook:mkdirectory
-etc = $${buildout:directory}/etc
-var = $${buildout:directory}/var
-srv = $${buildout:directory}/srv
-bin = $${buildout:directory}/bin
-tmp = $${buildout:directory}/tmp
-dot-theia = $${buildout:directory}/.theia/
-pidfiles = $${:var}/run
-
-services = $${:etc}/service
-runner = $${:srv}/runner
-project = $${:srv}/project
-frontend-static = $${:srv}/frontend-static
-frontend-static-public = $${:frontend-static}/public
-frontend-static-css = $${:frontend-static}/css
-
-bash-completions = $${buildout:directory}/.local/share/bash-completion/completions/
-fish-completions = $${buildout:directory}/.config/fish/completions/

stack/slapos.cfg

@@ -200,7 +200,7 @@ setuptools-dso = 1.7
 rubygemsrecipe  = 0.3.0
 six = 1.12.0
 slapos.cookbook = 1.0.171
-slapos.core = 1.6.4
+slapos.core = 1.6.5
 slapos.extension.strip = 0.4
 slapos.extension.shared = 1.0
 slapos.libnetworkcache = 0.20