Update Release Candidate

component/depot_tools/buildout.cfg

@@ -7,5 +7,5 @@ parts =
 [depot_tools]
 recipe = slapos.recipe.build:gitclone
 repository = https://chromium.googlesource.com/chromium/tools/depot_tools.git
-revision = e023d4482012d89690f6a483e877eceb47c4501e
+revision = eb48a6ac0fa5835353ddd137ac35f44eee011716
 git-executable = ${git:location}/bin/git

component/headless-chromium/buildout.cfg

@@ -45,9 +45,9 @@ gclient-location = ${buildout:parts-directory}/${:_buildout_section_name_}
 # called "src".
 name = src
 
-# 96.0.4664.129 version is the latest stable version in December 2021.
-# Note that we need a version compiling without python2
-version = 96.0.4664.129
+
+# 114.0.5735.340 version is the latest stable version in November 2023.
+version = 114.0.5735.340
 
 
 [headless-chromium]

software/headless-chromium/README.md

@@ -5,7 +5,7 @@ exposes an interface to connect to it remotely from another browser.
 After deployment, the instance is configured like this:
 
 ```
-   Caddy frontend
+   Rapid CDN Frontend
      |
    (HTTPS, IPv6)
      |
@@ -27,7 +27,7 @@ The following instance parameters can be configured:
 - nginx-proxy-port:       Port for Ningx proxy to listen on.
 - monitor-httpd-port:     Port for monitor.
 - incognito:              Force Incognito mode
-- window-size:            Initial windo size
+- window-size:            Initial window size
 - block-new-web-contents: Block new web contents
 
 See `instance-headless-chromium-input-schema.json` for default values.

software/headless-chromium/buildout.hash.cfg

 [template-cfg]
 filename = instance.cfg.in
-md5sum = 6315598b2c7c19f9e2d9cdf090492e2c
+md5sum = c6cdcee1e16dd4bd3bc462d286dcb999
 
 [instance-headless-chromium]
 _update_hash_filename_ = instance-headless-chromium.cfg.in
-md5sum = feaef60353c94e02d38cfec66f0eb861
+md5sum = 8a7e024569d92b0992f40ddac232cff5
 
 [template-nginx-conf]
 _update_hash_filename_ = templates/nginx.conf.in
-md5sum = 1f35f91fa7e490cd1e2194264a8a6ed8
+md5sum = 6ba793ce45bc67882ab2eea319984e3f
 
 [template-mime-types]
 _update_hash_filename_ = templates/mime_types.in
 md5sum = 4ef94a7b458d885cd79ba0b930a5727e
+
+[template-index-html]
+_update_hash_filename_ = templates/index.html
+md5sum = 9314b30f97535a4e516f4ea0c2029ab0

software/headless-chromium/instance-headless-chromium.cfg.in

@@ -9,6 +9,8 @@ log = ${:home}/log
 etc = ${:home}/etc
 ssl = ${:etc}/ssl
 service = ${:etc}/service
+srv = ${:home}/srv
+nginx-root = ${:srv}/nginx-root
 
 # Options for instance configuration. See README.md for a list of
 # options that can be configured when requesting an instance.
@@ -17,9 +19,7 @@ ipv4 = {{ partition_ipv4 }}
 ipv6 = {{ partition_ipv6 }}
 remote-debugging-port = {{ parameter_dict['remote-debugging-port'] }}
 target-url = {{ parameter_dict['target-url'] }}
-incognito = {{ parameter_dict['incognito'] }}
 window-size = {{ parameter_dict['window-size'] }}
-block-new-web-contents = {{ parameter_dict['block-new-web-contents'] }}
 remote-debugging-address = ${:ipv4}:${:remote-debugging-port}
 devtools-frontend-root = {{ parameter_list['devtools-frontend'] }}
 
@@ -34,7 +34,8 @@ nginx-htpasswd-file = ${directory:etc}/.htpasswd
 nginx-key-file = ${frontend-instance-certificate:key-file}
 nginx-cert-file = ${frontend-instance-certificate:cert-file}
 nginx-mime-types = ${directory:etc}/mime-types
-
+nginx-root = ${directory:nginx-root}
+nginx-index-html = ${:nginx-root}/index.html
 
 # Create a wrapper script in /bin/chromium for the headless shell
 # executable.
@@ -45,10 +46,11 @@ command-line =
   {{ parameter_list['chromium-wrapper'] }}
   --remote-debugging-address=${headless-chromium:ipv4}
   --remote-debugging-port=${headless-chromium:remote-debugging-port}
+  --remote-allow-origins=*
   --user-data-dir=${directory:tmp}
   --window-size="${headless-chromium:window-size}"
-  {% if parameter_dict['incognito'] %}--incognito{% endif -%}
-  {% if parameter_dict['block-new-web-contents'] %}--block-new-web-contents{% endif -%}
+  {% if parameter_dict['incognito'] %} --incognito{% endif -%}
+  {% if parameter_dict['block-new-web-contents'] %} --block-new-web-contents{% endif -%}
 {{ '\n  "${headless-chromium:target-url}"' }}
 environment =
   FONTCONFIG_FILE=${font-config:output}
@@ -74,6 +76,11 @@ recipe = slapos.recipe.template
 url = {{ parameter_list['template-mime-types'] }}
 output = ${headless-chromium:nginx-mime-types}
 
+[nginx-index-html]
+recipe = slapos.recipe.template
+url = {{ parameter_list['template-index-html'] }}
+output = ${headless-chromium:nginx-index-html}
+
 [nginx-launcher]
 recipe = slapos.cookbook:wrapper
 command-line =
@@ -195,6 +202,7 @@ parts =
   generate-passwd-file
   nginx-config
   nginx-mime-types
+  nginx-index-html
   nginx-launcher
   logrotate-entry-nginx
   remote-debugging-frontend

software/headless-chromium/instance.cfg.in

@@ -17,6 +17,7 @@ template-nginx-config = {{ template_nginx_config_target }}
 template-fonts-conf = {{ template_fonts_conf_target }}
 template-monitor = {{ template_monitor }}
 template-mime-types = {{ template_mime_types_target }}
+template-index-html = {{ template_index_html_target }}
 
 [instance-headless-chromium]
 recipe = slapos.recipe.template:jinja2

software/headless-chromium/software.cfg

@@ -27,6 +27,7 @@ context =
   key devtools_frontend headless-chromium:devtools-frontend
   key template_nginx_config_target template-nginx-conf:target
   key template_mime_types_target template-mime-types:target
+  key template_index_html_target template-index-html:target
   key template_fonts_conf_target template-fonts-conf:output
   key template_instance_headless_chromium_target instance-headless-chromium:target
   key template_monitor monitor2-template:output
@@ -43,3 +44,6 @@ url = ${:_profile_base_location_}/${:_update_hash_filename_}
 
 [template-mime-types]
 <= download-base
+
+[template-index-html]
+<= download-base

software/headless-chromium/templates/index.html

+<script>
+    fetch("/json")
+        .then(r => r.json())
+        .then(pages => window.location.replace(new URL(pages[0].devtoolsFrontendUrl, window.location)))
+</script>
\ No newline at end of file

software/headless-chromium/templates/nginx.conf.in

@@ -12,6 +12,7 @@ http {
 
   include {{ param_headless_chromium['nginx-mime-types'] }};
   default_type application/octet-stream;
+  root {{ param_headless_chromium['nginx-root'] }};
 
   server {
     listen {{ param_headless_chromium['proxy-address'] }} ssl;
@@ -30,8 +31,13 @@ http {
     uwsgi_temp_path {{ param_headless_chromium['nginx-temp-path'] }};
     scgi_temp_path {{ param_headless_chromium['nginx-temp-path'] }};
 
-    # All websocket connections are served from /devtools.
-    location /devtools {
+    # A minimal page to bootstrap the DevTools frontend
+    location = / {
+      try_files $uri $uri/index.html =404;
+    }
+
+    # All websocket connections are served from /devtools/page.
+    location /devtools/page {
       proxy_http_version 1.1;
       proxy_set_header Host {{ param_headless_chromium['remote-debugging-address'] }};
       proxy_pass http://{{ param_headless_chromium['remote-debugging-address'] }};
@@ -39,9 +45,9 @@ http {
       proxy_set_header Connection "Upgrade";
     }
 
-    # The DevTools frontend is served from /serve_file/@{version_hash}.
-    location ~ "^\/serve_file\/@[0-9a-f]{5,40}\/(.*)" {
-      alias {{ param_headless_chromium['devtools-frontend-root'] }}/$1;
+    # Static content from DevTools frontend
+    location /devtools {
+      alias {{ param_headless_chromium['devtools-frontend-root'] }};
     }
 
     location / {
@@ -59,11 +65,11 @@ http {
       # frontend CDN URL. The tricky thing is that the frontend URL is
       # not available yet when this file is built; what we do instead is
       # use the given Host header.
-      sub_filter "ws={{ param_headless_chromium['remote-debugging-address'] }}" "wss=$host";
+      sub_filter "ws={{ param_headless_chromium['remote-debugging-address'] }}" "wss=$http_host";
       sub_filter_once on;
       sub_filter_types application/json;
 
-      sub_filter "ws://{{ param_headless_chromium['remote-debugging-address'] }}" "wss://$host";
+      sub_filter "ws://{{ param_headless_chromium['remote-debugging-address'] }}" "wss://$http_host";
       sub_filter_types application/json;
 
       # We want to use our own DevTools frontend rather than

software/headless-chromium/test/setup.py

@@ -44,6 +44,7 @@ setup(
         'slapos.core',
         'slapos.libnetworkcache',
         'requests',
+        'websocket-client',
     ],
     zip_safe=True,
     test_suite='test',

software/headless-chromium/test/test.py

@@ -25,8 +25,13 @@
 #
 ##############################################################################
 
+import base64
 import os
+import ssl
+import urllib.parse
+
 import requests
+import websocket
 
 from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
 
@@ -35,16 +40,17 @@ setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
     os.path.abspath(
         os.path.join(os.path.dirname(__file__), '../software.cfg')))
 
+
 class TestHeadlessChromium(SlapOSInstanceTestCase):
   def setUp(self):
-    self.connection_parameters = self.requestDefaultInstance().getConnectionParameterDict()
+    self.connection_parameters = self.computer_partition.getConnectionParameterDict()
 
   def test_remote_debugging_port(self):
     # The headless browser should respond at /json with a nonempty list
     # of available pages, each of which has a webSocketDebuggerUrl and a
     # devtoolsFrontendUrl.
     url = self.connection_parameters['remote-debug-url']
-    response = requests.get('%s/json' % url)
+    response = requests.get(urllib.parse.urljoin(url, '/json'))
 
     # Check that request was successful and the response was a nonempty
     # list.
@@ -53,27 +59,72 @@ class TestHeadlessChromium(SlapOSInstanceTestCase):
 
     # Check that the first page has the correct fields.
     first_page = response.json()[0]
-    self.assertIn('webSocketDebuggerUrl', first_page)
     self.assertIn('devtoolsFrontendUrl', first_page)
+    websocket.create_connection(first_page['webSocketDebuggerUrl'], sslopt={"cert_reqs": ssl.CERT_NONE}).close()
 
   def test_devtools_frontend_ok(self):
-    # The proxy should serve the DevTools frontend from
-    # /serve_file/@{hash}/inspector.html, where {hash} is a 5-32 digit
-    # hash.
-    proxyURL = self.connection_parameters['proxy-url']
-    username = self.connection_parameters['username']
-    password = self.connection_parameters['password']
-    frontend = '/serve_file/@aaaaa/inspector.html'
-
-    response = requests.get(proxyURL + frontend, verify=False,
-                            auth=(username, password))
-    self.assertEqual(requests.codes['ok'], response.status_code)
+    param = self.computer_partition.getConnectionParameterDict()
+
+    # when accessed through RapidCDN, frontend rewrite WSS URLs with the host header but without port.
+    page, = requests.get(
+      urllib.parse.urljoin(param['proxy-url'], '/json'),
+      auth=(param['username'], param['password']),
+      headers={
+        'Host': 'hostname'
+      },
+      verify=False).json()
+    ws_debug_url = urllib.parse.urlparse(page['webSocketDebuggerUrl'])
+    self.assertEqual(
+      (ws_debug_url.scheme, ws_debug_url.netloc), ('wss', 'hostname'))
+
+    devtools_frontend_url = dict(
+      urllib.parse.parse_qsl(page['devtoolsFrontendUrl'].split('?')[1]))
+    # devtoolsFrontendUrl is a relative URL, like this:
+    # 'devtoolsFrontendUrl': '/devtools/inspector.html?wss=[::1]:9442/devtools/page/22C91CF307002BFA22DF0B4E34D2D026'
+    # and the query string argument wss must also have been rewritten:
+    self.assertTrue(
+      devtools_frontend_url['wss'].startswith('hostname/devtools/page/'))
+
+    requests.get(
+        urllib.parse.urljoin(param['proxy-url'], page['devtoolsFrontendUrl']),
+        auth=(param['username'], param['password']),
+        headers={
+          'Host': 'hostname'
+        },
+        verify=False).raise_for_status()
+
+    # when accessed directly, the :port is kept, as a consequence the debugger interface can
+    # be accessed directly from the nginx ipv6
+    page, = requests.get(
+      urllib.parse.urljoin(param['proxy-url'], '/json'),
+      auth=(param['username'], param['password']),
+      verify=False).json()
+    ws_debug_url = urllib.parse.urlparse(page['webSocketDebuggerUrl'])
+    self.assertEqual(ws_debug_url.port, 9224)
+
+    devtools_frontend_url = dict(urllib.parse.parse_qsl(page['devtoolsFrontendUrl'].split('?')[1]))
+    # devtoolsFrontendUrl is not rewritten
+    self.assertEqual(f"wss://{devtools_frontend_url['wss']}", page['webSocketDebuggerUrl'])
+
+    requests.get(
+        urllib.parse.urljoin(param['proxy-url'], page['devtoolsFrontendUrl']),
+        auth=(param['username'], param['password']),
+        verify=False).raise_for_status()
+
+    # the websocket is usable
+    websocket.create_connection(
+      page['webSocketDebuggerUrl'],
+      sslopt={"cert_reqs": ssl.CERT_NONE},
+      header={'Authorization': 'Basic ' + base64.b64encode(
+        f"{param['username']}:{param['password']}".encode()).strip().decode()}).close()
+
 
 class TestHeadlessChromiumParameters(SlapOSInstanceTestCase):
 
   instance_parameter_dict = {
     # this website echoes the get request for debugging purposes
     'target-url': 'https://httpbin.org/get?a=6&b=4',
+    # TODO: this does not work, this software uses 'xml' serialisation and only support strings
     'incognito': True,
     "block-new-web-contents": False,
     "window-size": "900,600"

software/ors-amarisoft/buildout.hash.cfg

@@ -16,7 +16,11 @@
 
 [template]
 filename = instance.cfg
-md5sum = e2e286ba8d40790da3da462d55276e29
+md5sum = df4d6e24453b649cc20564d7fcc38e4b
+
+[slaplte.jinja2]
+_update_hash_filename_ = slaplte.jinja2
+md5sum = c31dffa87765d93327f18ffd89ce36ca
 
 [amarisoft-stats.jinja2.py]
 _update_hash_filename_ = amarisoft-stats.jinja2.py
@@ -28,7 +32,7 @@ md5sum = ab666fdfadbfc7d8a16ace38d295c883
 
 [ru_libinstance.jinja2.cfg]
 _update_hash_filename_ = ru/libinstance.jinja2.cfg
-md5sum = 91ef336b7e913c07d89b82b27ffad11d
+md5sum = a6b710ca5132276d72f90b76b873fe98
 
 [ru_sdr_libinstance.jinja2.cfg]
 _update_hash_filename_ = ru/sdr/libinstance.jinja2.cfg
@@ -36,7 +40,7 @@ md5sum = c20b620111a4dc4bc2bcae57c2007cbe
 
 [ru_lopcomm_libinstance.jinja2.cfg]
 _update_hash_filename_ = ru/lopcomm/libinstance.jinja2.cfg
-md5sum = 02a7a12b933544b4287599afed0fc68d
+md5sum = a150743e78f9ecafc40b715f2aa80295
 
 [ru_sunwave_libinstance.jinja2.cfg]
 _update_hash_filename_ = ru/sunwave/libinstance.jinja2.cfg
@@ -44,11 +48,11 @@ md5sum = 0450e9fa50844e4d6e51d608625c57f6
 
 [ru_lopcomm_ncclient_common.py]
 _update_hash_filename_ = ru/lopcomm/ncclient_common.py
-md5sum = 6f8d0592cc4b0b695cea5a0c25aafc4e
+md5sum = 8dbe6a48fc0fca4f0cbd0c746be1aeda
 
 [ru_lopcomm_stats.jinja2.py]
 _update_hash_filename_ = ru/lopcomm/stats.jinja2.py
-md5sum = 3a5a3c5588e1698ee81e983bb08e70d7
+md5sum = b7ec0025a92e0947e4ac6abc4b06bf19
 
 [ru_lopcomm_config.jinja2.py]
 _update_hash_filename_ = ru/lopcomm/config.jinja2.py
@@ -66,10 +70,6 @@ md5sum = 9741fbc99aaf768e9cc3ab48925dfee5
 _update_hash_filename_ = ru/lopcomm/software.jinja2.py
 md5sum = 2b08bb666c5f3ab287cdddbfdb4c9249
 
-[ru_lopcomm_supervision.jinja2.py]
-_update_hash_filename_ = ru/lopcomm/supervision.jinja2.py
-md5sum = 437cc45f132d6c74647a716c6a1920e1
-
 [ru_tapsplit]
 _update_hash_filename_ = ru/tapsplit
 md5sum = 2b8b57c5771b2a2203c0e7767e629e55
@@ -80,7 +80,7 @@ md5sum = 52da9fe3a569199e35ad89ae1a44c30e
 
 [template-enb]
 _update_hash_filename_ = instance-enb.jinja2.cfg
-md5sum = 815d57a36c7e3b4361230fbd3c76a9ef
+md5sum = b52a8584712f9cee338c71a6dedc7dad
 
 [template-gnb]
 _update_hash_filename_ = instance-gnb.jinja2.cfg
@@ -108,7 +108,7 @@ md5sum = dcaac06553a3222b14c0013a13f4a149
 
 [enb.jinja2.cfg]
 filename = config/enb.jinja2.cfg
-md5sum = 8e49dfc9318da43bd817b8891cba24b7
+md5sum = a961cc1469bd2534645470f914f12905
 
 [drb_lte.jinja2.cfg]
 filename = config/drb_lte.jinja2.cfg

software/ors-amarisoft/config/enb.jinja2.cfg

+{%- import 'slaplte.jinja2' as slaplte  with context %}
+
+{#- for standalone testing via slapos-render-config.py
+    NOTE: keep in sync with ru/libinstance.jinja2.cfg   #}
+{%- if _standalone is defined %}
+{%-   set cell_list = {}  %}
+{%-   do slaplte.load_cell(cell_list) %}
+{%- endif %}
+
 {%- set cell_count = cell_list|length %}
 
 {%- if slapparameter_dict.get('tdd_ul_dl_config', '[Configuration 2] 5ms 2UL 6DL (default)') == '[Configuration 2] 5ms 2UL 6DL (default)' %}

software/ors-amarisoft/instance-enb.jinja2.cfg

@@ -32,6 +32,7 @@ eggs-directory = {{ eggs_directory }}
 develop-eggs-directory = {{ develop_eggs_directory }}
 offline = true
 
+{%- import 'slaplte.jinja2'            as slaplte with context  %}
 {%- import 'ru_libinstance.jinja2.cfg' as rulib   with context  %}
 {{ rulib.buildout() }}
 
@@ -367,6 +368,9 @@ extra-context =
     json cell_list {{ rulib.cell_list | tojson }}
     key sib23_file sib-config:output
     key drb_file   drb-config:output
+import-list =
+    rawfile slaplte.jinja2 {{ slaplte_template }}
+
 
 [publish-connection-information]
 <= monitor-publish

software/ors-amarisoft/instance.cfg

@@ -40,6 +40,7 @@ context =
     raw ru ${rf-mode:ru}
     $${:extra-context}
 import-list =
+    rawfile slaplte.jinja2 ${slaplte.jinja2:target}
     rawfile ru_libinstance.jinja2.cfg ${ru_libinstance.jinja2.cfg:target}
     rawfile ru_sdr_libinstance.jinja2.cfg ${ru_sdr_libinstance.jinja2.cfg:target}
     rawfile ru_lopcomm_libinstance.jinja2.cfg ${ru_lopcomm_libinstance.jinja2.cfg:target}
@@ -275,6 +276,7 @@ extra-context =
     key enb amarisoft:enb
     key sdr amarisoft:sdr
     raw enb_template   ${enb.jinja2.cfg:target}
+    raw slaplte_template ${slaplte.jinja2:target}
     raw drb_lte_template ${drb_lte.jinja2.cfg:target}
     raw sib23_template   ${sib23.jinja2.asn:target}
     raw amarisoft_stats_template   ${amarisoft-stats.jinja2.py:target}
@@ -282,7 +284,6 @@ extra-context =
     raw ru_lopcomm_stats_template   ${ru_lopcomm_stats.jinja2.py:target}
     raw ru_lopcomm_config_template   ${ru_lopcomm_config.jinja2.py:target}
     raw ru_lopcomm_software_template   ${ru_lopcomm_software.jinja2.py:target}
-    raw ru_lopcomm_supervision_template   ${ru_lopcomm_supervision.jinja2.py:target}
     raw ru_lopcomm_reset_info_template   ${ru_lopcomm_reset-info.jinja2.py:target}
     raw ru_lopcomm_reset_template   ${ru_lopcomm_reset.jinja2.py:target}
     raw ru_lopcomm_CreateProcessingEle_template   ${ru_lopcomm_CreateProcessingEle.jinja2.xml:target}

software/ors-amarisoft/ru/libinstance.jinja2.cfg

@@ -3,15 +3,16 @@
     Use buildout() macro to emit instance-level code to
     handle configured RUs.
 
+    NOTE: before importing package slaplte.jinja2 needs to already loaded as
+
+      {%- import 'slaplte.jinja2'  as slaplte     with context %}
+
     NOTE: driver-specific logic is implemented in rudrv.buildout_ru() .
 #}
 
 {#- cell_list keeps cell registry #}
-{%- set cell_list = slapparameter_dict.get('cell_list', {'default': {}}) %}
-{%- for i, k in enumerate(cell_list) %}
-{%-   set cell = cell_list[k] %}
-{%-   do cell.setdefault('cpri_port_number', i) %}
-{%- endfor %}
+{%- set cell_list = {} %}
+{%- do slaplte.load_cell(cell_list) %}
 
 
 {%- macro buildout()  %}
@@ -39,13 +40,6 @@ config-stats-period = {{ slapparameter_dict.get("enb_stats_fetch_period", 60) }}
                                  lopcomm=rudrv_lopcomm,
                                  sunwave=rudrv_sunwave) %}
 
-{#-   slaptap indicates tap interface, that slapos told us to use,
-      or 'xxx-notap-xxx' if slapos provided us either nothing or empty string. #}
-{%-   set slaptap = slap_configuration.get('tap-name', '')   %}
-{%-   if slaptap == '' %}
-{%-     set slaptap = 'xxx-notap-xxx'   %}
-{%-   endif %}
-
 {#-   split slapos tap interface for each RU
       fallback to non-split approach for ntap <= 1 to avoid hard-dependecy on setcap/tapsplit
 
@@ -58,7 +52,7 @@ config-stats-period = {{ slapparameter_dict.get("enb_stats_fetch_period", 60) }}
 [vtap]
 recipe = plone.recipe.command
 ntap = {{ ntap }}
-command = {{ netcapdo }} {{ pythonwitheggs }} {{ ru_tapsplit }} {{ slaptap }} ${:ntap}
+command = {{ netcapdo }} {{ pythonwitheggs }} {{ ru_tapsplit }} {{ slaplte.tap }} ${:ntap}
 update-command = ${:command}
 stop-on-error = true
 
@@ -68,8 +62,8 @@ ntap = 0
 stop-on-error = false
 
 {%-     if ntap == 1  %}
-{%-       do vtap_list.append(slaptap) %}
-[vtap.{{ slaptap }}]
+{%-       do vtap_list.append(slaplte.tap) %}
+[vtap.{{ slaplte.tap }}]
 network = {{ slap_configuration['tap-ipv6-network'] }}
 gateway = {{ slap_configuration['tap-ipv6-gateway'] }}
 addr    = {{ slap_configuration['tap-ipv6-addr'] }}
@@ -78,7 +72,7 @@ addr    = {{ slap_configuration['tap-ipv6-addr'] }}
 {%-   else  %}
 
 {%-   for i in range(1,1+ntap)  %}
-{%-     set tap = '%s-%d' % (slaptap, i)   %}
+{%-     set tap = '%s-%d' % (slaplte.tap, i)   %}
 {%-     do vtap_list.append(tap) %}
 [vtap.{{ tap }}]
 recipe = slapos.recipe.build
@@ -96,7 +90,7 @@ init =
   # simulate what tapsplit would assign to the tap
   # ( tap subinterface will be created for real later at install time - when it
   #   is too late to update section options )
-  slapnet = tapsplit.ifnet6('{{ slaptap }}')
+  slapnet = tapsplit.ifnet6('{{ slaplte.tap }}')
   tapnet  = tapsplit.netsplit(slapnet, {{ 1+ntap }}) [{{ i }}]
 
   options['network'] = str(tapnet)
@@ -119,17 +113,6 @@ init =
 {%- endfor %}
 
 
-{#-   assign TAP interfaces to RUs                              #}
-{%-   for i, (cell_ref, cell) in enumerate(cell_list|dictsort)  %}
-{%-     if len(cell_list) > 1                                   %}
-{%-       set ru_tap = "%s-%d" % (slaptap, i+1)                 %}
-{%-     else                                                    %}
-{%-       set ru_tap = slaptap                                  %}
-{%-     endif                                                   %}
-{%-     do cell.update({'_tap': ru_tap})                        %}
-{%-   endfor                                                    %}
-
-
 {#-   go through all RUs and for each RU invoke
       RU-specific buildout handler #}
 {%-   set ru_type = {'lopcomm': 'lopcomm', 'm2ru': 'sunwave'}.get(ru, 'sdr') %}

software/ors-amarisoft/ru/lopcomm/buildout.cfg

@@ -22,9 +22,6 @@ parts +=
 [ru_lopcomm_software.jinja2.py]
 <= download-base
 
-[ru_lopcomm_supervision.jinja2.py]
-<= download-base
-
 [ru_lopcomm_ncclient_common.py]
 <= download-base
 destination = ${buildout:directory}/ncclient_common.py

software/ors-amarisoft/ru/lopcomm/libinstance.jinja2.cfg

@@ -9,43 +9,9 @@ promise = check_socket_listening
 config-host = ${vtap.{{cell._tap}}:gateway}
 config-port = 830
 
-
-{#-  monitor state of netconf connection + keep on touching RU watchdog #}
-
-[{{ru_ref}}-supervision-template]
-recipe = slapos.recipe.template:jinja2
-extensions = jinja2.ext.do
-_logbase = ${directory:var}/log/{{ru_ref}}-supervision
-log-output = ${:_logbase}.log
-supervision-reply-json-log-output = ${:_logbase}-reply.json.log
-is_netconf_connected = ${directory:etc}/{{ru_ref}}.is_netconf_connected
-context =
-  section directory directory
-  section vtap vtap.{{ cell._tap }}
-  key slapparameter_dict slap-configuration:configuration
-  key log_file :log-output
-  key supervision_reply_json_log_file :supervision-reply-json-log-output
-  raw testing {{ slapparameter_dict.get("testing", False) }}
-  raw python_path {{ buildout_directory}}/bin/pythonwitheggs
-  raw buildout_directory_path {{ buildout_directory }}
-  key is_netconf_connected :is_netconf_connected
-  import netaddr netaddr
-mode = 0775
-url = {{ ru_lopcomm_supervision_template }}
-output = ${directory:bin}/{{ru_ref}}-supervision.py
-
-{{ part('%s-supervision-service' % ru_ref) }}
-recipe = slapos.cookbook:wrapper
-command-line = ${ {{- ru_ref}}-supervision-template:output}
-wrapper-path = ${directory:service}/{{ru_ref}}-supervision
-mode = 0775
-hash-files =
-  ${:command-line}
-
 {{ promise('%s-netconf-connection' % ru_ref) }}
 promise = check_command_execute
-config-command = [ -f ${ {{-ru_ref}}-supervision-template:is_netconf_connected} ]
-
+config-command = [ -f ${ {{-ru_ref}}-stats-template:is_netconf_connected} ]
 
 {#-  push firmware to RU #}
 
@@ -133,6 +99,8 @@ cfg-json-log-output = ${:_logbase}-config.json.log
 supervision-json-log-output = ${:_logbase}-supervision.json.log
 ncsession-json-log-output = ${:_logbase}-ncsession.json.log
 software-json-log-output = ${:_logbase}-software.json.log
+supervision-reply-json-log-output = ${:_logbase}-supervision-reply.json.log
+is_netconf_connected = ${directory:etc}/{{ru_ref}}.is_netconf_connected
 context =
   section directory directory
   section vtap vtap.{{ cell._tap }}
@@ -141,6 +109,8 @@ context =
   key json_log_file :json-log-output
   key cfg_json_log_file :cfg-json-log-output
   key supervision_json_log_file :supervision-json-log-output
+  key supervision_reply_json_log_file :supervision-reply-json-log-output
+  key is_netconf_connected :is_netconf_connected
   key ncsession_json_log_file :ncsession-json-log-output
   key software_json_log_file :software-json-log-output
   raw testing {{ slapparameter_dict.get("testing", False) }}

software/ors-amarisoft/ru/lopcomm/ncclient_common.py

@@ -103,24 +103,24 @@ class LopcommNetconfClient:
         sub = self.conn.create_subscription()
         self.logger.info('Subscription to %s successful' % (self.address,))
     def get_notification(self):
-        result = None
-        while result == None:
-            self.logger.debug('Waiting for notification from %s...' % (self.address,))
-            result = self.conn.take_notification(block=True)
-            if result:
-              self.logger.debug('Got new notification from %s...' % (self.address,))
-              result_in_xml = result._raw
-              data_dict = xmltodict.parse(result_in_xml)
-              if 'alarm-notif' in data_dict['notification']:
-                self.json_logger.info('', extra={'data': json.dumps(data_dict)})
-              elif 'supervision-notification' in data_dict['notification']:
-                self.supervision_json_logger.info('', extra={'data': json.dumps(data_dict)})
-              elif 'netconf-session-start' in data_dict['notification'] or 'netconf-session-end' in data_dict['notification']:
-                self.ncsession_json_logger.info('', extra={'data': json.dumps(data_dict)})
-              elif any(event in data_dict['notification'] for event in ['install-event', 'activation-event', 'download-event']):
-                  self.software_json_logger.info('', extra={'data': json.dumps(data_dict)})
-              else:
-                self.cfg_json_logger.info('', extra={'data': json.dumps(data_dict)})
+        self.logger.debug('Waiting for notification from %s...' % (self.address,))
+        result = self.conn.take_notification(block=True, timeout=120)
+        if result:
+          self.logger.debug('Got new notification from %s...' % (self.address,))
+          result_in_xml = result._raw
+          data_dict = xmltodict.parse(result_in_xml)
+          if 'alarm-notif' in data_dict['notification']:
+            self.json_logger.info('', extra={'data': json.dumps(data_dict)})
+          elif 'supervision-notification' in data_dict['notification']:
+            self.supervision_json_logger.info('', extra={'data': json.dumps(data_dict)})
+          elif 'netconf-session-start' in data_dict['notification'] or 'netconf-session-end' in data_dict['notification']:
+            self.ncsession_json_logger.info('', extra={'data': json.dumps(data_dict)})
+          elif any(event in data_dict['notification'] for event in ['install-event', 'activation-event', 'download-event']):
+              self.software_json_logger.info('', extra={'data': json.dumps(data_dict)})
+          else:
+            self.cfg_json_logger.info('', extra={'data': json.dumps(data_dict)})
+        else:
+            raise TimeoutError
     def edit_config(self, config_files):
         for config_file in config_files:
             with open(config_file) as f:
@@ -199,6 +199,23 @@ class LopcommNetconfClient:
             "running_slot_name_build_version": running_slot_name_build_version
         }
 
+    def supervision_reset(self, interval=60, margin=10):
+      self.logger.info("NETCONF server supervision replying...")
+      supervision_watchdog_rpc_xml = f"""
+          <supervision-watchdog-reset xmlns="urn:o-ran:supervision:1.0">
+              <supervision-notification-interval>{interval}</supervision-notification-interval>
+              <guard-timer-overhead>{margin}</guard-timer-overhead>
+          </supervision-watchdog-reset>
+      """
+      supervision_watchdog_reply_xml = self.custom_rpc_request(supervision_watchdog_rpc_xml)
+      replied = False
+      if supervision_watchdog_reply_xml:
+        replied = True
+        self.logger.info("NETCONF server supervision replied")
+        supervision_watchdog_data = xmltodict.parse(supervision_watchdog_reply_xml)
+        self.supervision_reply_json_logger.info('', extra={'data': json.dumps(supervision_watchdog_data)})
+      return replied
+
     def close(self):
         # Close not compatible between ncclient and netconf server
         #self.conn.close()

software/ors-amarisoft/ru/lopcomm/stats.jinja2.py

 #!{{ python_path }}
 import time
 import sys
+import os
+import threading
+
 sys.path.append({{ repr(buildout_directory_path) }})
 from ncclient_common import LopcommNetconfClient
 
+# Shared variable to indicate error occurred
+error_occurred = False
+lock = threading.Lock()
+
+def get_notification_continuously(nc):
+    global error_occurred
+    try:
+        while not error_occurred:
+            nc.get_notification()
+            pass
+    except Exception as e:
+        with lock:
+            error_occurred = True
+            nc.logger.error(f'Error in get_notification_continuously: {e}')
+
+# supervision watchdog keeps on
+def run_supervision_reset_continuously(nc):
+    global error_occurred
+    netconf_check_file = '{{ is_netconf_connected }}'
+    interval = 60
+    margin   = 10
+    try:
+        while not error_occurred:
+            t0 = time.time()
+            replied = nc.supervision_reset(interval, margin)
+            if replied:
+                with open(netconf_check_file, "w") as f:
+                    f.write('')
+            elif os.path.exists(netconf_check_file):
+                os.remove(netconf_check_file)
+
+            t1 = time.time()
+            time.sleep(interval - (t1 - t0))
+    except Exception as e:
+        with lock:
+            error_occurred = True
+            nc.logger.error(f'Error in run_supervision_reset_continuously: {e}')
+
 if __name__ == '__main__':
-  nc = LopcommNetconfClient(
-    log_file="{{ log_file }}",
-    json_log_file="{{ json_log_file }}",
-    cfg_json_log_file="{{ cfg_json_log_file }}",
-    supervision_json_log_file="{{ supervision_json_log_file }}",
-    ncsession_json_log_file="{{ ncsession_json_log_file }}",
-    software_json_log_file="{{ software_json_log_file }}"
+    nc = LopcommNetconfClient(
+        log_file="{{ log_file }}",
+        json_log_file="{{ json_log_file }}",
+        cfg_json_log_file="{{ cfg_json_log_file }}",
+        supervision_json_log_file="{{ supervision_json_log_file }}",
+        ncsession_json_log_file="{{ ncsession_json_log_file }}",
+        software_json_log_file="{{ software_json_log_file }}",
+        supervision_reply_json_log_file="{{ supervision_reply_json_log_file }}"
     )
-  while True:
+    threads = []
+
     try:
         nc.connect("{{ netaddr.IPAddress(vtap.gateway) }}", 830, "oranuser", "oranpassword")
         nc.subscribe()
-        while True:
-          nc.get_notification()
+
+        notification_thread = threading.Thread(target=get_notification_continuously, args=(nc,))
+        supervision_thread = threading.Thread(target=run_supervision_reset_continuously, args=(nc,))
+
+        threads.append(notification_thread)
+        threads.append(supervision_thread)
+
+        for thread in threads:
+            thread.start()
+
+        for thread in threads:
+            thread.join()
+
     except Exception as e:
         nc.logger.debug('Got exception, waiting 10 seconds before reconnecting...')
         nc.logger.debug(e)

software/ors-amarisoft/slaplte.jinja2

+{#- Package slaplte provides helpers for configuring Amarisoft LTE services in SlapOS.
+
+    - load_cell initializes cell registry.
+-#}
+
+
+{#- tap indicates tap interface, that slapos told us to use,
+    or 'xxx-notap-xxx' if slapos provided us either nothing or empty string. #}
+{%- set tap = slap_configuration.get('tap-name', '')   %}
+{%- if tap == '' %}
+{%-   set tap = 'xxx-notap-xxx'   %}
+{%- endif %}
+
+
+{#- ---- loading ---- #}
+
+{#- load_cell initializes cell registry.
+
+    cell_list keeps configured cells: {} cell reference -> cell parameters
+#}
+{%- macro load_cell(cell_list)	%}
+{%-   do cell_list.update( slapparameter_dict.get('cell_list', {'default': {}}) )  %}
+{%-   for i, k in enumerate(cell_list) %}
+{%-     set cell = cell_list[k] %}
+{%-     do cell.setdefault('cpri_port_number', i) %}
+{%-   endfor %}
+
+{#-   assign TAP interfaces to RUs  #}
+{%-   for i, (cell_ref, cell) in enumerate(cell_list|dictsort)  %}
+{%-     if len(cell_list) > 1 %}
+{%-       set ru_tap = "%s-%d" % (tap, i+1) %}
+{%-     else  %}
+{%-       set ru_tap = tap  %}
+{%-     endif %}
+{%-     do cell.update({'_tap': ru_tap})  %}
+{%-   endfor  %}
+
+{%- endmacro  %}

software/ors-amarisoft/slapos-render-config.py

@@ -12,6 +12,8 @@ import json
 # j2render renders config/<cfg>.jinja2.cfg into config/<cfg>.cfg with provided json parameters.
 def j2render(cfg, jcfg):
     ctx = json.loads(jcfg)
+    assert '_standalone' not in ctx
+    ctx['_standalone'] = True
     textctx = ''
     for k, v in ctx.items():
         textctx += 'json %s %s\n' % (k, json.dumps(v))
@@ -21,6 +23,8 @@ def j2render(cfg, jcfg):
       'url': 'config/{}.jinja2.cfg'.format(cfg),
       'output': 'config/{}.cfg'.format(cfg),
       'context': textctx,
+      'import-list': '''
+        rawfile slaplte.jinja2 slaplte.jinja2''',
       })
     #print(r.context)
 
@@ -34,54 +38,57 @@ def j2render(cfg, jcfg):
       f.write(r._render().decode())
 
 
-config = "gnb"
-json_params_empty = """{
-    "rf_mode": 'fdd',
-    "slap_configuration": {
-    },
-    "directory": {
-    },
-    "slapparameter_dict": {
-    }
-}"""
-json_params = """{
-    "rf_mode": "tdd",
-    "trx": "sdr",
-    "bbu": "ors",
-    "ru": "ors",
-    "one_watt": "True",
-    "earfcn": 646666,
-    "nr_arfcn": 646666,
-    "nr_band": 43,
-    "tx_gain": 62,
-    "rx_gain": 43,
-    "sib23_file": "sib",
-    "slap_configuration": {
-        "tap-name": "slaptap9",
-        "configuration.default_lte_bandwidth": "10 MHz",
-        "configuration.default_lte_imsi": "001010123456789",
-        "configuration.default_lte_k": "00112233445566778899aabbccddeeff",
-        "configuration.default_lte_inactivity_timer": 10000,
-        "configuration.default_nr_bandwidth": 40,
-        "configuration.default_nr_imsi": "001010123456789",
-        "configuration.default_nr_k": "00112233445566778899aabbccddeeff",
-        "configuration.default_nr_ssb_pos_bitmap": "10000000",
-        "configuration.default_n_antenna_dl": 2,
-        "configuration.default_n_antenna_ul": 2,
-        "configuration.default_nr_inactivity_timer": 10000,
-        "configuration.com_ws_port": 9001,
-        "configuration.com_addr": "127.0.1.2",
-        "configuration.amf_addr": "127.0.1.100",
-        "configuration.gtp_addr": "127.0.1.1"
-    },
-    "directory": {
-        "log": "log",
-        "etc": "etc",
-        "var": "var"
-    },
-    "slapparameter_dict": {
-        "tdd_ul_dl_config": "5ms 8UL 1DL 2/10 (maximum uplink)"
-    }
-}"""
+def do(cfg, slapparameter_dict):
+    jslapparameter_dict = json.dumps(slapparameter_dict)
+    json_params_empty = """{
+        "rf_mode": 'fdd',
+        "slap_configuration": {
+        },
+        "directory": {
+        },
+        "slapparameter_dict": %(jslapparameter_dict)s
+    }"""
+    json_params = """{
+        "rf_mode": "tdd",
+        "trx": "sdr",
+        "bbu": "ors",
+        "ru": "ors",
+        "one_watt": "True",
+        "earfcn": 646666,
+        "nr_arfcn": 646666,
+        "nr_band": 43,
+        "tx_gain": 62,
+        "rx_gain": 43,
+        "sib23_file": "sib",
+        "drb_file": "drb",
+        "slap_configuration": {
+            "tap-name": "slaptap9",
+            "configuration.default_lte_bandwidth": "10 MHz",
+            "configuration.default_lte_imsi": "001010123456789",
+            "configuration.default_lte_k": "00112233445566778899aabbccddeeff",
+            "configuration.default_lte_inactivity_timer": 10000,
+            "configuration.default_nr_bandwidth": 40,
+            "configuration.default_nr_imsi": "001010123456789",
+            "configuration.default_nr_k": "00112233445566778899aabbccddeeff",
+            "configuration.default_nr_ssb_pos_bitmap": "10000000",
+            "configuration.default_n_antenna_dl": 2,
+            "configuration.default_n_antenna_ul": 2,
+            "configuration.default_nr_inactivity_timer": 10000,
+            "configuration.com_ws_port": 9001,
+            "configuration.com_addr": "127.0.1.2",
+            "configuration.amf_addr": "127.0.1.100",
+            "configuration.mme_addr": "127.0.1.100",
+            "configuration.gtp_addr": "127.0.1.1"
+        },
+        "directory": {
+            "log": "log",
+            "etc": "etc",
+            "var": "var"
+        },
+        "slapparameter_dict": %(jslapparameter_dict)s
+    }"""
 
-j2render(config, json_params)
+    j2render(cfg, json_params % locals())
+
+do('enb', {"tdd_ul_dl_config": "[Configuration 6] 5ms 5UL 3DL (maximum uplink)"})
+do('gnb', {"tdd_ul_dl_config": "5ms 8UL 1DL 2/10 (maximum uplink)"})

software/ors-amarisoft/software-base.cfg

@@ -147,6 +147,8 @@ filename = ue-lte.jinja2.cfg
 [ue-nr.jinja2.cfg]
 <= copy-config-to-instance
 filename = ue-nr.jinja2.cfg
+[slaplte.jinja2]
+<= download-base
 
 # Download gadget files
 [software.cfg.html]

software/theia/buildout.hash.cfg

@@ -23,7 +23,7 @@ md5sum = 9658a11340c018de816d0de40683706a
 
 [instance-import]
 _update_hash_filename_ = instance-import.cfg.jinja.in
-md5sum = 6520d2aa0c1c6094cbf276080594ec1a
+md5sum = c8d1be7aee980deb0f0d1a3126c9b9da
 
 [instance-export]
 _update_hash_filename_ = instance-export.cfg.jinja.in

software/theia/instance-import.cfg.jinja.in

 {%- set parameter_dict = dict(default_parameter_dict, **parameter_dict) -%}
 {%- set additional_frontend = parameter_dict['additional-frontend-guid'] -%}
+{%- set namebase = parameter_dict['namebase'] -%}
+{%- set theia_number = parameter_dict['number'] -%}
+{%- set theia_id = '%s%s' % (namebase, theia_number) -%}
+{%- set frontend_name = parameter_dict['frontend-name'] -%}
 
 [buildout]
 extends = {{ theia_instance_cfg }}
@@ -16,16 +20,16 @@ parts +=
 # The import template then expects to receive it in
 # slap-parameter:namebase
 [slap-parameter]
-namebase = {{ parameter_dict['namebase'] }}
+namebase = {{ namebase }}
 
 
 # Change frontend name to avoid conflicts
 [remote-frontend]
-name = Import {{ parameter_dict['frontend-name'] }}
+name = {{ frontend_name }} for {{ theia_id }}
 
 {% if additional_frontend -%}
 [remote-additional-frontend]
-name = Import {{ parameter_dict['additional-frontend-name'] }}
+name = {{ parameter_dict['additional-frontend-name'] }} for {{ theia-id }}
 {%- endif %}
 
 

stack/slapos.cfg

@@ -300,7 +300,7 @@ simplegeneric = 0.8.1
 singledispatch = 3.4.0.3
 six = 1.16.0
 slapos.cookbook = 1.0.329
-slapos.core = 1.10.4
+slapos.core = 1.10.5
 slapos.extension.shared = 1.0
 slapos.libnetworkcache = 0.25
 slapos.rebootstrap = 4.5