Commit f9417613 authored by Thomas Gambier's avatar Thomas Gambier 🚴🏼

Update Release Candidate

parents c4b68ad5 873a49f6
......@@ -20,7 +20,7 @@ patch-options = -p1
patches =
${:_profile_base_location_}/noroot.patch#05fc6333e05576ea8e5a49f27a6ef951
configure-command = make
configure-options = makefiles CCARGS='-DUSE_SASL_AUTH -DUSE_CYRUS_SASL -DUSE_TLS -DHAS_PCRE -DHAS_DB -I${libdb:location}/include -I${pcre:location}/include -I${openssl-1.0:location}/include -I${cyrus-sasl:location}/include/sasl' AUXLIBS='-L${openssl-1.0:location}/lib -L${pcre:location}/lib -L${libdb:location}/lib -L${cyrus-sasl:location}/lib -lssl -lpcre -ldb -lcrypto -lsasl2 -Wl,-rpath=${openssl-1.0:location}/lib -Wl,-rpath=${pcre:location}/lib -Wl,-rpath=${libdb:location}/lib -Wl,-rpath=${cyrus-sasl:location}/lib'
configure-options = makefiles CCARGS='-DUSE_SASL_AUTH -DUSE_CYRUS_SASL -DUSE_TLS -DHAS_PCRE -DHAS_DB -I${libdb:location}/include -I${pcre:location}/include -I${openssl:location}/include -I${cyrus-sasl:location}/include/sasl' AUXLIBS='-L${openssl:location}/lib -L${pcre:location}/lib -L${libdb:location}/lib -L${cyrus-sasl:location}/lib -lssl -lpcre -ldb -lcrypto -lsasl2 -Wl,-rpath=${openssl:location}/lib -Wl,-rpath=${pcre:location}/lib -Wl,-rpath=${libdb:location}/lib -Wl,-rpath=${cyrus-sasl:location}/lib'
make-targets = non-interactive-package install_root=${:location}
environment =
PATH=${patch:location}/bin:${m4:location}/bin:%(PATH)s
......@@ -15,15 +15,15 @@
[template]
filename = instance.cfg.in
md5sum = e6d5c7bb627b4f1d3e7c99721b7c58fe
md5sum = 399b398a8eabfa6126d2a521dc779f9b
[template-kvm]
filename = instance-kvm.cfg.jinja2
md5sum = 31b17b55200ea065cb97085283ef5568
md5sum = 704b6ac6bf42837bcd8f4582c5a746c0
[template-kvm-cluster]
filename = instance-kvm-cluster.cfg.jinja2.in
md5sum = 28a00c28a972f42627849b25c2792abb
md5sum = 80b9b70ba1ccbc09deb8f9cad60f352c
[template-kvm-resilient]
filename = instance-kvm-resilient.cfg.jinja2
......@@ -31,7 +31,7 @@ md5sum = 7de5756f59ef7d823cd8ed33e6d15230
[template-kvm-import]
filename = instance-kvm-import.cfg.jinja2.in
md5sum = bd7e5db872b0dbe7716ec49c3907c401
md5sum = 3e7ff2ba85762ca47b5f90495d492570
[template-kvm-import-script]
filename = template/kvm-import.sh.jinja2
......@@ -39,7 +39,7 @@ md5sum = cd0008f1689dfca9b77370bc4d275b70
[template-kvm-export]
filename = instance-kvm-export.cfg.jinja2
md5sum = f12df4256eb5bd31a01c0ddc4b3897bb
md5sum = c15f7600389b0c641622fcfdc42260d8
[template-kvm-export-script]
filename = template/kvm-export.sh.jinja2
......@@ -88,3 +88,11 @@ md5sum = 9c67058edcc4edae0b57956c0932a9fc
[image-download-config-creator]
_update_hash_filename_ = template/image-download-config-creator.py
md5sum = 54261e418ab9860efe73efd514c4d47f
[whitelist-firewall-download-controller]
_update_hash_filename_ = template/whitelist-firewall-download-controller.py
md5sum = bc64e29546833817636261d1b28aa6dc
[whitelist-domains-default]
_update_hash_filename_ = template/whitelist-domains-default
md5sum = e9d40162ba77472775256637a2617d14
......@@ -556,6 +556,12 @@
"title": "FreeBSD 12.1 RELEASE bootonly x86_64"
}
]
},
"whitelist-domains": {
"title": "Whitelist domains",
"description": "List of whitelisted domain names to be accessed from the VM. They will be resolved to IPs depending on where the VM end up. IPs can be used too.",
"type": "string",
"textarea": true
}
},
"type": "object"
......
......@@ -107,6 +107,12 @@
"title": "FreeBSD 12.1 RELEASE bootonly x86_64"
}
]
},
"whitelist-domains": {
"title": "Whitelist domains",
"description": "List of whitelisted domain names to be accessed from the VM. They will be resolved to IPs depending on where the VM end up. IPs can be used too.",
"type": "string",
"textarea": true
}
},
"type": "object"
......
......@@ -24,7 +24,7 @@ global-ipv6 = {{ ipv6 }}
[slap-parameter]
{% for k, v in slapparameter_dict.items() -%}
{{ k }} = {{ v }}
{{ k }} = {{ dumps(v) }}
{% endfor -%}
[request-common]
......@@ -135,14 +135,13 @@ config-document-host = ${apache-conf:ip}
config-document-port = ${apache-conf:port}
config-document-path = ${hash-code:passwd}
config-keyboard-layout-language = {{ dumps(kvm_parameter_dict.get('keyboard-layout-language', 'fr')) }}
{%- if 'boot-image-url-list' in kvm_parameter_dict %}
{#- play nice: if parameter was not constructed by the original request, do not send it at all #}
config-boot-image-url-list = {{ kvm_parameter_dict['boot-image-url-list'] }}
{%- endif %}
{%- if 'boot-image-url-select' in kvm_parameter_dict %}
{#- play nice: if parameter was not constructed by the original request, do not send it at all #}
config-boot-image-url-select = {{ kvm_parameter_dict['boot-image-url-select'] }}
{%- endif %}
{%- for k in ['boot-image-url-list', 'boot-image-url-select', 'whitelist-domains'] %}
{#- play nice - use parameter only if present #}
{%- if k in kvm_parameter_dict %}
{#- play safe - dumps value #}
config-{{ k }} = {{ dumps(kvm_parameter_dict[k]) }}
{%- endif %}
{%- endfor %}
config-type = cluster
{% set bootstrap_script_url = slapparameter_dict.get('bootstrap-script-url', kvm_parameter_dict.get('bootstrap-script-url', '')) -%}
......
......@@ -5,24 +5,11 @@ extends =
parts +=
cron-entry-backup
certificate-authority
certificate-authority-service
publish-connection-information
kvm-vnc-promise
kvm-disk-image-corruption-promise
websockify-sighandler
websockify-sighandler-service
novnc-promise
cron
cron-service
frontend-promise
# monitor parts
monitor-base
${instance-kvm-parts:parts}
[slap-parameter]
{% for k, v in slapparameter_dict.items() -%}
{{ k }} = {{ v }}
{{ k }} = {{ dumps(v) }}
{% endfor -%}
# Create the exporter executable, which is a simple shell script
......
......@@ -14,7 +14,7 @@ extends =
[slap-parameter]
{% for k, v in slapparameter_dict.items() -%}
{{ k }} = {{ v }}
{{ k }} = {{ dumps(v) }}
{% endfor -%}
[resilient-publish-connection-parameter]
......
......@@ -419,6 +419,12 @@
"title": "FreeBSD 12.1 RELEASE bootonly x86_64"
}
]
},
"whitelist-domains": {
"title": "Whitelist domains",
"description": "List of whitelisted domain names to be accessed from the VM. They will be resolved to IPs depending on where the VM end up. IPs can be used too.",
"type": "string",
"textarea": true
}
}
}
{# Workaround empty parameter passing #}
{# In case of resilient '' is converted to 'None' string, but with slapproxy '' becomes None #}
{% for k, v in slapparameter_dict.items() %}
{% if v == 'None' or v is none %}
{% do slapparameter_dict.__setitem__(k, '') %}
{% endif %}
{% endfor %}
{% set additional_frontend = slapparameter_dict.get('frontend-additional-instance-guid') %}
{% set enable_http = str(slapparameter_dict.get('enable-http-server', False)).lower() == 'true' -%}
{% set use_tap = str(slapparameter_dict.get('use-tap', True)).lower() == 'true' -%}
......@@ -9,6 +16,7 @@
{% set instance_type = slapparameter_dict.get('type', 'standalone') -%}
{% set nat_rule_list = slapparameter_dict.get('nat-rules', '22 80 443') -%}
{% set disk_device_path = slapparameter_dict.get('disk-device-path', None) -%}
{% set whitelist_domains = slapparameter_dict.get('whitelist-domains', '') -%}
{% set boot_image_url_list_enabled = 'boot-image-url-list' in slapparameter_dict %}
{% set boot_image_url_select_enabled = 'boot-image-url-select' in slapparameter_dict %}
{% set cpu_max_count = dumps(slapparameter_dict.get('cpu-max-count', int(slapparameter_dict.get('cpu-count', 1)) + 1)) %}
......@@ -92,22 +100,14 @@ config-state = empty
config-url = ${monitor-base:base-url}/private/boot-image-url-select/${:filename} with username ${monitor-publish-parameters:monitor-user} and password ${monitor-publish-parameters:monitor-password}
[boot-image-url-select-source-config]
# generates configuration of the image from the user parameter
# special "magic" is used, to properly support multiline boot-image-url-select
# but in the same time correctly generate the configuration file
recipe = slapos.recipe.template:jinja2
template = inline:
{#- Do special trick to support boot-image-url-select being None, if key is present with value "" #}
{%- raw %}
{%- set boot_image_url_select = slap_parameter.get('boot-image-url-select') or '' %}
{%- if boot_image_url_select == 'None' %}
{#- That's insane protection, is it 'None' because of type = array? #}
{%- set boot_image_url_select = '' %}
{%- endif %}
{{ boot_image_url_select }}
{% endraw -%}
boot-image-url-select = {{ dumps(slapparameter_dict['boot-image-url-select']) }}
context =
section slap_parameter slap-parameter
key boot_image_url_select :boot-image-url-select
rendered = ${directory:etc}/boot-image-url-select.json
[boot-image-url-select-processed-config]
......@@ -199,17 +199,14 @@ config-state = empty
config-url = ${monitor-base:base-url}/private/boot-image-url-list/${:filename} with username ${monitor-publish-parameters:monitor-user} and password ${monitor-publish-parameters:monitor-password}
[boot-image-url-list-source-config]
# generates configuration of the image from the user parameter
# special "magic" is used, to properly support multiline boot-image-url-list
# but in the same time correctly generate the configuration file
recipe = slapos.recipe.template:jinja2
template = inline:
{#- Do special trick to support boot-image-url-list being None, if key is present with value "" #}
{%- raw %}
{{ slap_parameter.get('boot-image-url-list') or '' }}
{{ boot_image_url_list }}
{% endraw -%}
boot-image-url-list = {{ dumps(slapparameter_dict['boot-image-url-list']) }}
context =
section slap_parameter slap-parameter
key boot_image_url_list :boot-image-url-list
rendered = ${directory:etc}/boot-image-url-list.conf
[boot-image-url-list-processed-config]
......@@ -976,7 +973,7 @@ keyboard-layout-language = fr
{% set key_list = v.split('\n') -%}
{{ k }} =
{{ key_list | join('\n ') }}
{% elif k == 'boot-image-url-list' %}
{% elif k in ['boot-image-url-list', 'boot-image-url-select', 'whitelist-domains'] %}
{# needs to decorate possibly multiline or maybe unsafe value #}
{{ k }} = {{ dumps(v) }}
{% else -%}
......@@ -1042,7 +1039,62 @@ command-line =
{% endif -%}
[buildout]
{% do part_list.append('whitelist-firewall') -%}
[whitelist-firewall]
recipe = slapos.cookbook:wrapper
hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
wrapper-path = ${directory:scripts}/${:_buildout_section_name_}
command-line =
{{ dnsresolver_executable }}
--style list
--output ${:output}
${:source}
source =
${whitelist-domains-request:rendered}
{{ whitelist_domains_default }}
${whitelist-domains-resolv.conf:output}
${whitelist-domains-download:output}
output = ${buildout:directory}/.slapos-whitelist-firewall
[whitelist-firewall-directory]
recipe = plone.recipe.command
location = ${buildout:parts-directory}/whitelist-firewall
command = mkdir -p ${:location}
update-command = ${:command}
[whitelist-domains-request]
recipe = slapos.recipe.template:jinja2
template = inline:
{%- raw %}
{%- for domain in whitelist_domains.split() %}
{{ domain }}
{%- endfor %}
{% endraw -%}
rendered = ${whitelist-firewall-directory:location}/${:_buildout_section_name_}.txt
whitelist-domains = {{ dumps(whitelist_domains) }}
extensions = jinja2.ext.do
context =
key whitelist_domains :whitelist-domains
[whitelist-domains-resolv.conf]
recipe = plone.recipe.command
output = ${whitelist-firewall-directory:location}/${:_buildout_section_name_}.txt
update-command = ${:command}
command =
egrep ^nameserver /etc/resolv.conf | cut -d ' ' -f 2 > ${:output}
[whitelist-domains-download]
recipe = slapos.cookbook:wrapper
hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
wrapper-path = ${directory:scripts}/${:_buildout_section_name_}
output = ${whitelist-firewall-directory:location}/${:_buildout_section_name_}.txt
interval = 3600
command-line = {{ python_executable }} {{ whitelist_firewall_download_controller }} {{ curl_executable_location }} 3600 ${:output} ${:url}
url = https://stream.nxdcdn.com/rapidspace-whitelist-domains
[instance-kvm-parts]
parts =
certificate-authority
certificate-authority-service
......@@ -1081,6 +1133,9 @@ parts =
# Complete parts with sections
{{ part_list | join('\n ') }}
[buildout]
parts = ${instance-kvm-parts:parts}
extends =
# Add extends list
{{ extends_list | join('\n ') }}
......
......@@ -80,9 +80,12 @@ extra-context =
raw ansible_promise_tpl ${template-ansible-promise:location}/${template-ansible-promise:filename}
raw curl_executable_location ${curl:location}/bin/curl
raw dash_executable_location ${dash:location}/bin/dash
raw dnsresolver_executable ${buildout:bin-directory}/dnsresolver
raw dcron_executable_location ${dcron:location}/sbin/crond
raw debian_amd64_netinst_location ${debian-amd64-netinst.iso:location}/${debian-amd64-netinst.iso:filename}
raw file_download_script ${file-download-script:location}/${file-download-script:filename}
raw whitelist_domains_default ${whitelist-domains-default:location}/${whitelist-domains-default:filename}
raw whitelist_firewall_download_controller ${whitelist-firewall-download-controller:target}
raw image_download_controller ${image-download-controller:target}
raw image_download_config_creator ${image-download-config-creator:target}
raw logrotate_cfg ${template-logrotate-base:rendered}
......
......@@ -182,6 +182,10 @@ path = download_file.in
filename = download_file
on-update = true
[whitelist-domains-default]
<= download-template-base
filename = whitelist-domains-default
[template-httpd]
recipe = slapos.recipe.template:jinja2
template = ${:_profile_base_location_}/instance-kvm-http.cfg.in
......@@ -199,6 +203,10 @@ mode = 640
[image-download-config-creator]
<= image-download-controller
[whitelist-firewall-download-controller]
<= image-download-controller
[versions]
websockify = 0.9.0
......
# Minimal whitelisted domains needed to instantiate the instance
# Does not guarantee good usage of the guest VM itself
# The full list shall end up in whitelist-domains-download:url
# shcache.nxdcdn.com is default source for a lot of operations
shacache.nxdcdn.com
# stream.nxdcdn.com is needed by partition itself
stream.nxdcdn.com
# partition has to access default SlapOS Master
slap.vifib.com
slapos.vifib.com
# Partition needs access to SlapOS Master related resources
hnode.cdn.vifib.com
node.cdn.vifib.com
#!/usr/bin/env python
import os
import subprocess
import sys
import time
import logging
# Note: Assuring only one running instance is not done, as this script is only
# run from supervisord, which does it already
if __name__ == "__main__":
curl, sleep, output, url = sys.argv[1:]
sleep = int(sleep)
tmp_output = output + '.tmp'
logging.basicConfig(
format='%%(asctime)s [%%(levelname)s] %s : %%(message)s' % (url,),
level=logging.DEBUG)
logging.info('Redownloading each %is', sleep)
while True:
logging.info('Fetching')
try:
subprocess.check_output([
curl,
'--location', # follow redirects
'--no-progress-meter', # do not tell too much
'--max-time', '600', # 10 minutes is maximum
'--fail', # fail in case of wrong HTTP code
'--output', tmp_output, url],
stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
logging.error('Problem while downloading: %r', e.output.strip())
if os.path.exists(tmp_output):
logging.info('Stored output')
os.rename(tmp_output, output)
logging.info('Sleeping for %is', sleep)
time.sleep(sleep)
This diff is collapsed.
......@@ -5,6 +5,7 @@ extends =
../../component/openssl/buildout.cfg
../../component/logrotate/buildout.cfg
../../component/apache/buildout.cfg
../../component/geoip2/buildout.cfg
../../component/git/buildout.cfg
../../component/python-cryptography/buildout.cfg
../../stack/slapos.cfg
......@@ -36,11 +37,13 @@ eggs =
${python-cffi:egg}
${python-cryptography:egg}
pyOpenSSL
re6stnet
re6stnet[geoip]
initialization =
import os
path = os.environ.get('PATH', '')
os.environ['PATH'] = '${git:location}/bin' + (path and ':' + path)
path = os.environ.get('PATH', '/bin:/usr/bin')
os.environ['PATH'] = '${openssl:location}/bin:${git:location}/bin' + (
path and ':' + path)
os.environ['GEOIP2_MMDB'] = "${geolite2-country:location}/GeoLite2-Country.mmdb"
depends = ${re6stnet-develop:recipe}
[re6stnet-backup]
......@@ -59,6 +62,7 @@ template =
h=`sqlite3 "$2" .dump |git hash-object --stdin -w`
git update-index --add --cacheinfo 0644 "$h" registry.sql
git diff --cached --quiet || GIT_WORK_TREE=$PWD GIT_DIR=$PWD git \
-c user.name="$${0##*/}" -c user.email="$USER@`hostname`" \
-c gc.auto=100 -c gc.autoDetach=false commit --allow-empty-message -qm ''
[ ! "$3" ] || git push --mirror "$3"
context =
......
......@@ -216,9 +216,9 @@ slapos.rebootstrap = 4.5
slapos.recipe.build = 0.46
slapos.recipe.cmmi = 0.17
slapos.recipe.template = 4.5
slapos.toolbox = 0.117
slapos.toolbox = 0.118
stevedore = 1.21.0
subprocess32 = 3.5.3
subprocess32 = 3.5.4
unicodecsv = 0.14.1
wcwidth = 0.2.5
wheel = 0.35.1
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment