Skip to content

slapos
slapos
Commit 61929809 authored by Saurabh's avatar Saurabh Committed by Julien Muchembled
Browse files
Options

NEO: complete review of instanciation without backward compatibility 

- There's now a single software type 'neo' (in addition to the default)
  which contains:
  - 1 master
  - 1 admin
  - a configurable number of storage nodes, that share the same MySQL server
- Review of parameters to configure MySQL server.
  (innodb_buffer_pool_size is important)
- Check of meaningless values of 'replicas'.
parent 400ccad4
Hide whitespace changes
Inline Side-by-side
Showing with 148 additions and 143 deletions
software/neoppod/README.txt
View file @ 61929809
...@@ -30,20 +30,20 @@ cluster. ...@@ -30,20 +30,20 @@ cluster.
'partitions' (int, optional) 'partitions' (int, optional)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Number of partitions. You cannot change this value once you created a cluster. Number of partitions. You cannot change this value once you created a cluster.
Defautls to 12. Defaults to 12.
'replicas' (int, optional) 'replicas' (int, optional)
~~~~~~~~~~~~~~~~~~~~~~~~~~ ~~~~~~~~~~~~~~~~~~~~~~~~~~
Number of replicates. Number of replicates.
Defaults to 0 (no resilience). Defaults to 0 (no resilience).
'mysql-storage-count' (int, optional) 'node-list' (list, optional)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Number of MySQL-based storage nodes to deploy. One master node is deployed List of dictionaries. Defaults to list containing one empty dictionary.
along with each storage. One can specify following parameters in each of the dictionary.
Defaults to 1.
* 'storage-count': Number of storage nodes to deploy. Defaults to 1.
One master and one admin node is deployed with each storage.
* 'mysql': Dictionary containing configuration options for MySQL.
'admin-count' (int, optional)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Number of admin nodes to deploy.
Defaults to 1.
software/neoppod/cluster.cfg.in
View file @ 61929809
{% macro assert(x) %}{{ ("",)[not x] }}{% endmacro -%}
{% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%} {% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%}
{% set part_list = [] -%} {% set part_list = [] -%}
{% set master_list = [] -%} {% set section_id_list = [] -%}
{% set admin_list = [] -%}
[request-common] [request-common]
recipe = slapos.cookbook:request recipe = slapos.cookbook:request.serialised
software-url = ${slap-connection:software-release-url} software-url = ${slap-connection:software-release-url}
sla-computer_guid = ${slap-connection:computer-id} sla-computer_guid = ${slap-connection:computer-id}
server-url = ${slap-connection:server-url} server-url = ${slap-connection:server-url}
...@@ -12,68 +12,57 @@ cert-file = ${slap-connection:cert-file} ...@@ -12,68 +12,57 @@ cert-file = ${slap-connection:cert-file}
computer-id = ${slap-connection:computer-id} computer-id = ${slap-connection:computer-id}
partition-id = ${slap-connection:partition-id} partition-id = ${slap-connection:partition-id}
config-cluster = {{ dumps(slapparameter_dict['cluster']) }} config-cluster = {{ dumps(slapparameter_dict['cluster']) }}
{% set replicas = slapparameter_dict.get('replicas', 0) -%}
[node-base]
< = request-common
config-masters = ${all-masters:addresses}
[mysql-storage-base]
< = request-common
config-partitions = {{ dumps(slapparameter_dict.get('partitions', 12)) }} config-partitions = {{ dumps(slapparameter_dict.get('partitions', 12)) }}
config-replicas = {{ dumps(slapparameter_dict.get('replicas', 0)) }} config-replicas = {{ dumps(replicas) }}
config-upstream-cluster = {{ dumps(slapparameter_dict.get('upstream-cluster', '')) }} config-upstream-cluster = {{ dumps(slapparameter_dict.get('upstream-cluster', '')) }}
config-upstream-masters = {{ dumps(slapparameter_dict.get('upstream-masters', '')) }} config-upstream-masters = {{ dumps(slapparameter_dict.get('upstream-masters', '')) }}
software-type = {{ mysql_storage_software_type }} software-type = neo
[mysql-storage-request-base]
< = mysql-storage-base
return = master
[admin-base]
< = node-base
return = admin
software-type = {{ admin_software_type }}
[publish] [publish]
recipe = slapos.cookbook:publish recipe = slapos.cookbook:publish.serialised
masters = ${all-masters:addresses} masters = ${node-0-final:connection-masters}
admins = ${all-admins:addresses} admins = ${node-0-final:connection-admins}
{% for node_number in range(slapparameter_dict.get('mysql-storage-count', 1)) -%} {% set node_list = slapparameter_dict.get('node-list', ({},)) -%}
{% set section_id = 'storage-%i' % (node_number, ) -%} {% do assert(replicas < len(node_list)) -%}
{% set final_section_id = 'final-' ~ section_id -%} {% for i, node in enumerate(node_list) -%}
{% do master_list.append(section_id) -%} {% set section_id = 'node-' ~ i -%}
[{{ section_id }}] {% do section_id_list.append(section_id) -%}
< = mysql-storage-request-base
name = {{ section_id }}
[{{ section(final_section_id) }}] [{{ section_id }}-base]
< = mysql-storage-base
node-base
name = {{ section_id }} name = {{ section_id }}
{% for k, v in node.iteritems() -%}
config-{{ k }} = {{ dumps(v) }}
{% endfor -%}
[{{ section_id }}]
<= request-common
{{ section_id }}-base
return =
master
admin
{% endfor -%}
[final-base]
{% for i, section_id in enumerate(section_id_list) -%}
config-master-{{i}} = {{ '${' + section_id + ':connection-master}' }}
config-admin-{{i}} = {{ '${' + section_id + ':connection-admin}' }}
{% endfor -%} {% endfor -%}
{% for node_number in range(slapparameter_dict.get('admin-count', 1)) -%}
{% set section_id = 'admin-%i' % (node_number, ) -%}
{% do admin_list.append(section_id)%}
[{{ section(section_id) }}]
< = admin-base
name = {{ section_id }}
{% for section_id in section_id_list -%}
[{{ section(section_id + '-final') }}]
<= request-common
final-base
{{ section_id }}-base
{% if loop.first -%}
return =
masters
admins
{% endif -%}
{% endfor -%} {% endfor -%}
[buildout] [buildout]
parts = parts =
{{ part_list | join('\n\t') }} {{ part_list | join('\n\t') }}
publish publish
[all-masters]
addresses = {% for master in master_list -%}
{{ '${' + master + ':connection-master}' -}}
{% if not loop.last %} {% endif -%}
{% endfor %}
[all-admins]
addresses = {% for admin in admin_list -%}
{{ '${' + admin + ':connection-admin}' -}}
{% if not loop.last %} {% endif -%}
{% endfor %}
software/neoppod/instance-neo-admin.cfg.in
View file @ 61929809
[buildout] [buildout]
extends = {{ logrotate_cfg }}
parts += parts +=
neo-admin-promise neo-admin-promise
logrotate-admin logrotate-admin
...@@ -7,25 +6,12 @@ parts += ...@@ -7,25 +6,12 @@ parts +=
[neo-admin] [neo-admin]
recipe = slapos.cookbook:neoppod.admin recipe = slapos.cookbook:neoppod.admin
binary = {{bin_directory}}/neoadmin binary = {{bin_directory}}/neoadmin
wrapper = ${directory:run}/neoadmin wrapper = ${directory:etc_run}/neoadmin
logfile = ${directory:log}/neoadmin.log logfile = ${directory:log}/neoadmin.log
ip = ${publish:ip} ip = ${publish:ip}
port = ${publish:port} port = ${publish:port-admin}
cluster = {{ dumps(slapparameter_dict['cluster']) }} cluster = {{ dumps(slapparameter_dict['cluster']) }}
masters = {{ dumps(slapparameter_dict['masters']) }} masters = ${publish:masters}
[publish]
recipe = slapos.cookbook:publish
# TODO: make port a partition parameter
ip = {{ (ipv4_set | list)[0] }}
port = 10002
admin = ${:ip}:${:port}
[directory]
recipe = slapos.cookbook:mkdirectory
promises = ${buildout:directory}/etc/promises
run = ${buildout:directory}/etc/run
log = ${buildout:directory}/var/log
[neo-admin-promise] [neo-admin-promise]
recipe = slapos.cookbook:check_port_listening recipe = slapos.cookbook:check_port_listening
......
software/neoppod/instance-neo-master.cfg.in
View file @ 61929809
[buildout] [buildout]
extends = {{ logrotate_cfg }}
parts += parts +=
neo-master-promise neo-master-promise
logrotate-master logrotate-master
...@@ -7,33 +6,20 @@ parts += ...@@ -7,33 +6,20 @@ parts +=
[neo-master] [neo-master]
recipe = slapos.cookbook:neoppod.master recipe = slapos.cookbook:neoppod.master
binary = {{bin_directory}}/neomaster binary = {{bin_directory}}/neomaster
wrapper = ${directory:run}/neomaster wrapper = ${directory:etc_run}/neomaster
logfile = ${directory:log}/neomaster.log logfile = ${directory:log}/neomaster.log
ip = ${publish:ip} ip = ${publish:ip}
port = ${publish:port} port = ${publish:port-master}
cluster = {{ dumps(slapparameter_dict['cluster']) }} cluster = {{ dumps(slapparameter_dict['cluster']) }}
partitions = {{ dumps(slapparameter_dict['partitions']) }} partitions = {{ slapparameter_dict['partitions'] }}
replicas = {{ dumps(slapparameter_dict['replicas']) }} replicas = {{ slapparameter_dict['replicas'] }}
upstream-cluster = {{ dumps(slapparameter_dict['upstream-cluster']) }} upstream-cluster = {{ dumps(slapparameter_dict['upstream-cluster']) }}
upstream-masters = {{ dumps(slapparameter_dict['upstream-masters']) }} upstream-masters = {{ dumps(slapparameter_dict['upstream-masters']) }}
# "masters" parameter is not provided when just requesting a partition. # "masters" parameter is not provided when just requesting a partition.
# No actuall installation takes place at that time # No actual installation takes place at that time
# (slapos.cookbook:neoppod.master raises), but cfg expansion must succeed. So # (slapos.cookbook:neoppod.master raises), but cfg expansion must succeed. So
# this default value is required. # this default value is required.
masters = {{ dumps(slapparameter_dict.get('masters', '')) }} masters = ${publish:masters}
[publish]
recipe = slapos.cookbook:publish
# TODO: make port a partition parameter
ip = {{ (ipv4_set | list)[0] }}
port = 10000
master = ${:ip}:${:port}
[directory]
recipe = slapos.cookbook:mkdirectory
promises = ${buildout:directory}/etc/promises
run = ${buildout:directory}/etc/run
log = ${buildout:directory}/var/log
[neo-master-promise] [neo-master-promise]
recipe = slapos.cookbook:check_port_listening recipe = slapos.cookbook:check_port_listening
......
software/neoppod/instance-neo-storage-mysql.cfg.in
View file @ 61929809
[buildout] {% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%}
extends = {{ master_cfg }} {% set part_list = [] -%}
parts += {% set init_list = [] -%}
neo-storage
logrotate-mysql
logrotate-storage
[mysqld] [mysqld]
recipe = slapos.cookbook:generic.mysql.wrap_mysqld recipe = slapos.cookbook:generic.mysql.wrap_mysqld
...@@ -20,51 +17,87 @@ data-directory = ${mysqld:data-directory} ...@@ -20,51 +17,87 @@ data-directory = ${mysqld:data-directory}
pid-file = ${directory:var_run}/mariadb.pid pid-file = ${directory:var_run}/mariadb.pid
error-log = ${directory:log}/mariadb_error.log error-log = ${directory:log}/mariadb_error.log
slow-query-log = ${directory:log}/mariadb_slowquery.log slow-query-log = ${directory:log}/mariadb_slowquery.log
long-query-time = {{ dumps(slapparameter_dict.get('long-query-time', 1)) }} extra-dict = {{ dumps(slapparameter_dict.get('mysql', {})) }}
relaxed-writes = {{ dumps(slapparameter_dict.get('relaxed-writes', False)) }}
init-file = ${init-script:rendered} init-file = ${init-script:rendered}
[my-cnf] [my-cnf]
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
extensions = jinja2.ext.do
rendered = ${directory:etc}/mariadb.cnf rendered = ${directory:etc}/mariadb.cnf
template = {{ template_my_cnf }} template = {{ template_my_cnf }}
context = section parameter_dict my-cnf-parameters context = section parameter_dict my-cnf-parameters
[init-script] {% set master_list = [] -%}
recipe = slapos.recipe.template:jinja2 {% set admin_list = [] -%}
# XXX: is there a better location ? {% for k, v in slapparameter_dict.iteritems() -%}
rendered = ${directory:etc}/mariadb_initial_setup.sql {% if k.startswith('master-') -%}
database = neo {% do master_list.append(v) -%}
template = inline: {% endif -%}
CREATE DATABASE IF NOT EXISTS ${:database}; {% if k.startswith('admin-') -%}
{% do admin_list.append(v) -%}
{% endif -%}
{% endfor -%}
[publish]
recipe = slapos.cookbook:publish.serialised
# TODO: make port a partition parameter
ip = {{ (ipv4_set | list)[0] }}
port-master = 10000
port-admin = 10002
master = ${:ip}:${:port-master}
admin = ${:ip}:${:port-admin}
masters = {{ ' '.join(sorted(master_list)) }}
{%- if admin_list %}
admins = {{ ' '.join(sorted(admin_list)) }}
{%- endif %}
[neo-storage] [neo-storage]
recipe = slapos.cookbook:neoppod.storage recipe = slapos.cookbook:neoppod.storage
binary = {{ bin_directory }}/neostorage binary = {{ bin_directory }}/neostorage
wrapper = ${directory:etc_run}/neostorage ip = ${publish:ip}
logfile = ${directory:log}/neostorage.log
ip = {{ (ipv4_set | list)[0] }}
cluster = {{ dumps(slapparameter_dict['cluster']) }} cluster = {{ dumps(slapparameter_dict['cluster']) }}
masters = {{ dumps(slapparameter_dict.get('masters', '')) }} masters = ${publish:masters}
database-adapter = MySQL database-adapter = MySQL
database-parameters = root@${init-script:database}${my-cnf-parameters:socket}
wait-database = 60 wait-database = 60
[logrotate-storage]
recipe = slapos.cookbook:logrotate.d
logrotate-entries = ${logrotate:logrotate-entries}
backup = ${logrotate:backup}
{% for i in range(slapparameter_dict.get('storage-count', 1)) -%}
{% set storage_id = 'neo-storage-' ~ i -%}
[{{ section(storage_id) }}]
< = neo-storage
wrapper = ${directory:etc_run}/{{ 'neostorage-' ~ i }}
logfile = ${directory:log}/{{ 'neostorage-' ~ i }}.log
{% do init_list.append('CREATE DATABASE IF NOT EXISTS neo' ~ i ~ ';') -%}
database-parameters = root@neo{{ i }}${my-cnf-parameters:socket}
[{{ section('logrotate-storage-' ~ i) }}]
< = logrotate-storage
name = {{ storage_id }}
log = {{ '${' + storage_id + ':logfile}' }}
post = {{ bin_directory }}/slapos-kill -n neostorage -s RTMIN+1 ${:log}
{% endfor -%}
[init-script]
recipe = slapos.recipe.template:jinja2
# XXX: is there a better location ?
rendered = ${directory:etc}/mariadb_initial_setup.sql
template = inline:
{{ init_list | join('\n\t') }}
[directory] [directory]
recipe = slapos.cookbook:mkdirectory
promises = ${buildout:directory}/etc/promises
bin = ${buildout:directory}/bin bin = ${buildout:directory}/bin
etc = ${buildout:directory}/etc etc = ${buildout:directory}/etc
var = ${buildout:directory}/var var = ${buildout:directory}/var
etc_run = ${:etc}/run etc_run = ${:etc}/run
var_run = ${:var}/run var_run = ${:var}/run
srv_mariadb = ${buildout:directory}/srv/mariadb srv_mariadb = ${buildout:directory}/srv/mariadb
log = ${buildout:directory}/var/log
[logrotate-storage]
recipe = slapos.cookbook:logrotate.d
logrotate-entries = ${logrotate:logrotate-entries}
backup = ${logrotate:backup}
name = neo-storage
log = ${neo-storage:logfile}
post = {{ bin_directory }}/slapos-kill -n neostorage -s RTMIN+1 ${:log}
[logrotate-mysql] [logrotate-mysql]
recipe = slapos.cookbook:logrotate.d recipe = slapos.cookbook:logrotate.d
...@@ -73,3 +106,12 @@ backup = ${logrotate:backup} ...@@ -73,3 +106,12 @@ backup = ${logrotate:backup}
name = mariadb name = mariadb
log = ${my-cnf-parameters:error-log} ${my-cnf-parameters:slow-query-log} log = ${my-cnf-parameters:error-log} ${my-cnf-parameters:slow-query-log}
post = ${mysqld:mysql-base-directory}/bin/mysql --defaults-file="${my-cnf:rendered}" -e "FLUSH LOGS" post = ${mysqld:mysql-base-directory}/bin/mysql --defaults-file="${my-cnf:rendered}" -e "FLUSH LOGS"
[buildout]
extends =
{{ logrotate_cfg }}
{{ master_cfg }}
{{ admin_cfg }}
parts +=
{{ '\n '.join(part_list) }}
logrotate-mysql
software/neoppod/instance.cfg.in
View file @ 61929809
{% set admin_software_type = 'neo-admin' -%}
{% set mysql_storage_software_type = 'neo-storage-mysql' -%}
[buildout] [buildout]
parts = switch-softwaretype parts = switch-softwaretype
eggs-directory = {{ eggs_directory }} eggs-directory = {{ eggs_directory }}
...@@ -28,9 +26,6 @@ context = ...@@ -28,9 +26,6 @@ context =
[neo-cluster] [neo-cluster]
<= jinja2-template-base <= jinja2-template-base
template = {{ cluster }} template = {{ cluster }}
extra-context =
raw admin_software_type {{ admin_software_type }}
raw mysql_storage_software_type {{ mysql_storage_software_type }}
[neo-admin] [neo-admin]
<= jinja2-template-base <= jinja2-template-base
...@@ -45,6 +40,7 @@ template = {{ neo_master }} ...@@ -45,6 +40,7 @@ template = {{ neo_master }}
template = {{ neo_storage_mysql }} template = {{ neo_storage_mysql }}
extra-context = extra-context =
key master_cfg neo-master:rendered key master_cfg neo-master:rendered
key admin_cfg neo-admin:rendered
raw mariadb_location {{ mariadb_location }} raw mariadb_location {{ mariadb_location }}
raw template_my_cnf {{ template_my_cnf }} raw template_my_cnf {{ template_my_cnf }}
...@@ -54,5 +50,4 @@ override = {{ dumps(override_switch_softwaretype |default) }} ...@@ -54,5 +50,4 @@ override = {{ dumps(override_switch_softwaretype |default) }}
default = neo-cluster:rendered default = neo-cluster:rendered
# BBB # BBB
RootSoftwareInstance = ${:default} RootSoftwareInstance = ${:default}
{{ admin_software_type }} = neo-admin:rendered neo = neo-storage-mysql:rendered
{{ mysql_storage_software_type }} = neo-storage-mysql:rendered
software/neoppod/my.cnf.in
View file @ 61929809
{% macro assert(x) %}{{ ("",)[not x] }}{% endmacro -%}
{% set socket = parameter_dict['socket'] -%} {% set socket = parameter_dict['socket'] -%}
{% set extra_dict = parameter_dict['extra-dict'] -%}
[mysqld] [mysqld]
skip_networking skip_networking
socket = {{ socket }} socket = {{ socket }}
...@@ -7,7 +9,6 @@ pid_file = {{ parameter_dict['pid-file'] }} ...@@ -7,7 +9,6 @@ pid_file = {{ parameter_dict['pid-file'] }}
log_error = {{ parameter_dict['error-log'] }} log_error = {{ parameter_dict['error-log'] }}
slow_query_log slow_query_log
slow_query_log_file = {{ parameter_dict['slow-query-log'] }} slow_query_log_file = {{ parameter_dict['slow-query-log'] }}
long_query_time = {{ parameter_dict['long-query-time'] }}
init_file = {{ parameter_dict['init-file'] }} init_file = {{ parameter_dict['init-file'] }}
log_warnings = 1 log_warnings = 1
...@@ -17,17 +18,23 @@ disable-log-bin ...@@ -17,17 +18,23 @@ disable-log-bin
max_allowed_packet = 128M max_allowed_packet = 128M
query_cache_size = 32M query_cache_size = 32M
innodb_file_per_table = 0
innodb_locks_unsafe_for_binlog = 1 innodb_locks_unsafe_for_binlog = 1
# Some dangerous settings you may want to uncomment temporarily # Some dangerous settings you may want to uncomment temporarily
# if you only want performance or less disk access. # if you only want performance or less disk access.
{% set x = '' if parameter_dict['relaxed-writes'] else '#' -%} {% set x = '' if extra_dict.pop('relaxed-writes', False) else '#' -%}
{{x}}innodb_flush_log_at_trx_commit = 0 {{x}}innodb_flush_log_at_trx_commit = 0
{{x}}innodb_flush_method = nosync {{x}}innodb_flush_method = nosync
{{x}}innodb_doublewrite = 0 {{x}}innodb_doublewrite = 0
{{x}}sync_frm = 0 {{x}}sync_frm = 0
# Extra parameters.
{%- do extra_dict.setdefault('innodb_file_per_table', '0') %}
{%- for k, v in extra_dict.iteritems() %}
{%- do assert('-' not in k) %}
{{ k }} = {{ v }}
{%- endfor %}
# Force utf8 usage # Force utf8 usage
collation_server = utf8_unicode_ci collation_server = utf8_unicode_ci
character_set_server = utf8 character_set_server = utf8
......
software/neoppod/software.cfg
View file @ 61929809
...@@ -41,31 +41,31 @@ scripts = ...@@ -41,31 +41,31 @@ scripts =
[cluster] [cluster]
recipe = slapos.recipe.build:download recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_buildout_section_name_}.cfg.in url = ${:_profile_base_location_}/${:_buildout_section_name_}.cfg.in
md5sum = 85e88660335bbdfb13ba9026c24858b0 md5sum = 5e422ddd4ecfe85daa3e151cde261c75
# XXX: following mode should be the default # XXX: following mode should be the default
mode = 644 mode = 644
[instance-neo-admin] [instance-neo-admin]
< = cluster < = cluster
md5sum = 43eb053841e9dabdbed2a3501a0a3f13 md5sum = 16d11f0fe74de06aebbadcff3527db1c
[instance-neo-master] [instance-neo-master]
< = cluster < = cluster
md5sum = 2cba73dbfa3f6e1a60d54de3bae38fee md5sum = 9563ce56676bf9ae5e77fe12e9020289
[instance-neo-storage-mysql] [instance-neo-storage-mysql]
< = cluster < = cluster
md5sum = 70f7dfc268ceb677913c3a318656f834 md5sum = 5f671be9b78c71718a964120a6494c7c
[template-my-cnf] [template-my-cnf]
< = cluster < = cluster
url = ${:_profile_base_location_}/my.cnf.in url = ${:_profile_base_location_}/my.cnf.in
md5sum = 38b4eb7225f9b7c18875b4d2ab398278 md5sum = b8d0c0a6f7a7fe46c8c2f916ee6edced
[template] [template]
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
template = ${:_profile_base_location_}/instance.cfg.in template = ${:_profile_base_location_}/instance.cfg.in
md5sum = 9eb934d9fc06b253799c661f5dfba179 md5sum = 6101da4361793eddde62b6be1639c25e
# XXX: "template.cfg" is hardcoded in instanciation recipe # XXX: "template.cfg" is hardcoded in instanciation recipe
rendered = ${buildout:directory}/template.cfg rendered = ${buildout:directory}/template.cfg
context = context =
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7