Initial neoppod commit.

setup.py

@@ -103,6 +103,9 @@ setup(name=name,
           'mydumper = slapos.recipe.mydumper:Recipe',
           'mysql = slapos.recipe.mysql:Recipe',
           'nbdserver = slapos.recipe.nbdserver:Recipe',
+          'neoppod.admin = slapos.recipe.neoppod:Admin',
+          'neoppod.master = slapos.recipe.neoppod:Master',
+          'neoppod.storage = slapos.recipe.neoppod:Storage',
           'nosqltestbed = slapos.recipe.nosqltestbed:NoSQLTestBed',
           'notifier.callback = slapos.recipe.notifier:Callback',
           'notifier.notify = slapos.recipe.notifier:Notify',

slapos/recipe/neoppod.py

+from slapos.recipe.librecipe import GenericBaseRecipe
+
+class NeoBaseRecipe(GenericBaseRecipe):
+
+  _binding_port_mandatory = True
+
+  def install(self):
+    options = self.options
+    if not options['masters']:
+      # All parameters are always provided.
+      # This parameter needs special care, because it is initially generated
+      # empty, until all requested master nodes get their partitions
+      # allocated.
+      # Only then can this recipe start succeeding and actually doing anything
+      # useful, as per NEO deploying constraints.
+      raise Exception('"masters" parameter is mandatory')
+    option_list = [
+      options['binary'],
+      '-l', options['logfile'],
+      '-m', options['masters'],
+      '-b', self._getBindingAddress(),
+      # TODO: reuse partition reference for better log readability.
+      #'-n', options['name'],
+      '-c', options['cluster'],
+    ]
+    if options['verbose']:
+      option_list.append('-v')
+    option_list.extend(self._getOptionList())
+    return [self.createPythonScript(
+      options['wrapper'],
+      'slapos.recipe.librecipe.execute.execute',
+      option_list
+    )]
+
+  def _getBindingAddress(self):
+    options = self.options
+    bind = options['ip']
+    if 'port' in options:
+      # Some node types support port auto-allocation when no binding port is
+      # requested.
+      bind = bind + ':' + options['port']
+    elif self._binding_port_mandatory:
+      raise ValueError('"port" option is mandatory.')
+    return bind
+
+  def _getOptionList(self):
+    raise NotImplementedError
+
+class Storage(NeoBaseRecipe):
+
+  _binding_port_mandatory = False
+
+  def _getOptionList(self):
+    return [
+      '-d', self.options['database-parameters'],
+      '-a', self.options['database-adapter'],
+      '-w', self.options['wait-database'],
+    ]
+
+class Admin(NeoBaseRecipe):
+  def _getOptionList(self):
+    return []
+
+class Master(NeoBaseRecipe):
+  def _getOptionList(self):
+    options = self.options
+    return [
+      '-p', options['partitions'],
+      '-r', options['replicas'],
+    ]

software/neoppod/README.txt

+Software types
+==============
+Which software type is an entry point and can be used for root software
+instance.
+
+Parameters are expected to be passed as of *.serialised recipes expect them.
+Minimal parameters::
+
+```
+<?xml version='1.0' encoding='utf-8'?>
+<instance>
+  <parameter id="_">{
+    'cluster': 'dummy',
+  }</parameter>
+</instance>
+```
+
+For each available key in the outmost dict are described below.
+
+default (default)
+-----------------
+Deploy a NEO cluster.
+
+'cluster' (str, mandatory)
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+Cluster unique identifier. Your last line of defense against mixing up neo
+clusters and corrupting your data. Choose a unique value for each of your
+cluster.
+
+'partitions' (int, optional)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Number of partitions. You cannot change this value once you created a cluster.
+Defautls to 12.
+
+'replicas' (int, optional)
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+Number of replicates.
+Defaults to 0 (no resilience).
+
+'verbose' (int, optional)
+~~~~~~~~~~~~~~~~~~~~~~~~~
+Non-zero to increase verbosity.
+Defaults to 0.
+
+'mysql-storage-count' (int, optional)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Number of MySQL-based storage nodes to deploy. One master node is deployed
+along with each storage.
+Defaults to 1.
+
+'admin-count' (int, optional)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Number of admin nodes to deploy.
+Defaults to 1.

software/neoppod/cluster.cfg.in

+{% if slap_software_type not in (
+    mysql_storage_software_type,
+    admin_software_type,
+) -%}
+{% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%}
+{% set part_list = [] -%}
+{% set master_list = [] -%}
+{% set admin_list = [] -%}
+[request-common]
+recipe = slapos.cookbook:request
+software-url = ${slap-connection:software-release-url}
+sla = computer_guid
+sla-computer_guid = ${slap-connection:computer-id}
+server-url = ${slap-connection:server-url}
+key-file = ${slap-connection:key-file}
+cert-file = ${slap-connection:cert-file}
+computer-id = ${slap-connection:computer-id}
+partition-id = ${slap-connection:partition-id}
+config-cluster = {{ slapparameter_dict['cluster'] }}
+config-partitions = {{ slapparameter_dict.get('partitions', 12) }}
+config-replicas = {{ slapparameter_dict.get('replicas', 0) }}
+config-verbose = {{ dumps(slapparameter_dict.get('verbose', 0)) }}
+
+[request-config]
+base = cluster verbose
+master = partitions replicas
+node = masters
+
+[node-base]
+< = request-common
+config = ${request-config:base} ${request-config:node}
+config-masters = ${all-masters:addresses}
+
+[mysql-storage-base]
+< = request-common
+config = ${request-config:base} ${request-config:master}
+software-type = {{ mysql_storage_software_type }}
+
+[mysql-storage-request-base]
+< = mysql-storage-base
+return = master
+
+[admin-base]
+< = node-base
+return = admin
+software-type = {{ admin_software_type }}
+
+[publish]
+recipe = slapos.cookbook:publish
+masters = ${all-masters:addresses}
+admins = ${all-admins:addresses}
+
+{% for node_number in range(slapparameter_dict.get('mysql-storage-count', 1)) -%}
+{%   set section_id = 'storage-%i' % (node_number, ) -%}
+{%   set final_section_id = 'final-' ~ section_id -%}
+{%   do master_list.append(section_id) -%}
+[{{ section_id }}]
+< = mysql-storage-request-base
+name = {{ section_id }}
+
+[{{ section(final_section_id) }}]
+< = mysql-storage-base
+	node-base
+name = {{ section_id }}
+config = ${request-config:base} ${request-config:master} ${request-config:node}
+
+{% endfor -%}
+{% for node_number in range(slapparameter_dict.get('admin-count', 1)) -%}
+{%   set section_id = 'admin-%i' % (node_number, ) -%}
+{%   do admin_list.append(section_id)%}
+[{{ section(section_id) }}]
+< = admin-base
+name = {{ section_id }}
+
+{% endfor -%}
+[buildout]
+parts = 
+	{{ part_list | join('\n\t') }}
+	publish
+eggs-directory = {{ eggs_directory }}
+develop-eggs-directory = {{ develop_eggs_directory }}
+offline = true
+
+[all-masters]
+addresses = {% for master in master_list -%}
+{{   '${' + master + ':connection-master}' -}}
+{%   if not loop.last %} {% endif -%}
+{% endfor %}
+
+[all-admins]
+addresses = {% for admin in admin_list -%}
+{{  '${' + admin + ':connection-admin}' -}}
+{%  if not loop.last %} {% endif -%}
+{% endfor %}
+{% endif -%}

software/neoppod/instance-neo-admin.cfg.in

+[buildout]
+parts =
+  neo-admin-run-wrapper
+  neo-admin-promise
+
+eggs-directory = {{eggs_directory}}
+develop-eggs-directory = {{develop_eggs_directory}}
+offline = true
+
+[neo-admin]
+recipe = slapos.cookbook:neoppod.admin
+binary = {{bin_directory}}/neoadmin
+wrapper = ${directory:bin}/neoadmin
+logfile = ${directory:log}/neoadmin.log
+ip = ${publish:ip}
+port = ${publish:port}
+cluster = ${slap-parameter:cluster}
+masters = ${slap-parameter:masters}
+verbose = ${slap-parameter:verbose}
+
+[publish]
+recipe = slapos.cookbook:publish
+# TODO: make port a partition parameter
+# TODO: stop using slap-network-information
+ip = [${slap-network-information:global-ipv6}]
+port = 10002
+admin = ${:ip}:${:port}
+
+[neo-admin-run-wrapper]
+recipe = slapos.cookbook:symbolic.link
+target-directory = ${directory:run}
+link-binary = ${neo-admin:wrapper}
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+run = ${buildout:directory}/etc/run
+log = ${buildout:directory}/var/log
+bin = ${buildout:directory}/bin
+
+[neo-admin-promise]
+recipe = slapos.cookbook:check_port_listening
+hostname = ${neo-admin:ip}
+port = ${neo-admin:port}
+path = ${directory:promises}/neo-admin-promise

software/neoppod/instance-neo-master.cfg.in

+[buildout]
+parts =
+  neo-master-run-wrapper
+  neo-master-promise
+eggs-directory = {{eggs_directory}}
+develop-eggs-directory = {{develop_eggs_directory}}
+offline = true
+
+[neo-master]
+recipe = slapos.cookbook:neoppod.master
+binary = {{bin_directory}}/neomaster
+wrapper = ${directory:bin}/neomaster
+logfile = ${directory:log}/neomaster.log
+ip = ${publish:ip}
+port = ${publish:port}
+cluster = ${slap-parameter:cluster}
+partitions = ${slap-parameter:partitions}
+replicas = ${slap-parameter:replicas}
+masters = ${slap-parameter:masters}
+verbose = ${slap-parameter:verbose}
+
+[publish]
+recipe = slapos.cookbook:publish
+# TODO: make port a partition parameter
+# TODO: stop using slap-network-information
+ip = [${slap-network-information:global-ipv6}]
+port = 10000
+master = ${:ip}:${:port}
+
+[neo-master-run-wrapper]
+recipe = slapos.cookbook:symbolic.link
+target-directory = ${directory:run}
+link-binary = ${neo-master:wrapper}
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+run = ${buildout:directory}/etc/run
+log = ${buildout:directory}/var/log
+bin = ${buildout:directory}/bin
+
+[neo-master-promise]
+recipe = slapos.cookbook:check_port_listening
+hostname = ${neo-master:ip}
+port = ${neo-master:port}
+path = ${directory:promises}/neo-master-promise
+
+[slap-parameter]
+# "masters" parameter is not provided when just requesting a partition.
+# No actuall installation takes place at that time
+# (slapos.cookbook:neoppod.master raises), but cfg expansion must succeed. So
+# this default value is required.
+masters =

software/neoppod/instance-neo-storage-mysql.cfg.in

+[buildout]
+extends = {{ master_cfg }}
+parts += neo-storage-run-wrapper
+
+[mariadb-instance]
+recipe = slapos.cookbook:generic.mysql
+user = user
+database = neo
+
+conf-file = ${directory:etc}/mariadb.cnf
+# No networking (ip & port options)
+data-directory = ${directory:srv_mariadb}
+pid-file = ${directory:var_run}/mariadb.pid
+socket = ${directory:var_run}/mariadb.sock
+error-log = ${directory:log}/mariadb_error.log
+slow-query-log = ${directory:log}/mariadb_slowquery.log
+
+wrapper = ${directory:etc_run}/mariadb
+update-wrapper = ${directory:etc_run}/mariadb_update
+mysql-base-directory = {{ mariadb_location }}
+mysql-binary = {{ mariadb_location }}/bin/mysql
+mysql-install-binary = {{ mariadb_location }}/scripts/mysql_install_db
+mysql-upgrade-binary = {{ mariadb_location }}/bin/mysql_upgrade
+mysqld-binary = {{ mariadb_location }}/bin/mysqld
+
+mroonga =
+
+[neo-storage]
+recipe = slapos.cookbook:neoppod.storage
+binary = {{ bin_directory }}/neostorage
+wrapper = ${directory:bin}/neostorage
+logfile = ${directory:log}/neostorage.log
+ip = [${slap-network-information:global-ipv6}]
+cluster = ${slap-parameter:cluster}
+masters = ${slap-parameter:masters}
+verbose = ${slap-parameter:verbose}
+database-adapter = MySQL
+database-parameters = ${mariadb-instance:user}:${mariadb-instance:password}@${mariadb-instance:database}${mariadb-instance:socket}
+wait-database = 60
+
+[neo-storage-run-wrapper]
+recipe = slapos.cookbook:symbolic.link
+target-directory = ${directory:etc_run}
+link-binary = ${neo-storage:wrapper}
+
+[directory]
+etc = ${buildout:directory}/etc
+var = ${buildout:directory}/var
+etc_run =  ${:etc}/run
+var_run =  ${:var}/run
+srv_mariadb = ${buildout:directory}/srv/mariadb

software/neoppod/instance.cfg.in

+{% set admin_software_type = 'neo-admin' -%}
+{% set mysql_storage_software_type = 'neo-storage-mysql' -%}
+[buildout]
+parts = switch-softwaretype
+eggs-directory = {{ eggs_directory }}
+develop-eggs-directory = {{ develop_eggs_directory }}
+offline = true
+
+[slap-connection]
+computer-id = ${slap_connection:computer_id}
+partition-id = ${slap_connection:partition_id}
+server-url = ${slap_connection:server_url}
+software-release-url = ${slap_connection:software_release_url}
+key-file = ${slap_connection:key_file}
+cert-file = ${slap_connection:cert_file}
+
+[slap-configuration]
+recipe = slapos.cookbook:slapconfiguration.serialised
+computer = ${slap-connection:computer-id}
+partition = ${slap-connection:partition-id}
+url = ${slap-connection:server-url}
+key = ${slap-connection:key-file}
+cert = ${slap-connection:cert-file}
+
+[neo-cluster]
+recipe = slapos.recipe.template:jinja2
+template = {{ cluster }}
+rendered = ${buildout:parts-directory}/${:_buildout_section_name_}/cluster.cfg
+extensions = jinja2.ext.do
+context =
+    key eggs_directory buildout:eggs-directory
+    key develop_eggs_directory buildout:develop-eggs-directory
+    key slap_software_type slap-configuration:slap-software-type
+    key slapparameter_dict slap-configuration:configuration
+    raw admin_software_type {{ admin_software_type }}
+    raw mysql_storage_software_type {{ mysql_storage_software_type }}
+
+[switch-softwaretype]
+recipe = slapos.cookbook:softwaretype
+default = ${neo-cluster:rendered}
+{{ admin_software_type }} = {{ neo_admin }}
+{{ mysql_storage_software_type }} = {{ neo_storage_mysql }}

software/neoppod/software.cfg

+# Note on COOKBOOK/END COOKBOOK: the areas delimited by those tags should be
+# dropped when a cookbook version containing neoppod recipe is released as an
+# egg on pypi.
+# Note on LXML/END LXML: they delimit areas where lxml magic is needed. lxml is
+# a slapos.cookbook dependency, so it should be fetched automatically. But when
+# automatically fetched, it gets built against system headers/libs, which is
+# forbidden in slapos. So we need to fetch lxml explicitly so it is properly
+# built.
+
+[buildout]
+extends =
+    ../../stack/slapos.cfg
+#COOKBOOK
+    ../../component/git/buildout.cfg
+#END COOKBOOK
+#LXML
+    ../../component/lxml-python/buildout.cfg
+#END LXML
+    ../../component/python-2.7/buildout.cfg
+    ../../component/mariadb/buildout.cfg
+    ../../component/mysql-python/buildout.cfg
+
+develop =
+#COOKBOOK
+# Note: as we develop a path we fetch, first buildout execution will fail as
+# develop couldn't find the clone. Second (and later) runs will succeed.
+    ${:parts-directory}/slapos.cookbook-repository
+#END COOKBOOK
+
+parts =
+#COOKBOOK
+    slapos.cookbook-repository
+    check-recipe
+#END COOKBOOK
+    slapos-deps-eggs
+# NEO & dependencies
+    python2.7
+    mariadb
+    mysql-python
+    neoppod
+# NEO instanciation
+    template-instance
+
+[slapos-deps-eggs]
+recipe = zc.recipe.egg
+eggs =
+#LXML
+    ${lxml-python:egg}
+#END LXML
+#COOKBOOK
+    slapos.cookbook
+#END COOKBOOK
+
+#LXML
+[lxml-python]
+python = python2.7
+#END LXML
+
+#COOKBOOK
+[slapos.cookbook-repository]
+recipe = slapos.recipe.build:gitclone
+repository = http://git.erp5.org/repos/slapos.git
+branch = neoppod
+git-executable = ${git:location}/bin/git
+
+[check-recipe]
+recipe = plone.recipe.command
+stop-on-error = true
+update-command = ${:command}
+command = test -e ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link
+#END COOKBOOK
+
+[base-template]
+recipe = slapos.recipe.template:jinja2
+template = ${:_profile_base_location_}/${:filename}.in
+rendered = ${buildout:directory}/${:filename}
+extensions = jinja2.ext.do
+extra-context =
+context =
+    key eggs_directory buildout:eggs-directory
+    key develop_eggs_directory buildout:develop-eggs-directory
+    key bin_directory buildout:bin-directory
+    ${:extra-context}
+
+[template-neo-master]
+< = base-template
+filename = instance-neo-master.cfg
+md5sum = b48e9446fb8fdce24d954d7d4f4c7460
+
+[template-neo-storage-mysql]
+< = base-template
+filename = instance-neo-storage-mysql.cfg
+md5sum = e767c66e7de58232de30bdf26188efe1
+extra-context =
+    key mariadb_location mariadb:location
+    key master_cfg template-neo-master:rendered
+
+[template-neo-admin]
+< = base-template
+filename = instance-neo-admin.cfg
+md5sum = fcf1a5c937a609a934c8ccd7c1326726
+
+[template-cluster]
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/cluster.cfg.in
+md5sum = fe37b2ff36fee65b44945393f63e51c3
+
+[template-instance]
+< = base-template
+template = ${:_profile_base_location_}/instance.cfg.in
+# XXX: "template.cfg" is hardcoded in instanciation recipe
+filename = template.cfg
+md5sum = 17e761b371487b55b39da1c39ebb1bb6
+extra-context =
+    key neo_master template-neo-master:rendered
+    key neo_storage_mysql template-neo-storage-mysql:rendered
+    key neo_admin template-neo-admin:rendered
+    key cluster template-cluster:target
+
+[neoppod]
+recipe = zc.recipe.egg
+eggs = neoppod[admin, master, storage-mysqldb]