repman: rework software release architecture

component/replication-manager/buildout.cfg

@@ -9,6 +9,7 @@ parts =
 golang  = ${golang1.12:location}
 install =
 buildflags =  -v --tags server --ldflags "-extldflags 'static' -w -s -X main.GoOS=linux -X main.GoArch=amd64 -X main.Version=2.1 -X main.FullVersion=$FULLVERSION -X main.Build=$(date +%FT%T%z) -X main.WithProvisioning=ON -X main.WithOpenSVC=OFF -X main.WithHaproxy=ON -X main.WithMaxscale=ON  -X main.WithMariadbshardproxy=ON -X  main.WithProxysql=ON -X  main.WithSphinx=ON -X main.WithArbitration=OFF -X main.WithArbitrationClient=ON -X main.WithMonitoring=ON -X main.WithHttp=ON -X main.WithBackup=ON -X main.WithMail=ON -X main.WithEnforce=ON -X main.WithDeprecate=ON"
+gcc-bin-directory = ${golang1.12:gcc-bin-directory}
 
 [gowork.goinstall]
 depends_gitfetch  =

software/repman/buildout.hash.cfg

+# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
+# The only allowed lines here are (regexes):
+# - "^#" comments, copied verbatim
+# - "^[" section beginings, copied verbatim
+# - lines containing an "=" sign which must fit in the following categorie.
+#   - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
+#     Copied verbatim.
+#   - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
+#     by the re-generation script.
+#     Re-generated.
+# - other lines are copied verbatim
+# Substitution (${...:...}), extension ([buildout] extends = ...) and
+# section inheritance (< = ...) are NOT supported (but you should really
+# not need these here).
+[instance.cfg]
+filename = instance.cfg.in
+md5sum = 704e79c3e6bb0c5c8081f1438e85bba3
+
+[instance-repman.cfg]
+_update_hash_filename_ = instance-repman.cfg.jinja2.in
+md5sum = 1a4f2657c4f606406024a77b1c97b6ea
+
+[config-toml.in]
+_update_hash_filename_ = templates/config.toml.in
+md5sum = 2f3d76fc9bd201fb32cc13972be72685
\ No newline at end of file

software/repman/instance-repman.cfg.jinja2.in

+
+[instance-parameter]
+recipe = slapos.cookbook:slapconfiguration
+computer = ${slap-connection:computer-id}
+partition = ${slap-connection:partition-id}
+url = ${slap-connection:server-url}
+key = ${slap-connection:key-file}
+cert = ${slap-connection:cert-file}
+
+repman-port = 10005
+
+[slap-configuration]
+# apache-frontend reads from  a part named [slap-configuration]
+recipe = slapos.cookbook:slapconfiguration.serialised
+computer = ${slap-connection:computer-id}
+partition = ${slap-connection:partition-id}
+url = ${slap-connection:server-url}
+key = ${slap-connection:key-file}
+cert = ${slap-connection:cert-file}
+
+
+
+# Create all needed directories, depending on your needs
+[directory]
+recipe = slapos.cookbook:mkdirectory
+home = ${buildout:directory}
+etc = ${:home}/etc
+var = ${:home}/var
+script = ${:etc}/run/
+service = ${:etc}/service
+promise = ${:etc}/promise/
+log = ${:var}/log
+data = ${:var}/lib
+
+[replication-manager-credential]
+recipe = slapos.cookbook:generate.password
+username = admin
+bytes = 12
+
+[replication-manager]
+recipe = slapos.cookbook:wrapper
+# repman service is listening on:
+# - global IPv6 address, and
+# - fixed port
+#
+# NOTE because every computer partition is allocated its own global IPv6
+# address, it is ok to fix the port - different hello-world instances will have
+# different IPv6 addresses and they all will be accessible at the same time.
+port = ${instance-parameter:repman-port}
+ipv6 = ${instance-parameter:ipv6-random}
+# full URL - for convenience
+url = https://[${:ipv6}]:${:port}
+logfile = ${directory:log}/repman.log
+command-line =
+   {{ gowork_bin }}/replication-manager
+   --monitoring-basedir=system
+   --monitoring-sharedir=software_release/src/share
+   --http-root=software_release/src/dashboard
+   --monitoring-datadir=${directory:var}/lib
+   --config=${directory:etc}/config.toml
+   --log-file=${:logfile} monitor
+
+wrapper-path = ${directory:service}/replication-manager
+
+
+[template-jinja2-base]
+recipe = slapos.recipe.template:jinja2
+template = {{ config_toml_in }}
+rendered =${directory:etc}/config.toml
+extra-context =
+context =
+  key service_ipv6 replication-manager:ipv6
+  key service_user replication-manager-credential:username
+  key service_pwd replication-manager-credential:passwd
+
+
+[repman-listen-promise]
+<= monitor-promise-base
+module = check_port_listening
+name = repman_service_listen.py
+config-hostname = ${replication-manager:ipv6}
+config-port = ${instance-parameter:repman-port}
+
+
+[publish-connection-parameter]
+<= monitor-publish
+recipe = slapos.cookbook:publish
+url = ${replication-manager:url}
+username = ${replication-manager-credential:username}
+password = ${replication-manager-credential:passwd}
+
+
+#############################
+#
+# Deploy replication-manager instance
+#
+#############################
+[buildout]
+extends =
+  {{ template_monitor_cfg }}
+parts =
+  directory
+  publish-connection-parameter
+  instance-parameter
+  template-jinja2-base
+  replication-manager
+  repman-listen-promise
+  monitor-base
+
+
+eggs-directory = {{ eggs_directory }}
+develop-eggs-directory = {{ develop_eggs_directory }}
+offline = true

software/repman/instance.cfg.in

-#############################
-#
-# Deploy replication-manager instance
-#
-#############################
+# GitLab "switch-softwaretype" instance
 [buildout]
-parts =
-  directory
-  publish-connection-parameter
-  instance-parameter
-  template-jinja2-base
-  replication-manager
-  repman-promise
+parts = switch-softwaretype
 
-
-# Define egg directories to be the one from Software Release
-# (/opt/slapgrid/...)
-# Always the same.
+# std stuff for slapos instance
 eggs-directory = ${buildout:eggs-directory}
 develop-eggs-directory = ${buildout:develop-eggs-directory}
 offline = true
 
 
-[instance-parameter]
-# Fetch arbitrary parameters defined by the user in SlapOS Master for his instance.
-# We use the slapconfiguration recipe with a few parameters (partition id,
-# computer id, certificate, etc).
-# It will then authenticate to SlapOS Master and fetch the instance parameters.
-# The parameters are accessible from $${instance-parameter:configuration.name-of-parameter}
-# Always the same. Just copy/paste.
-# See docstring of slapos.cookbook:slapconfiguration for more information.
-recipe = slapos.cookbook:slapconfiguration
-computer = $${slap-connection:computer-id}
-partition = $${slap-connection:partition-id}
-url = $${slap-connection:server-url}
-key = $${slap-connection:key-file}
-cert = $${slap-connection:cert-file}
-
-
-[slap-configuration]
-# apache-frontend reads from  a part named [slap-configuration]
-recipe = slapos.cookbook:slapconfiguration.serialised
-computer = $${slap-connection:computer-id}
-partition = $${slap-connection:partition-id}
-url = $${slap-connection:server-url}
-key = $${slap-connection:key-file}
-cert = $${slap-connection:cert-file}
-
-
-
-# Create all needed directories, depending on your needs
-[directory]
-recipe = slapos.cookbook:mkdirectory
-home = $${buildout:directory}
-etc = $${:home}/etc
-var = $${:home}/var
-# Executables put here will be started but not monitored (for startup scripts)
-script = $${:etc}/run/
-# Executables put here will be started and monitored (for daemons)
-service = $${:etc}/service
-# Executables put here will be launched after buildout has completed to see
-# if instance is running
-promise = $${:etc}/promise/
-# Path of the log directory used by our service (see [helloweb])
-log = $${:var}/log
-data = $${:var}/lib
-
-[replication-manager-credential]
-recipe = slapos.cookbook:generate.password
-username = admin
-bytes = 12
+[switch-softwaretype]
+recipe = slapos.cookbook:softwaretype
+repman  = $${template-instance-repman.cfg:rendered}
+default = $${:repman}
 
-[replication-manager]
-# repman service is listening on:
-# - global IPv6 address, and
-# - fixed port
-#
-# NOTE because every computer partition is allocated its own global IPv6
-# address, it is ok to fix the port - different hello-world instances will have
-# different IPv6 addresses and they all will be accessible at the same time.
-port = 10005
-ipv6 = $${instance-parameter:ipv6-random}
-# full URL - for convenience
-url = https://[$${:ipv6}]:$${:port}
 
-# the service will log here
-logfile = $${directory:log}/repman.log
-
-# Actual script that starts the service:
-# This recipe will try to "exec" the command-line after separating parameters.
-recipe = slapos.cookbook:wrapper
-command-line =
-   ${gowork:bin}/replication-manager  --monitoring-basedir=system --monitoring-sharedir=software_release/src/share --http-root=software_release/src/dashboard --monitoring-datadir=$${directory:var}/lib --config=$${directory:etc}/config.toml --log-file=$${:logfile} monitor
-
-# Put this shell script in the "etc/service" directory. Each executable of this
-# repository will be started and monitored by supervisord. If a service
-# exits/crashes, it will trigger a "bang" and cause a re-run of the instance.
-wrapper-path = $${directory:service}/replication-manager
+# macro: render instance-*.cfg from instance-*.cfg.in
+[instance-cfg]
+recipe  = slapos.recipe.template:jinja2
+mode    = 0644
+rendered= $${buildout:directory}/$${:_buildout_section_name_}
+context =
 
-name = config.toml
+    key bin_directory           buildout:bin-directory
+    key eggs_directory          buildout:eggs-directory
+    key develop_eggs_directory  buildout:develop-eggs-directory
 
+# program binaries
+    raw bash_bin                    ${bash:location}/bin/bash
+    raw gowork_bin                  ${gowork:bin}
+    raw template_monitor_cfg        ${monitor2-template:rendered}
 
+# config files
+    raw config_toml_in             ${config-toml.in:target}
 
-[template-jinja2-base]
-recipe = slapos.recipe.template:jinja2
-template = ${download-config-toml:destination}
-rendered =$${directory:etc}/config.toml
+    $${:extra-context}
 extra-context =
-context =
-  key service_ipv6 replication-manager:ipv6
-  key service_user replication-manager-credential:username
-  key service_pwd replication-manager-credential:passwd
-# promise, that checks that helloweb service is alive
-[repman-promise]
-recipe = slapos.cookbook:check_port_listening
-path = $${directory:promise}/replication-manager-ipv6
-hostname= $${replication-manager:ipv6}
-port    = 10005
-
 
-# Publish all the parameters needed for the user to connect to the instance.
-# It can be anything: URL(s), password(s), or arbitrary parameters.
-# Here we'll just echo back the entered name as instance parameter
-[publish-connection-parameter]
-recipe = slapos.cookbook:publish
-url = $${replication-manager:url}
-username = $${replication-manager-credential:username}
-password = $${replication-manager-credential:passwd}
+[template-instance-repman.cfg]
+<= instance-cfg
+template= ${instance-repman.cfg:target}
+extra-context =

software/repman/software.cfg

 [buildout]
 
 extends =
-  ../../stack/slapos.cfg
+  buildout.hash.cfg
   ../../stack/monitor/buildout.cfg
   ../../component/replication-manager/buildout.cfg
+  ../../component/mariadb/buildout.cfg
   ../../component/git/buildout.cfg
+  ../../stack/monitor/buildout.cfg
+  ../../stack/slapos.cfg
 
 parts =
   slapos-cookbook
-  download-config-toml
-  template
+  instance.cfg
+  mariadb
   repman
   repman-repository
 
 
-[template]
-recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/${:filename}
-mode = 0644
-output = ${buildout:directory}/template.cfg
-filename = instance.cfg.in
-
-
-[download-config-toml]
-recipe = slapos.recipe.build:download
-url = ${:_profile_base_location_}/templates/config.toml.in
-destination = ${buildout:directory}/config.toml.in
-mode = 644
-
-# Pin versions of eggs used that are not already pinned by stack/slapos.cfg
-[versions]
-slapos.recipe.template = 4.3
-rubygemsrecipe  = 0.2.2+slapos001
+[instance.cfg]
+recipe  = slapos.recipe.template
+url     = ${:_profile_base_location_}/${:filename}
+output  = ${buildout:directory}/instance.cfg
 
 [repman-repository]
 recipe  = slapos.recipe.build:gitclone
@@ -39,3 +28,19 @@ git-executable = ${git:location}/bin/git
 location = ${buildout:directory}/src
 branch = 2.1
 repository = https://github.com/signal18/replication-manager
+
+[download-file]
+recipe  = slapos.recipe.build:download
+url     = ${:_profile_base_location_}/${:_update_hash_filename_}
+destination = ${buildout:directory}/${:_buildout_section_name_}
+
+[config-toml.in]
+<= download-file
+
+[instance-repman.cfg]
+<= download-file
+
+# Pin versions of eggs used that are not already pinned by stack/slapos.cfg
+[versions]
+slapos.recipe.template = 4.3
+rubygemsrecipe  = 0.2.2+slapos001
\ No newline at end of file