monitor: run promise and services scripts

stack/monitor/buildout.cfg

@@ -44,20 +44,12 @@ filename = monitor.cfg
 md5sum = 51284c0aeb62eccd37f8a4e1621ee28c
 mode = 0644
 
-[monitor-site-template]
-recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance-monitor.cfg.jinja2.in
-output = ${buildout:directory}/template-monitor.cfg
-filename = instance-monitor.cfg.jinja2.in
-md5sum = 51284c0aeb62eccd37f8a4e1621ee28c
-mode = 0644
-
 [monitor-site-template]
 recipe = slapos.recipe.template:jinja2
 filename = template-monitor.cfg
 template = ${:_profile_base_location_}/instance-monitor.cfg.jinja2.in
 rendered = ${buildout:directory}/template-monitor.cfg
-md5sum = 566c3fad3da5e4db1efcdaf66a3e49af
+md5sum = c792bdee9049d7061e36c12e2343eb87
 context =
     key apache_location apache:location
     key gzip_location gzip:location
@@ -68,15 +60,18 @@ context =
     raw dcron_executable_location ${dcron:location}/sbin/crond
     raw logrotate_executable_location ${logrotate:location}/usr/sbin/logrotate
     raw monitor_httpd_template ${monitor-httpd-conf:location}/${monitor-httpd-conf:filename}
+    raw monitor_service_run ${monitor-service-template-run:location}/${monitor-service-template-run:filename}
+    raw monitor_service_template_conf ${monitor-service-template-conf:location}/${monitor-service-template-conf:filename}
     raw openssl_executable_location ${openssl:location}/bin/openssl
     raw python_executable ${buildout:executable}
+    raw promise_executor_py ${run-promise-py:location}/${run-promise-py:filename}
     raw template_wrapper ${template-wrapper:output}
 
 [monitor-httpd-conf]
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/templates/${:filename}
 download-only = true
-md5sum = f3ac37fa505e6ce34e343f4824779cd9
+md5sum = f24951a2de34d3f3675e8712944a70a0
 filename = monitor-httpd.conf.in
 mode = 0644
 
@@ -88,6 +83,14 @@ md5sum = 24885347932935fac93e8bc2a80ec7a2
 filename = monitor-service.conf.in
 mode = 0644
 
+[monitor-service-template-run]
+recipe = hexagonit.recipe.download
+url = ${:_profile_base_location_}/templates/${:filename}
+download-only = true
+md5sum = eac72b9a452a1fb0a955c7cebb2d3c23
+filename = monitor-service-run.in
+mode = 0644
+
 [monitor-html-static]
 recipe = slapos.recipe.build:download-unpacked
 url = https://nexedi.erp5.net/monitor-5c66a4518b466d45cb31cb7e8a225cde20247589.tar.gz
@@ -98,10 +101,18 @@ strip-top-level-dir = true
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/${:filename}
 download-only = true
-md5sum = 8a5af1a03086788006345e27647c2a1d
+md5sum = 68a17075eeeebabd01965be2ee25721d
 filename = monitor.py.in
 mode = 0644
 
+[run-promise-py]
+recipe = hexagonit.recipe.download
+url = ${:_profile_base_location_}/${:filename}
+download-only = true
+md5sum = 0454e5e28b83da48e52770c977eade8b
+filename = run-promise.py
+mode = 0644
+
 [monitor-httpd-template]
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/${:filename}

stack/monitor/instance-monitor.cfg.jinja2.in

@@ -30,21 +30,11 @@ name = logrotate
 frequency = 0 0 * * *
 command = ${logrotate:wrapper}
 
-
-[cron]
-recipe = slapos.cookbook:cron
-dcrond-binary = ${dcron:location}/sbin/crond
-cron-entries = $${monitor-directory:cron-entries}
-crontabs = $${monitor-directory:crontabs}
-cronstamps = $${monitor-directory:cronstamps}
-catcher = $${cron-simplelogger:wrapper}
-binary = $${monitor-directory:services}/crond
-
 # Add log to cron
 [cron-simplelogger]
 recipe = slapos.cookbook:simplelogger
-wrapper = $${monitor-directory:bin}/cron_simplelogger
-log = $${monitor-directory:log}/cron.log
+wrapper = ${monitor-directory:bin}/cron_simplelogger
+log = ${monitor-directory:log}/cron.log
 
 [directory]
 recipe = slapos.cookbook:mkdirectory
@@ -63,13 +53,16 @@ monitor = ${:srv}/monitor
 [monitor-directory]
 recipe = slapos.cookbook:mkdirectory
 etc = ${directory:etc}
-run = ${directory:scripts}
+run = ${directory:monitor}/run
+#run = ${directory:scripts}
+pids = ${directory:run}/monitor
 cgi-bin = ${directory:monitor}/cgi-bin
 public = ${directory:monitor}/public
 private = ${directory:monitor}/private
 services-conf = ${directory:etc}/monitor.conf.d
 www = ${directory:monitor}/web
 log = ${directory:log}/monitor
+promises = ${directory:monitor}/promise-scripts
 
 [logrotate-directory]
 recipe = slapos.cookbook:mkdirectory
@@ -117,10 +110,14 @@ recipe = slapos.cookbook:zero-knowledge.write
 filename = ${monitor-directory:etc}/monitor.conf
 
 title = ${slap-parameter:monitor-title}
+service-executable-dir = ${monitor-directory:run}
+template-service-run = {{ monitor_service_run }}
 public-folder = ${monitor-directory:public}
 private-folder = ${monitor-directory:private}
 web-folder = ${monitor-static-web:web-dir}
-monitor-json = ${monitor-static-web:web-dir}/monitor.json
+monitor-hal-json = ${monitor-static-web:web-dir}/monitor.haljson
+service-pid-folder = ${monitor-directory:pids}
+crond-folder = ${logrotate-directory:cron-entries}
 public-path-list = 
   ${directory:log}
 private-path-list = 
@@ -165,6 +162,7 @@ wrapper-path = ${directory:bin}/monitor-httpd
 wait-for-files =
   ${ca-directory:certs}/httpd.key
   ${ca-directory:certs}/httpd.crt
+  ${cgi-httpd-graceful-wrapper:rendered}
 
 [cgi-httpd-graceful-wrapper]
 recipe = slapos.recipe.template:jinja2
@@ -191,6 +189,14 @@ context =
   raw python_executable {{ python_executable }}
   key configuration_location monitor:filename
 
+[monitor-run-promise]
+recipe = slapos.recipe.template:jinja2
+template = {{ promise_executor_py }}
+rendered = ${directory:bin}/monitor-promise
+mode = 700
+context = 
+  raw python_executable {{ python_executable }}
+
 [monitor-promise]
 recipe = slapos.cookbook:check_url_available
 path = ${directory:promises}/monitor
@@ -199,6 +205,41 @@ check-secure = 1
 dash_path = {{ dash_executable_location }}
 curl_path = {{ curl_executable_location }}
 
+[promise-parameters-dict]
+
+[monitor-base-promise-conf]
+recipe = slapos.recipe.template:jinja2
+template = {{ monitor_service_template_conf }}
+public-path = 
+context =
+  key name :name
+  key script_path :executable-file
+  key title :title
+  key frequency :frequency
+  key public_path_list :public-path
+  key private_path_list :public-path
+  section parameters promise-parameters-dict
+
+# Create monitor directory for this promise
+[promise-directory]
+recipe = slapos.cookbook:mkdirectory
+monitor-httpd = ${monitor-directory:public}/monitor-httpd
+
+[monitor-monitor-promise]
+recipe = slapos.cookbook:wrapper
+pid-file = ${monitor-directory:pids}/monitor-promise.pid
+status-file = ${promise-directory:monitor-httpd}/status.json
+command-line = ${monitor-run-promise:rendered} ${:pid-file} ${:status-file} ${monitor-promise:path}
+wrapper-path = ${monitor-directory:promises}/monitor-httpd
+
+[conf-monitor-promise]
+<= monitor-base-promise-conf
+rendered = ${monitor-directory:services-conf}/monitor-httpd-promise.conf
+name = monitor-httpd-promise
+title = Monitor promise httpd
+executable-file = ${monitor-monitor-promise:wrapper-path}
+frequency = */5 * * * *
+
 [publish-connection-information]
 recipe = slapos.cookbook:publish
 monitor_url_v6 = ${monitor-httpd-conf:url}
@@ -213,4 +254,5 @@ parts =
   monitor
   start-monitor
   ca-httpd
+  conf-monitor-promise
   publish-connection-information

stack/monitor/monitor.py.in

@@ -46,7 +46,7 @@ def main():
       tmp = softConfigGet(service_config, "service", "interface-url")
       if tmp:
         service_dict["_links"]["interface"] = {"href": tmp}
-  with open(config.get("monitor", "monitor-json"), "w") as fp:
+  with open(config.get("monitor", "monitor-hal-json"), "w") as fp:
     json.dump(monitor_dict, fp)
   # create symlinks from service configurations
   for service_config in service_config_list:
@@ -54,6 +54,24 @@ def main():
     createSymlinksFromConfig((config, "monitor", "private-folder"), (service_config, "service", "private-path-list"))
   # run scripts according to frequency
   # XXX
+  
+  for filename in os.listdir(configuration_folder_location):
+    config_file = os.path.join(configuration_folder_location, filename)
+    script_config = loadConfig(config_file)
+    base_name = script_config.get("service", "name")
+    pid_file = os.path.join(config.get("monitor", "service-pid-folder"),
+                                        "%s.pid" % base_name)
+    service_run_location = os.path.join(config.get("monitor", "service-executable-dir"),
+                                        base_name)
+    cron_run_location = os.path.join(config.get("monitor", "crond-folder"),
+                                      base_name)
+    mapping_dict = {"configuration_location": config_file,
+                    "process_pid_file": pid_file}
+    createServiceWrapper(mapping_dict, service_run_location,
+                        config.get("monitor", "template-service-run"))
+    addCronEntry(cron_run_location,
+                [service_run_location],
+                script_config.get("service", "frequency"))
   return 0
 
 
@@ -80,6 +98,16 @@ def createSymlinksFromConfig(destination_folder_config_tuple, source_list_config
           if e.errno != os.errno.EEXIST:
             raise
 
+def createServiceWrapper(mapping_dict, wrapper_file, template_wrapper):
+  with open(template_wrapper, 'r') as template:
+    with open(wrapper_file, 'w') as wrapper:
+      wrapper.write(template.read() % mapping_dict)
+  os.chmod(wrapper_file, 0700)
+
+def addCronEntry(file_path, command_list, frequency):
+  with open(file_path, 'w') as cfile:
+    cfile.write("%s %s" % (frequency, ' '.join(command_list)))
+
 class Popen(subprocess.Popen):
   def set_timeout(self, timeout):
     self.set_timeout = None # assert we're not called twice

stack/monitor/templates/monitor-service-run.in

+#!/usr/bin/env python
+
+configuration_location = "%(configuration_location)s"
+process_pid_file = "%(process_pid_file)s"
+
+import sys
+import os
+import ConfigParser
+import json
+import subprocess
+
+def loadConfig(config_file):
+  config = ConfigParser.ConfigParser()
+  config.read(config_file)
+  return config
+
+def main():
+  config = loadConfig(configuration_location)
+  exec_env = os.environ.copy()
+  script_path = config.get("service", "script-path")
+  executable_folder = os.path.dirname(script_path)
+  executable = os.path.basename(script_path)
+  parameter_json = os.path.join(os.getcwd(), 'parameters.json')
+  with open(parameter_json, 'w') as fjson:
+    fjson.write(json.dumps(dict(config.items("parameter"))))
+
+  process = subprocess.Popen(
+    [script_path, parameter_json],
+    stdin=None,
+    stdout=subprocess.PIPE,
+    stderr=subprocess.PIPE
+  )
+  with open(process_pid_file, "w") as pidfile:
+    pidfile.write(str(process.pid))
+
+if __name__ == "__main__":
+  sys.exit(main())
\ No newline at end of file