Add missing supervisord service with celery processes

software/patrowl/buildout.hash.cfg

@@ -15,11 +15,11 @@
 
 [instance]
 filename = instance.cfg.in
-md5sum = 1eca3808d5caededd6170e0d2126c66c
+md5sum = 4a40a6a7668905c8d2df6a52ee6e65aa
 
 [template-patrowl]
 filename = instance-patrowl.cfg.jinja2.in
-md5sum = 552bf4e4e4625f6fdefb054d06578cf7
+md5sum = 19f7e8d3e7cd25d62bcd49e40d24e768
 
 [template-patrowl-engines]
 filename = instance-patrowl-engines.cfg.jinja2.in

software/patrowl/instance-patrowl.cfg.jinja2.in

@@ -18,6 +18,65 @@ mqlog = ${:log}/rabbitmq
 mqlib = ${:var}/lib/rabbitmq
 nginx = ${:srv}/nginx
 
+{% import "supervisord" as supervisord with context %}
+{{ supervisord.supervisord("patrowl-controller", buildout_bin_directory, supervisord_conf, use_service_hash=False, ipv4=local_ipv4, port='${instance-parameters:supervisord-port}') }}
+
+# add celery program to supervisord
+[celery-parameters]
+flow-port = 5555
+flow-log = ${directory:log}/celeryd.flower.log
+flow-errlog = ${directory:log}/celeryderr.flower.log
+beat-pid = ${directory:run}/celerybeat.pid
+beat-log = ${directory:log}/celeryd.beat.log
+beat-errlog = ${directory:log}/celeryderr.beat.log
+default-log = ${directory:log}/celeryd.default.log
+default-errlog = ${directory:log}/celeryderr.default.log
+scan-log = ${directory:log}/celeryd.scan.log
+scan-errlog = ${directory:log}/celeryderr.scan.log
+scanmgt-log = ${directory:log}/celeryd.scanmgt.log
+scanmgt-errlog = ${directory:log}/celeryderr.scanmgt.log
+
+[celery-bin]
+recipe = slapos.recipe.template
+inline =
+  #!/bin/bash -e
+  cd ${configure-service:patrowl-directory}
+  source ${patrowl-env.sh:output}
+  export PYTHONPATH=${configure-service:patrowl-directory}:$PYTHONPATH
+  exec {{ celery_bin }} "$@"
+output = ${directory:bin}/celery
+
+# celery-flow
+{% set celery_flow_dict = {"name": "celery-flow", "command": "${celery-bin:output} flower --app=app --port=${celery-parameters:flow-port} --address=" ~ local_ipv4,
+  "stdout_logfile": "${celery-parameters:flow-log}", "stderr_logfile": "${celery-parameters:flow-errlog}",
+  "stopwaitsecs": 60, "startsecs": 5, "autorestart": True, "environment": []} %}
+{{ supervisord.supervisord_program("celery-flow", celery_flow_dict) }}
+
+# celery-beat
+{% set celery_beat_dict = {"name": "celery-beat", "command": "${celery-bin:output} beat --app=app -l info -S django --pidfile=${celery-parameters:beat-pid}",
+  "stdout_logfile": "${celery-parameters:beat-log}", "stderr_logfile": "${celery-parameters:beat-errlog}",
+  "stopwaitsecs": 60, "startsecs": 5, "autorestart": True, "environment": []} %}
+{{ supervisord.supervisord_program("celery-beat", celery_beat_dict) }}
+
+# celery-default
+{% set celery_default_dict = {"name": "celery-default", "command": "${celery-bin:output} worker --hostname=default-node@%%n --app=app -l info -Q default --purge --without-mingle --without-gossip --without-heartbeat -Ofair",
+  "stdout_logfile": "${celery-parameters:default-log}", "stderr_logfile": "${celery-parameters:default-errlog}",
+  "stopwaitsecs": 60, "startsecs": 5, "autorestart": True, "environment": []} %}
+{{ supervisord.supervisord_program("celery-default", celery_default_dict) }}
+
+# celery-scan
+{% set celery_scan_dict = {"name": "celery-scan", "command": "${celery-bin:output} worker --hostname=scan-node@%%n --app=app -l info -Q scan --purge --without-mingle --without-gossip --without-heartbeat -Ofair",
+  "stdout_logfile": "${celery-parameters:scan-log}", "stderr_logfile": "${celery-parameters:scan-errlog}",
+  "stopwaitsecs": 60, "startsecs": 5, "autorestart": True, "environment": []} %}
+{{ supervisord.supervisord_program("celery-scan", celery_scan_dict) }}
+
+# celery-scanmgt
+{% set celery_scanmgt_dict = {"name": "celery-scanmgt", "command": "${celery-bin:output} worker --hostname=scanmgt-node@%%n --app=app -l info -Q scanmgt --purge --without-mingle --without-gossip --without-heartbeat -Ofair",
+  "stdout_logfile": "${celery-parameters:scanmgt-log}", "stderr_logfile": "${celery-parameters:scanmgt-errlog}",
+  "stopwaitsecs": 60, "startsecs": 5, "autorestart": True, "environment": []} %}
+{{ supervisord.supervisord_program("celery-scanmgt", celery_scanmgt_dict) }}
+
+
 [rabbitmq-password]
 recipe = slapos.cookbook:generate.password
 storage-path = ${directory:etc}/.rabbitmq
@@ -173,6 +232,7 @@ inline =
   export ALERTS_AUTO_MISSING_ENABLED=true
   export ASSET_DETECTION_RULES_FILENAME="app/assets_detection_rules.py"
   export ASSET_AUTOTAG_CPE=true
+  export SUPERVISORD_API_URL=http://{{ local_ipv4 }}:${instance-parameters:supervisord-port}
 
 output = ${directory:etc}/.env.sh
 
@@ -293,6 +353,7 @@ admin-password = ${admin-password:passwd}
 
 [instance-parameters]
 admin-email = admin@dev.patrowl.io
+supervisord-port = 9001
 
 
 [buildout]
@@ -300,6 +361,12 @@ extends = {{ template_monitor }}
 
 parts =
   publish-connection-information
+  supervisord-patrowl-controller
+  supervisord-celery-flow
+  supervisord-celery-beat
+  supervisord-celery-default
+  supervisord-celery-scan
+  supervisord-celery-scanmgt
   promise-postgresql
   service-rabbitmq
   rabbimq-configure-script

software/patrowl/instance.cfg.in

@@ -25,6 +25,9 @@ cert = ${slap-connection:cert-file}
 recipe = slapos.recipe.template:jinja2
 output = ${buildout:directory}/${:filename}
 extensions = jinja2.ext.do
+supervisord-lib = {{ supervisord_lib }}
+import-list =
+  file supervisord :supervisord-lib
 extra-context =
 context =
   key develop_eggs_directory buildout:develop-eggs-directory
@@ -36,12 +39,15 @@ context =
   key slapparameter_dict slap-configuration:configuration
   key computer_id slap-configuration:computer
   key partition_id slap-configuration:partition
+  raw buildout_bin_directory {{ bin_directory }}
   raw nginx_bin {{ nginx_bin }}
   raw gunicorn_bin {{ bin_directory }}/gunicorn
+  raw celery_bin {{ bin_directory }}/celery
   raw nginx_conf {{ template_nginx_conf }}
   raw nginx_mime_types {{ nginx_mime_types }}
   raw openssl_bin {{ openssl_location }}/bin/openssl
   raw python_bin {{ bin_directory }}/{{ python_with_eggs }}
+  raw supervisord_conf {{ supervisord_conf }}
   raw template_monitor {{ template_monitor_cfg }}
   raw logrotate_cfg {{ logrotate_cfg }}
   ${:extra-context}

software/patrowl/software.cfg

@@ -7,6 +7,7 @@ extends =
   ../../component/sslscan/buildout.cfg
   ../../component/openssl/buildout.cfg
   ../../component/nginx/buildout.cfg
+  ../../stack/supervisord/buildout.cfg
   ../../stack/monitor/buildout.cfg
   buildout.hash.cfg
 
@@ -59,6 +60,7 @@ eggs =
   ${Flask:egg}
   Flask-Cors
   flasgger
+  flower
   gevent
   greenlet
   itsdangerous
@@ -162,6 +164,8 @@ context =
     key python_with_eggs eggs:interpreter
     key rabbitmq_location rabbitmq:location
     raw sslscan_bin ${sslscan:location}/bin/sslscan
+    key supervisord_lib supervisord-library:target
+    key supervisord_conf supervisord-conf:target
     key template_create_admin template-create_default_admin.py.in:target
     key template_patrowl template-patrowl:target
     key telplate_patrowl_engines template-patrowl-engines:target
@@ -240,6 +244,7 @@ simplejson = 3.16.0
 singledispatch = 3.4.0.3
 # six = 1.10.0
 sqlparse = 0.4.2
+supervisor = 4.2.1
 thehive4py = 1.8.1:whl
 #thehive4py = 1.5.1
 PyJWT = 2.8.0