Commit fb49ccdb authored by Łukasz Nowak's avatar Łukasz Nowak

Feature Caddy Frontend Haproxy Active Check

See merge request !837
parents cc4d556b 7f2e592d
Pipeline #12221 failed with stage
in 0 seconds
......@@ -245,6 +245,15 @@ Necessary to activate cache.
``enable_cache`` is an optional parameter.
backend-active-check-*
~~~~~~~~~~~~~~~~~~~~~~
This set of parameters is used to control the way how the backend checks will be done. Such active checks can be really useful for `stale-if-error` caching technique and especially in case if backend is very slow to reply or to connect to.
`backend-active-check-http-method` can be used to configure the HTTP method used to check the backend. Special method `CONNECT` can be used to check only for connection attempt.
Please be aware that the `backend-active-check-timeout` is really short by default, so in case if `/` of the backend is slow to reply configure proper path with `backend-active-check-http-path` to not mark such backend down too fast, before increasing the check timeout.
Examples
========
......
......@@ -26,11 +26,11 @@ md5sum = e7d7e1448b6420657e953026573311ca
[profile-caddy-replicate]
filename = instance-apache-replicate.cfg.in
md5sum = 2d421ce6def12f7796cfa28f59eef0df
md5sum = c713aa837c5d0af8b59ed65e17c36c3a
[profile-slave-list]
_update_hash_filename_ = templates/apache-custom-slave-list.cfg.in
md5sum = ab143bfa2e20725aa35940c9033fa0ee
md5sum = 8f187ebb6807732c9e40bdf03e6a8113
[profile-replicate-publish-slave-information]
_update_hash_filename_ = templates/replicate-publish-slave-information.cfg.in
......@@ -50,7 +50,7 @@ md5sum = 266f175dbdfc588af7a86b0b1884fe73
[template-backend-haproxy-configuration]
_update_hash_filename_ = templates/backend-haproxy.cfg.in
md5sum = bf40f8d0a049a8dd924ccc731956c87e
md5sum = 7c171979aa1d5c88faafe4993f087edd
[template-log-access]
_update_hash_filename_ = templates/template-log-access.conf.in
......
......@@ -124,6 +124,35 @@ context =
{% elif slave_type not in [None, '', 'default', 'zope', 'redirect', 'notebook', 'websocket'] %}
{% do slave_error_list.append('type:%s is not supported' % (slave_type,)) %}
{% endif %}
{# Check backend-active-check-* #}
{% set backend_active_check = (str(slave.get('backend-active-check', False)) or 'false').lower() %}
{% if backend_active_check in TRUE_VALUES %}
{% set backend_active_check_http_method = slave.get('backend-active-check-http-method') or 'GET' %}
{% if backend_active_check_http_method not in ['GET', 'OPTIONS', 'CONNECT', 'POST'] %}
{% do slave_error_list.append('Wrong backend-active-check-http-method %s' % (backend_active_check_http_method,)) %}
{% endif %}
{% set backend_active_check_http_path = slave.get('backend-active-check-http-path') or '/' %}
{% set backend_active_check_http_version = slave.get('backend-active-check-http-version') or 'HTTP/1.1' %}
{% if backend_active_check_http_version not in ['HTTP/1.1', 'HTTP/1.0'] %}
{% do slave_error_list.append('Wrong backend-active-check-http-version %s' % (backend_active_check_http_version,)) %}
{% endif %}
{% set backend_active_check_timeout = (slave.get('backend-active-check-timeout') or '2') | int(False) %}
{% if backend_active_check_timeout in [False] or backend_active_check_timeout <= 0 %}
{% do slave_error_list.append('Wrong backend-active-check-timeout %s' % (slave.get('backend-active-check-timeout'),)) %}
{% endif %}
{% set backend_active_check_interval = (slave.get('backend-active-check-interval') or '5') | int(False) %}
{% if backend_active_check_interval in [False] or backend_active_check_interval <= 0 %}
{% do slave_error_list.append('Wrong backend-active-check-interval %s' % (slave.get('backend-active-check-interval'),)) %}
{% endif %}
{% set backend_active_check_rise = (slave.get('backend-active-check-rise') or '1') | int(False) %}
{% if backend_active_check_rise in [False] or backend_active_check_rise <= 0 %}
{% do slave_error_list.append('Wrong backend-active-check-rise %s' % (slave.get('backend-active-check-rise'),)) %}
{% endif %}
{% set backend_active_check_fall = (slave.get('backend-active-check-fall') or '1') | int(False) %}
{% if backend_active_check_fall in [False] or backend_active_check_fall <= 0 %}
{% do slave_error_list.append('Wrong backend-active-check-fall %s' % (slave.get('backend-active-check-fall'),)) %}
{% endif %}
{% endif %}
{# Check ciphers #}
{% set slave_cipher_list = slave.get('ciphers', '').strip().split() %}
{% if slave_cipher_list %}
......
......@@ -223,6 +223,68 @@
],
"title": "Authenticate to backend",
"type": "string"
},
"backend-active-check": {
"title": "Backend Active Check",
"description": "Enables active checks of the backend. For HTTP level checks the HTTP code shall be 2xx or 3xx, otherwise backend will be considered down.",
"enum": [
"false",
"true"
],
"default": "false",
"type": "string"
},
"backend-active-check-http-method": {
"title": "Backend Active Check HTTP Metod",
"description": "Selects method to do the active check. CONNECT means that connection will be enough for the check, otherwise it's HTTP method.",
"enum": [
"GET",
"OPTIONS",
"POST",
"CONNECT"
],
"default": "GET",
"type": "string"
},
"backend-active-check-http-path": {
"title": "Backend Active Check HTTP Path",
"description": "A path on which do the active check, unused in case of CONNECT.",
"default": "/",
"type": "string"
},
"backend-active-check-http-version": {
"title": "Backend Active Check HTTP Version",
"description": "A HTTP version to use to check the backend, unused in case of CONNECT.",
"enum": [
"HTTP/1.1",
"HTTP/1.0"
],
"default": "HTTP/1.1",
"type": "string"
},
"backend-active-check-timeout": {
"title": "Backend Active Check Timeout (seconds)",
"description": "A timeout to for the request to be fulfilled, after connection happen.",
"default": "2",
"type": "integer"
},
"backend-active-check-interval": {
"title": "Backend Active Check Interval (seconds)",
"description": "An interval of backend active check.",
"default": "5",
"type": "integer"
},
"backend-active-check-rise": {
"title": "Backend Active Check Rise",
"description": "Amount of correct responses from the backend to consider it up.",
"default": "1",
"type": "integer"
},
"backend-active-check-fall": {
"title": "Backend Active Check Fall",
"description": "Amount of bad responses from the backend to consider it down.",
"default": "1",
"type": "integer"
}
},
"title": "Input Parameters",
......
......@@ -113,6 +113,33 @@ context =
{%- endif %}
{%- endfor %}
{%- do slave_instance.__setitem__('authenticate-to-backend', ('' ~ slave_instance.get('authenticate-to-backend', '')).lower() in TRUE_VALUES) %}
{#- Setup active check #}
{%- do slave_instance.__setitem__('backend-active-check', ('' ~ slave_instance.get('backend-active-check', '')).lower() in TRUE_VALUES) %}
{%- if slave_instance['backend-active-check'] %}
{%- if 'backend-active-check-http-method' not in slave_instance %}
{%- do slave_instance.__setitem__('backend-active-check-http-method', 'GET') %}
{%- endif %}
{%- if 'backend-active-check-http-version' not in slave_instance %}
{%- do slave_instance.__setitem__('backend-active-check-http-version', 'HTTP/1.1') %}
{%- endif %}
{%- if 'backend-active-check-interval' not in slave_instance %}
{%- do slave_instance.__setitem__('backend-active-check-interval', '5') %}
{%- endif %}
{%- if 'backend-active-check-rise' not in slave_instance %}
{%- do slave_instance.__setitem__('backend-active-check-rise', '1') %}
{%- endif %}
{%- if 'backend-active-check-fall' not in slave_instance %}
{%- do slave_instance.__setitem__('backend-active-check-fall', '2') %}
{%- endif %}
{%- if 'backend-active-check-timeout' not in slave_instance %}
{%- do slave_instance.__setitem__('backend-active-check-timeout', '2') %}
{%- endif %}
{%- do slave_instance.__setitem__('backend-active-check-http-path', slave_instance.get('backend-active-check-http-path') or '/') %}
{%- else %}
{%- do slave_instance.__setitem__('backend-active-check-http-method', '') %}
{%- do slave_instance.__setitem__('backend-active-check-http-version', '') %}
{%- do slave_instance.__setitem__('backend-active-check-http-path', '') %}
{%- endif %} {# if backend_active_check #}
{#- Set Up log files #}
{%- do slave_parameter_dict.__setitem__('access_log', '/'.join([caddy_log_directory, '%s_access_log' % slave_reference])) %}
{%- do slave_parameter_dict.__setitem__('error_log', '/'.join([caddy_log_directory, '%s_error_log' % slave_reference])) %}
......
......@@ -100,7 +100,22 @@ backend {{ slave_instance['slave_reference'] }}-{{ scheme }}
timeout server {{ slave_instance['request-timeout'] }}s
timeout connect {{ slave_instance['backend-connect-timeout'] }}s
retries {{ slave_instance['backend-connect-retries'] }}
server {{ slave_instance['slave_reference'] }}-backend {{ hostname }}:{{ port }} {{ ' '.join(ssl_list) }}
{%- set active_check_list = [] %}
{%- set active_check_option_list = [] %}
{%- if slave_instance['backend-active-check'] %}
{%- do active_check_list.append('check') %}
{%- do active_check_list.append('inter %ss' % (slave_instance['backend-active-check-interval'])) %}
{%- do active_check_list.append('rise %s' % (slave_instance['backend-active-check-rise'])) %}
{%- do active_check_list.append('fall %s' % (slave_instance['backend-active-check-fall'])) %}
{%- if slave_instance['backend-active-check-http-method'] != 'CONNECT' %}
{%- do active_check_option_list.append('option httpchk %s %s %s' % (slave_instance['backend-active-check-http-method'], slave_instance['backend-active-check-http-path'] | urlencode, slave_instance['backend-active-check-http-version'])) %}
{%- endif %}
{%- do active_check_option_list.append('timeout check %ss' % (slave_instance['backend-active-check-timeout'])) %}
{%- endif %}
server {{ slave_instance['slave_reference'] }}-backend {{ hostname }}:{{ port }} {{ ' '.join(ssl_list) }} {{ ' ' + ' '.join(active_check_list)}}
{%- for active_check_option in active_check_option_list %}
{{ active_check_option }}
{%- endfor %}
{%- if path %}
http-request set-path {{ path }}%[path]
{%- endif %}
......
......@@ -48,6 +48,7 @@ from slapos.recipe.librecipe import generateHashFromFiles
import xml.etree.ElementTree as ET
import urlparse
import socket
import sys
try:
......@@ -430,6 +431,9 @@ def fakeHTTPResult(domain, real_ip, path, port=HTTP_PORT,
class TestHandler(BaseHTTPRequestHandler):
identification = None
def do_POST(self):
return self.do_GET()
def do_GET(self):
timeout = int(self.headers.dict.get('timeout', '0'))
compress = int(self.headers.dict.get('compress', '0'))
......@@ -6290,6 +6294,46 @@ class TestSlaveRejectReportUnsafeDamaged(SlaveHttpFrontendTestCase):
'ssl_key': '${section:option}ssl_keyunsafe\nunsafe',
'ssl_crt': '${section:option}ssl_crtunsafe\nunsafe',
},
'backend-active-check-http-method': {
'backend-active-check': True,
'backend-active-check-http-method': 'WRONG',
},
'backend-active-check-http-version': {
'backend-active-check': True,
'backend-active-check-http-version': 'WRONG/1.1',
},
'backend-active-check-timeout': {
'backend-active-check': True,
'backend-active-check-timeout': 'WRONG',
},
'backend-active-check-timeout-negative': {
'backend-active-check': True,
'backend-active-check-timeout': '-2',
},
'backend-active-check-interval': {
'backend-active-check': True,
'backend-active-check-interval': 'WRONG',
},
'backend-active-check-interval-negative': {
'backend-active-check': True,
'backend-active-check-interval': '-2',
},
'backend-active-check-rise': {
'backend-active-check': True,
'backend-active-check-rise': 'WRONG',
},
'backend-active-check-rise-negative': {
'backend-active-check': True,
'backend-active-check-rise': '-2',
},
'backend-active-check-fall': {
'backend-active-check': True,
'backend-active-check-fall': 'WRONG',
},
'backend-active-check-fall-negative': {
'backend-active-check': True,
'backend-active-check-fall': '-2',
}
}
def test_master_partition_state(self):
......@@ -6304,8 +6348,8 @@ class TestSlaveRejectReportUnsafeDamaged(SlaveHttpFrontendTestCase):
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
'domain': 'example.com',
'accepted-slave-amount': '7',
'rejected-slave-amount': '14',
'slave-amount': '21',
'rejected-slave-amount': '24',
'slave-amount': '31',
'rejected-slave-dict': {
'_HTTPS-URL': ['slave https-url "https://[fd46::c2ae]:!py!u\'123123\'"'
' invalid'],
......@@ -6340,6 +6384,26 @@ class TestSlaveRejectReportUnsafeDamaged(SlaveHttpFrontendTestCase):
'_EMPTY-BACKEND': [
"slave https-url '' invalid",
"slave url '' invalid"],
'_backend-active-check-fall': [
'Wrong backend-active-check-fall WRONG'],
'_backend-active-check-fall-negative': [
'Wrong backend-active-check-fall -2'],
'_backend-active-check-http-method': [
'Wrong backend-active-check-http-method WRONG'],
'_backend-active-check-http-version': [
'Wrong backend-active-check-http-version WRONG/1.1'],
'_backend-active-check-interval': [
'Wrong backend-active-check-interval WRONG'],
'_backend-active-check-interval-negative': [
'Wrong backend-active-check-interval -2'],
'_backend-active-check-rise': [
'Wrong backend-active-check-rise WRONG'],
'_backend-active-check-rise-negative': [
'Wrong backend-active-check-rise -2'],
'_backend-active-check-timeout': [
'Wrong backend-active-check-timeout WRONG'],
'_backend-active-check-timeout-negative': [
'Wrong backend-active-check-timeout -2'],
},
'warning-slave-dict': {
'_SSL_CA_CRT_ONLY': [
......@@ -7080,3 +7144,139 @@ class TestPassedRequestParameter(HttpFrontendTestCase):
expected_partition_parameter_dict_dict,
partition_parameter_dict_dict
)
class TestSlaveBackendActiveCheck(SlaveHttpFrontendTestCase, TestDataMixin):
@classmethod
def getInstanceParameterDict(cls):
return {
'domain': 'example.com',
'public-ipv4': cls._ipv4_address,
'port': HTTPS_PORT,
'plain_http_port': HTTP_PORT,
'kedifa_port': KEDIFA_PORT,
'caucase_port': CAUCASE_PORT,
'mpm-graceful-shutdown-timeout': 2,
'request-timeout': '12',
}
@classmethod
def getSlaveParameterDictDict(cls):
cls.setUpAssertionDict()
return {
'backend-active-check-disabled': {
'url': cls.backend_url,
},
'backend-active-check-default': {
'url': cls.backend_url,
'backend-active-check': True,
},
'backend-active-check-connect': {
'url': cls.backend_url,
'backend-active-check': True,
'backend-active-check-http-method': 'CONNECT',
},
'backend-active-check-custom': {
'url': cls.backend_url,
'backend-active-check': True,
'backend-active-check-http-method': 'POST',
'backend-active-check-http-path': '/POST-path to be encoded',
'backend-active-check-http-version': 'HTTP/1.0',
'backend-active-check-timeout': '7',
'backend-active-check-interval': '15',
'backend-active-check-rise': '3',
'backend-active-check-fall': '7',
},
}
@classmethod
def setUpAssertionDict(cls):
backend = urlparse.urlparse(cls.backend_url).netloc
cls.assertion_dict = {
'backend-active-check-disabled': """\
backend _backend-active-check-disabled-http
timeout server 12s
timeout connect 5s
retries 3
server _backend-active-check-disabled-backend %s""" % (backend,),
'backend-active-check-connect': """\
backend _backend-active-check-connect-http
timeout server 12s
timeout connect 5s
retries 3
server _backend-active-check-connect-backend %s check inter 5s"""
""" rise 1 fall 2
timeout check 2s""" % (backend,),
'backend-active-check-custom': """\
backend _backend-active-check-custom-http
timeout server 12s
timeout connect 5s
retries 3
server _backend-active-check-custom-backend %s check inter 15s"""
""" rise 3 fall 7
option httpchk POST /POST-path%%20to%%20be%%20encoded HTTP/1.0
timeout check 7s""" % (backend,),
'backend-active-check-default': """\
backend _backend-active-check-default-http
timeout server 12s
timeout connect 5s
retries 3
server _backend-active-check-default-backend %s check inter 5s"""
""" rise 1 fall 2
option httpchk GET / HTTP/1.1
timeout check 2s""" % (backend, )
}
def _get_backend_haproxy_configuration(self):
backend_configuration_file = glob.glob(os.path.join(
self.instance_path, '*', 'etc', 'backend-haproxy.cfg'))[0]
with open(backend_configuration_file) as fh:
return fh.read()
def _test(self, key):
parameter_dict = self.assertSlaveBase(key)
self.assertIn(
self.assertion_dict[key],
self._get_backend_haproxy_configuration()
)
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'],
'test-path/deep/.././deeper',
headers={
'Timeout': '10', # more than default backend-connect-timeout == 5
'Accept-Encoding': 'gzip',
}
)
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
self.assertEqualResultJson(result, 'Path', '/test-path/deeper')
def test_backend_active_check_disabled(self):
self._test('backend-active-check-disabled')
def test_backend_active_check_default(self):
self._test('backend-active-check-default')
def test_backend_active_check_connect(self):
self._test('backend-active-check-connect')
def test_backend_active_check_custom(self):
self._test('backend-active-check-custom')
if __name__ == '__main__':
class HTTP6Server(HTTPServer):
address_family = socket.AF_INET6
ip, port = sys.argv[1], int(sys.argv[2])
if ':' in ip:
klass = HTTP6Server
url_template = 'http://[%s]:%s/'
else:
klass = HTTPServer
url_template = 'http://%s:%s/'
server = klass((ip, port), TestHandler)
print url_template % server.server_address[:2]
server.serve_forever()
T-0/etc/cron.d/logrotate
T-0/etc/cron.d/monitor-configurator
T-0/etc/cron.d/monitor-globalstate
T-0/etc/cron.d/monitor_collect
T-1/etc/cron.d/logrotate
T-1/etc/cron.d/monitor-configurator
T-1/etc/cron.d/monitor-globalstate
T-1/etc/cron.d/monitor_collect
T-2/etc/cron.d/logrotate
T-2/etc/cron.d/monitor-configurator
T-2/etc/cron.d/monitor-globalstate
T-2/etc/cron.d/monitor_collect
T-2/etc/cron.d/trafficserver-logrotate
T-0/var/log/monitor-httpd-access.log
T-0/var/log/monitor-httpd-error.log
T-0/var/log/slapgrid-T-0-error.log
T-1/var/log/expose-csr_id.log
T-1/var/log/kedifa.log
T-1/var/log/monitor-httpd-access.log
T-1/var/log/monitor-httpd-error.log
T-2/var/log/backend-haproxy.log
T-2/var/log/frontend-access.log
T-2/var/log/frontend-error.log
T-2/var/log/httpd-csr_id/expose-csr_id.log
T-2/var/log/httpd/_backend-active-check-connect_access_log
T-2/var/log/httpd/_backend-active-check-connect_backend_log
T-2/var/log/httpd/_backend-active-check-connect_error_log
T-2/var/log/httpd/_backend-active-check-custom_access_log
T-2/var/log/httpd/_backend-active-check-custom_backend_log
T-2/var/log/httpd/_backend-active-check-custom_error_log
T-2/var/log/httpd/_backend-active-check-default_access_log
T-2/var/log/httpd/_backend-active-check-default_backend_log
T-2/var/log/httpd/_backend-active-check-default_error_log
T-2/var/log/httpd/_backend-active-check-disabled_access_log
T-2/var/log/httpd/_backend-active-check-disabled_backend_log
T-2/var/log/httpd/_backend-active-check-disabled_error_log
T-2/var/log/monitor-httpd-access.log
T-2/var/log/monitor-httpd-error.log
T-2/var/log/slave-introspection-access.log
T-2/var/log/slave-introspection-error.log
T-2/var/log/trafficserver/manager.log
T-0/etc/plugin/__init__.py
T-0/etc/plugin/aibcc-user-caucase-updater.py
T-0/etc/plugin/aikc-user-caucase-updater.py
T-0/etc/plugin/buildout-T-0-status.py
T-0/etc/plugin/caucased-backend-client.py
T-0/etc/plugin/check-free-disk-space.py
T-0/etc/plugin/monitor-bootstrap-status.py
T-0/etc/plugin/monitor-http-frontend.py
T-0/etc/plugin/monitor-httpd-listening-on-tcp.py
T-0/etc/plugin/rejected-slave-publish-ip-port-listening.py
T-0/etc/plugin/rejected-slave.py
T-1/etc/plugin/__init__.py
T-1/etc/plugin/buildout-T-1-status.py
T-1/etc/plugin/caucased.py
T-1/etc/plugin/check-free-disk-space.py
T-1/etc/plugin/expose-csr_id-ip-port-listening.py
T-1/etc/plugin/kedifa-http-reply.py
T-1/etc/plugin/monitor-bootstrap-status.py
T-1/etc/plugin/monitor-http-frontend.py
T-1/etc/plugin/monitor-httpd-listening-on-tcp.py
T-1/etc/plugin/promise-logrotate-setup.py
T-2/etc/plugin/__init__.py
T-2/etc/plugin/backend-client-caucase-updater.py
T-2/etc/plugin/backend-haproxy-configuration.py
T-2/etc/plugin/backend_haproxy_http.py
T-2/etc/plugin/backend_haproxy_https.py
T-2/etc/plugin/buildout-T-2-status.py
T-2/etc/plugin/caddy_frontend_ipv4_http.py
T-2/etc/plugin/caddy_frontend_ipv4_https.py
T-2/etc/plugin/caddy_frontend_ipv6_http.py
T-2/etc/plugin/caddy_frontend_ipv6_https.py
T-2/etc/plugin/caucase-updater.py
T-2/etc/plugin/check-free-disk-space.py
T-2/etc/plugin/expose-csr_id-ip-port-listening.py
T-2/etc/plugin/frontend-caddy-configuration-promise.py
T-2/etc/plugin/monitor-bootstrap-status.py
T-2/etc/plugin/monitor-http-frontend.py
T-2/etc/plugin/monitor-httpd-listening-on-tcp.py
T-2/etc/plugin/promise-logrotate-setup.py
T-2/etc/plugin/re6st-connectivity.py
T-2/etc/plugin/slave-introspection-configuration.py
T-2/etc/plugin/slave_introspection_https.py
T-2/etc/plugin/trafficserver-cache-availability.py
T-2/etc/plugin/trafficserver-port-listening.py
T-0/var/run/monitor-httpd.pid
T-1/var/run/kedifa.pid
T-1/var/run/monitor-httpd.pid
T-2/var/run/backend-haproxy-rsyslogd.pid
T-2/var/run/backend-haproxy.pid
T-2/var/run/backend_haproxy_configuration_last_state
T-2/var/run/backend_haproxy_graceful_configuration_state_signature
T-2/var/run/bhlog.sck
T-2/var/run/graceful_configuration_state_signature
T-2/var/run/httpd.pid
T-2/var/run/monitor-httpd.pid
T-2/var/run/slave-introspection.pid
T-2/var/run/slave_introspection_configuration_last_state
T-2/var/run/slave_introspection_graceful_configuration_state_signature
T-0:aibcc-user-caucase-updater-on-watch RUNNING
T-0:aikc-user-caucase-updater-on-watch RUNNING
T-0:bootstrap-monitor EXITED
T-0:caucased-backend-client-{hash-generic}-on-watch RUNNING
T-0:certificate_authority-{hash-generic}-on-watch RUNNING
T-0:crond-{hash-generic}-on-watch RUNNING
T-0:monitor-httpd-{hash-generic}-on-watch RUNNING
T-0:monitor-httpd-graceful EXITED
T-0:rejected-slave-publish-{hash-rejected-slave-publish}-on-watch RUNNING
T-1:bootstrap-monitor EXITED
T-1:caucase-updater-on-watch RUNNING
T-1:caucased-{hash-generic}-on-watch RUNNING
T-1:certificate_authority-{hash-generic}-on-watch RUNNING
T-1:crond-{hash-generic}-on-watch RUNNING
T-1:expose-csr_id-{hash-generic}-on-watch RUNNING
T-1:kedifa-{hash-generic}-on-watch RUNNING
T-1:kedifa-reloader EXITED
T-1:monitor-httpd-{hash-generic}-on-watch RUNNING
T-1:monitor-httpd-graceful EXITED
T-2:6tunnel-11080-{hash-generic}-on-watch RUNNING
T-2:6tunnel-11443-{hash-generic}-on-watch RUNNING
T-2:backend-client-login-certificate-caucase-updater-on-watch RUNNING
T-2:backend-haproxy-{hash-generic}-on-watch RUNNING
T-2:backend-haproxy-rsyslogd-{hash-generic}-on-watch RUNNING
T-2:backend-haproxy-safe-graceful EXITED
T-2:bootstrap-monitor EXITED
T-2:certificate_authority-{hash-generic}-on-watch RUNNING
T-2:crond-{hash-generic}-on-watch RUNNING
T-2:expose-csr_id-{hash-generic}-on-watch RUNNING
T-2:frontend-caddy-safe-graceful EXITED
T-2:frontend_caddy-{hash-caddy-T-2}-on-watch RUNNING
T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
T-2:kedifa-updater-{hash-generic}-on-watch RUNNING
T-2:monitor-httpd-{hash-generic}-on-watch RUNNING
T-2:monitor-httpd-graceful EXITED
T-2:slave-instrospection-nginx-{hash-generic}-on-watch RUNNING
T-2:slave-introspection-safe-graceful EXITED
T-2:trafficserver-{hash-generic}-on-watch RUNNING
T-2:trafficserver-reload EXITED
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment