Commit 4b92775c authored by Łukasz Nowak's avatar Łukasz Nowak

promise: Improve check_surykatka_json

Improvements:

 * support for ssl_certificate entry from surykatka
 * explanation of extended status codes
 * assurance that URLs are clickable in the monitor.app
 * format times for humans

/reviewed-on nexedi/slapos.toolbox!73
parents ded34b9c 83f161ae
......@@ -7,20 +7,48 @@ import email.utils
import json
import os
import time
try:
from urlparse import urlparse
except ImportError:
from urllib.parse import urlparse
@implementer(interface.IPromise)
class RunPromise(GenericPromise):
EXTENDED_STATUS_CODE_MAPPING = {
'520': 'Too many redirects',
'523': 'Connection error',
'524': 'Connection timeout',
'526': 'SSL Error',
}
def __init__(self, config):
super(RunPromise, self).__init__(config)
# Set frequency compatible to default surykatka interval - 2 minutes
self.setPeriodicity(float(self.getConfig('frequency', 2)))
self.error_list = []
self.info_list = []
def appendError(self, message):
self.error_list.append(message)
def appendInfo(self, message):
self.info_list.append(message)
def emitLog(self):
if len(self.error_list) > 0:
emit = self.logger.error
else:
emit = self.logger.info
emit(' '.join(self.error_list + self.info_list))
def senseBotStatus(self):
key = 'bot_status'
def logError(msg, *args):
self.logger.error(key + ': ' + msg, *args)
self.appendError(key + ': ' + msg % args)
if key not in self.surykatka_json:
logError("%r not in %r", key, self.json_file)
......@@ -35,26 +63,87 @@ class RunPromise(GenericPromise):
return
timetuple = email.utils.parsedate(bot_status['date'])
last_bot_datetime = datetime.datetime.fromtimestamp(time.mktime(timetuple))
last_bot_datetime_string = email.utils.formatdate(time.mktime(timetuple))
delta = self.utcnow - last_bot_datetime
# sanity check
if delta < datetime.timedelta(minutes=0):
logError('Last bot datetime %s is in future, UTC now %s',
last_bot_datetime, self.utcnow)
last_bot_datetime_string, self.utcnow_string)
return
if delta > datetime.timedelta(minutes=15):
logError('Last bot datetime %s is more than 15 minutes old, UTC now %s',
last_bot_datetime, self.utcnow)
last_bot_datetime_string, self.utcnow_string)
return
self.logger.info(
'%s: Last bot status from %s ok, UTC now is %s',
key, last_bot_datetime, self.utcnow)
self.appendInfo(
'%s: Last bot status from %s ok, UTC now is %s' %
(key, last_bot_datetime_string, self.utcnow_string))
def senseSslCertificate(self):
key = 'ssl_certificate'
def appendError(msg, *args):
self.appendError(key + ': ' + msg % args)
url = self.getConfig('url')
parsed_url = urlparse(url)
if parsed_url.scheme == 'https':
hostname = parsed_url.netloc
ssl_check = True
certificate_expiration_days = self.getConfig(
'certificate-expiration-days', '15')
try:
certificate_expiration_days = int(certificate_expiration_days)
except ValueError:
certificate_expiration_days = None
else:
ssl_check = False
certificate_expiration_days = None
if ssl_check is None:
return
if certificate_expiration_days is None:
appendError(
'certificate-expiration-days %r is incorrect',
self.getConfig('certificate-expiration-days'))
return
if not hostname:
appendError('url %r is incorrect', url)
return
if key not in self.surykatka_json:
appendError(
'No data for %s . If the error persist, please update surykatka.', url)
return
entry_list = [
q for q in self.surykatka_json[key] if q['hostname'] == hostname]
if len(entry_list) == 0:
appendError('No data for %s', url)
return
for entry in entry_list:
timetuple = email.utils.parsedate(entry['not_after'])
certificate_expiration_time = datetime.datetime.fromtimestamp(
time.mktime(timetuple))
if certificate_expiration_time - datetime.timedelta(
days=certificate_expiration_days) < self.utcnow:
appendError(
'Certificate for %s will expire on %s, which is less than %s days, '
'UTC now is %s',
url, entry['not_after'], certificate_expiration_days,
self.utcnow_string)
return
else:
self.appendInfo(
'%s: Certificate for %s will expire on %s, which is more than %s '
'days, UTC now is %s' %
(key, url, entry['not_after'], certificate_expiration_days,
self.utcnow_string))
return
def senseHttpQuery(self):
key = 'http_query'
error_list = []
def logError(msg, *args):
self.logger.error(key + ': ' + msg, *args)
self.appendError(key + ': ' + msg % args)
if key not in self.surykatka_json:
logError("%r not in %r", key, self.json_file)
......@@ -66,14 +155,22 @@ class RunPromise(GenericPromise):
entry_list = [q for q in self.surykatka_json[key] if q['url'] == url]
if len(entry_list) == 0:
logError('No data for %r', url)
logError('No data for %s', url)
return
error_list = []
for entry in entry_list:
if str(entry['status_code']) != str(status_code):
entry_status_code = str(entry['status_code'])
if entry_status_code != status_code:
status_code_explanation = self.EXTENDED_STATUS_CODE_MAPPING.get(
entry_status_code)
if status_code_explanation:
status_code_explanation = '%s (%s)' % (
entry_status_code, status_code_explanation)
else:
status_code_explanation = entry_status_code
error_list.append(
'IP %s got status code %s instead of %s' % (
entry['ip'], entry['status_code'], status_code))
entry['ip'], status_code_explanation, status_code))
db_ip_list = [q['ip'] for q in entry_list]
if len(ip_list):
if set(ip_list) != set(db_ip_list):
......@@ -81,16 +178,16 @@ class RunPromise(GenericPromise):
'expected IPs %s differes from got %s' % (
' '.join(ip_list), ' '.join(db_ip_list)))
if len(error_list):
logError('Problem with %s: ' % (url,) + ', '.join(error_list))
logError('Problem with %s : ' % (url,) + ', '.join(error_list))
return
if len(ip_list) > 0:
self.logger.info(
'%s: %s replied correctly with status code %s on ip list %s',
key, url, status_code, ' '.join(ip_list))
self.appendInfo(
'%s: %s replied correctly with status code %s on ip list %s' %
(key, url, status_code, ' '.join(ip_list)))
else:
self.logger.info(
'%s: %s replied correctly with status code %s',
key, url, status_code)
self.appendInfo(
'%s: %s replied correctly with status code %s' %
(key, url, status_code))
def sense(self):
"""
......@@ -100,28 +197,31 @@ class RunPromise(GenericPromise):
if test_utcnow:
self.utcnow = datetime.datetime.fromtimestamp(
time.mktime(email.utils.parsedate(test_utcnow)))
self.utcnow_string = test_utcnow
else:
self.utcnow = datetime.datetime.utcnow()
self.utcnow_string = email.utils.formatdate(time.mktime(
self.utcnow.timetuple()))
self.json_file = self.getConfig('json-file', '')
if not os.path.exists(self.json_file):
self.logger.error('File %r does not exists', self.json_file)
return
self.appendError('File %r does not exists' % self.json_file)
else:
with open(self.json_file) as fh:
try:
self.surykatka_json = json.load(fh)
except Exception:
self.logger.error("Problem loading JSON from %r", self.json_file)
return
self.appendError("Problem loading JSON from %r" % self.json_file)
else:
report = self.getConfig('report')
if report == 'bot_status':
return self.senseBotStatus()
self.senseBotStatus()
elif report == 'http_query':
return self.senseHttpQuery()
self.senseHttpQuery()
self.senseSslCertificate()
else:
self.logger.error("Report %r is not supported", report)
return
self.appendError("Report %r is not supported" % report)
self.emitLog()
def anomaly(self):
return self._test(result_count=3, failure_amount=3)
......@@ -114,8 +114,8 @@ class TestCheckSurykatkaJSONBotStatus(CheckSurykatkaJSONMixin):
self.launcher.run()
self.assertPassedMessage(
self.getPromiseResult(self.promise_name),
"bot_status: Last bot status from 2222-12-13 09:10:11 ok, "
"UTC now is 2222-12-13 09:11:12"
"bot_status: Last bot status from Fri, 13 Dec 2222 08:10:11 -0000 "
"ok, UTC now is Wed, 13 Dec 2222 09:11:12 -0000"
)
def test_bot_status_future(self):
......@@ -140,8 +140,8 @@ class TestCheckSurykatkaJSONBotStatus(CheckSurykatkaJSONMixin):
self.launcher.run()
self.assertFailedMessage(
self.getPromiseResult(self.promise_name),
"bot_status: Last bot datetime 2223-12-13 09:10:11 is in "
"future, UTC now 2222-12-13 09:11:12"
"bot_status: Last bot datetime Sat, 13 Dec 2223 08:10:11 -0000 is "
"in future, UTC now Wed, 13 Dec 2222 09:11:12 -0000"
)
def test_bot_status_old(self):
......@@ -166,8 +166,8 @@ class TestCheckSurykatkaJSONBotStatus(CheckSurykatkaJSONMixin):
self.launcher.run()
self.assertFailedMessage(
self.getPromiseResult(self.promise_name),
"bot_status: Last bot datetime 2223-12-13 09:10:11 is "
"more than 15 minutes old, UTC now 2223-12-13 09:26:12"
"bot_status: Last bot datetime Sat, 13 Dec 2223 08:10:11 -0000 is "
"more than 15 minutes old, UTC now Wed, 13 Dec 2223 09:26:12 -0000"
)
def test_not_bot_status(self):
......@@ -214,7 +214,8 @@ class TestCheckSurykatkaJSONHttpQuery(CheckSurykatkaJSONMixin):
'json-file': self.json_file,
'url': 'https://www.erp5.com/',
'status-code': '302',
'ip-list': '127.0.0.1 127.0.0.2'
'ip-list': '127.0.0.1 127.0.0.2',
'test-utcnow': 'Fri, 27 Dec 2019 15:11:12 -0000'
}
)
self.writeSurykatkaJson("""{
......@@ -237,6 +238,20 @@ class TestCheckSurykatkaJSONHttpQuery(CheckSurykatkaJSONMixin):
"status_code": 200,
"url": "https://www.erp5.org/"
}
],
"ssl_certificate": [
{
"date": "Fri, 27 Dec 2019 14:43:26 -0000",
"hostname": "www.erp5.com",
"ip": "127.0.0.1",
"not_after": "Mon, 13 Jul 2020 12:00:00 -0000"
},
{
"date": "Fri, 27 Dec 2019 14:43:26 -0000",
"hostname": "www.erp5.com",
"ip": "127.0.0.2",
"not_after": "Mon, 13 Jul 2020 12:00:00 -0000"
}
]
}
""")
......@@ -245,7 +260,10 @@ class TestCheckSurykatkaJSONHttpQuery(CheckSurykatkaJSONMixin):
self.assertPassedMessage(
self.getPromiseResult(self.promise_name),
"http_query: https://www.erp5.com/ replied correctly with "
"status code 302 on ip list 127.0.0.1 127.0.0.2"
"status code 302 on ip list 127.0.0.1 127.0.0.2 ssl_certificate: "
"Certificate for https://www.erp5.com/ will expire on Mon, 13 Jul "
"2020 12:00:00 -0000, which is more than 15 days, UTC now is "
"Fri, 27 Dec 2019 15:11:12 -0000"
)
def test_no_ip_list(self):
......@@ -255,6 +273,7 @@ class TestCheckSurykatkaJSONHttpQuery(CheckSurykatkaJSONMixin):
'json-file': self.json_file,
'url': 'https://www.erp5.com/',
'status-code': '302',
'test-utcnow': 'Fri, 27 Dec 2019 15:11:12 -0000'
}
)
self.writeSurykatkaJson("""{
......@@ -277,6 +296,20 @@ class TestCheckSurykatkaJSONHttpQuery(CheckSurykatkaJSONMixin):
"status_code": 200,
"url": "https://www.erp5.org/"
}
],
"ssl_certificate": [
{
"date": "Fri, 27 Dec 2019 14:43:26 -0000",
"hostname": "www.erp5.com",
"ip": "127.0.0.1",
"not_after": "Mon, 13 Jul 2020 12:00:00 -0000"
},
{
"date": "Fri, 27 Dec 2019 14:43:26 -0000",
"hostname": "www.erp5.com",
"ip": "127.0.0.2",
"not_after": "Mon, 13 Jul 2020 12:00:00 -0000"
}
]
}
""")
......@@ -284,8 +317,141 @@ class TestCheckSurykatkaJSONHttpQuery(CheckSurykatkaJSONMixin):
self.launcher.run()
self.assertPassedMessage(
self.getPromiseResult(self.promise_name),
"http_query: https://www.erp5.com/ replied correctly with "
"status code 302"
"http_query: https://www.erp5.com/ replied correctly with status "
"code 302 ssl_certificate: Certificate for https://www.erp5.com/ will "
"expire on Mon, 13 Jul 2020 12:00:00 -0000, which is more than 15 "
"days, UTC now is Fri, 27 Dec 2019 15:11:12 -0000"
)
def test_no_http_query_data(self):
self.writeSurykatkaPromise(
{
'report': 'http_query',
'json-file': self.json_file,
'url': 'https://www.erp5.com/',
'status-code': '302',
'ip-list': '127.0.0.1 127.0.0.2',
'test-utcnow': 'Fri, 27 Dec 2019 15:11:12 -0000'
}
)
self.writeSurykatkaJson("""{
"http_query": [
],
"ssl_certificate": [
{
"date": "Fri, 27 Dec 2019 14:43:26 -0000",
"hostname": "www.erp5.com",
"ip": "127.0.0.1",
"not_after": "Mon, 13 Jul 2020 12:00:00 -0000"
},
{
"date": "Fri, 27 Dec 2019 14:43:26 -0000",
"hostname": "www.erp5.com",
"ip": "127.0.0.2",
"not_after": "Mon, 13 Jul 2020 12:00:00 -0000"
}
]
}
""")
self.configureLauncher()
with self.assertRaises(PromiseError):
self.launcher.run()
self.assertFailedMessage(
self.getPromiseResult(self.promise_name),
"http_query: No data for https://www.erp5.com/ ssl_certificate: "
"Certificate for https://www.erp5.com/ will expire on Mon, 13 Jul "
"2020 12:00:00 -0000, which is more than 15 days, UTC now is "
"Fri, 27 Dec 2019 15:11:12 -0000"
)
def test_no_ssl_certificate_data(self):
self.writeSurykatkaPromise(
{
'report': 'http_query',
'json-file': self.json_file,
'url': 'https://www.erp5.com/',
'status-code': '302',
'ip-list': '127.0.0.1 127.0.0.2',
'test-utcnow': 'Fri, 27 Dec 2019 15:11:12 -0000'
}
)
self.writeSurykatkaJson("""{
"http_query": [
{
"date": "Wed, 11 Dec 2019 09:35:28 -0000",
"ip": "127.0.0.1",
"status_code": 302,
"url": "https://www.erp5.com/"
},
{
"date": "Wed, 11 Dec 2019 09:35:28 -0000",
"ip": "127.0.0.2",
"status_code": 302,
"url": "https://www.erp5.com/"
},
{
"date": "Wed, 11 Dec 2019 09:35:28 -0000",
"ip": "176.31.129.213",
"status_code": 200,
"url": "https://www.erp5.org/"
}
],
"ssl_certificate": [
]
}
""")
self.configureLauncher()
with self.assertRaises(PromiseError):
self.launcher.run()
self.assertFailedMessage(
self.getPromiseResult(self.promise_name),
"ssl_certificate: No data for https://www.erp5.com/ http_query: "
"https://www.erp5.com/ replied correctly with "
"status code 302 on ip list 127.0.0.1 127.0.0.2"
)
def test_no_ssl_certificate(self):
self.writeSurykatkaPromise(
{
'report': 'http_query',
'json-file': self.json_file,
'url': 'https://www.erp5.com/',
'status-code': '302',
'ip-list': '127.0.0.1 127.0.0.2',
'test-utcnow': 'Fri, 27 Dec 2019 15:11:12 -0000'
}
)
self.writeSurykatkaJson("""{
"http_query": [
{
"date": "Wed, 11 Dec 2019 09:35:28 -0000",
"ip": "127.0.0.1",
"status_code": 302,
"url": "https://www.erp5.com/"
},
{
"date": "Wed, 11 Dec 2019 09:35:28 -0000",
"ip": "127.0.0.2",
"status_code": 302,
"url": "https://www.erp5.com/"
},
{
"date": "Wed, 11 Dec 2019 09:35:28 -0000",
"ip": "176.31.129.213",
"status_code": 200,
"url": "https://www.erp5.org/"
}
]
}
""")
self.configureLauncher()
with self.assertRaises(PromiseError):
self.launcher.run()
self.assertFailedMessage(
self.getPromiseResult(self.promise_name),
"ssl_certificate: No data for https://www.erp5.com/ . If the error "
"persist, please update surykatka. http_query: https://www.erp5.com/ "
"replied correctly with status code 302 on ip list 127.0.0.1 127.0.0.2"
)
def test_bad_code(self):
......@@ -295,6 +461,7 @@ class TestCheckSurykatkaJSONHttpQuery(CheckSurykatkaJSONMixin):
'json-file': self.json_file,
'url': 'https://www.erp5.com/',
'status-code': '301',
'test-utcnow': 'Fri, 27 Dec 2019 15:11:12 -0000'
}
)
self.writeSurykatkaJson("""{
......@@ -317,6 +484,20 @@ class TestCheckSurykatkaJSONHttpQuery(CheckSurykatkaJSONMixin):
"status_code": 200,
"url": "https://www.erp5.org/"
}
],
"ssl_certificate": [
{
"date": "Fri, 27 Dec 2019 14:43:26 -0000",
"hostname": "www.erp5.com",
"ip": "127.0.0.1",
"not_after": "Mon, 13 Jul 2020 12:00:00 -0000"
},
{
"date": "Fri, 27 Dec 2019 14:43:26 -0000",
"hostname": "www.erp5.com",
"ip": "127.0.0.2",
"not_after": "Mon, 13 Jul 2020 12:00:00 -0000"
}
]
}
""")
......@@ -325,10 +506,72 @@ class TestCheckSurykatkaJSONHttpQuery(CheckSurykatkaJSONMixin):
self.launcher.run()
self.assertFailedMessage(
self.getPromiseResult(self.promise_name),
"http_query: Problem with https://www.erp5.com/: "
"IP 127.0.0.1 got status code 302 instead of 301"
"http_query: Problem with https://www.erp5.com/ : IP 127.0.0.1 got "
"status code 302 instead of 301 ssl_certificate: Certificate for "
"https://www.erp5.com/ will expire on Mon, 13 Jul 2020 12:00:00 "
"-0000, which is more than 15 days, UTC now is Fri, 27 Dec 2019 "
"15:11:12 -0000"
)
def _test_bad_code_explanation(self, status_code, explanation):
self.writeSurykatkaPromise(
{
'report': 'http_query',
'json-file': self.json_file,
'url': 'https://www.erp5.com/',
'status-code': '301',
'test-utcnow': 'Fri, 27 Dec 2019 15:11:12 -0000'
}
)
self.writeSurykatkaJson("""{
"http_query": [
{
"date": "Wed, 11 Dec 2019 09:35:28 -0000",
"ip": "127.0.0.1",
"status_code": %s,
"url": "https://www.erp5.com/"
}
],
"ssl_certificate": [
{
"date": "Fri, 27 Dec 2019 14:43:26 -0000",
"hostname": "www.erp5.com",
"ip": "127.0.0.1",
"not_after": "Mon, 13 Jul 2020 12:00:00 -0000"
},
{
"date": "Fri, 27 Dec 2019 14:43:26 -0000",
"hostname": "www.erp5.com",
"ip": "127.0.0.2",
"not_after": "Mon, 13 Jul 2020 12:00:00 -0000"
}
]
}
""" % status_code)
self.configureLauncher()
with self.assertRaises(PromiseError):
self.launcher.run()
self.assertFailedMessage(
self.getPromiseResult(self.promise_name),
"http_query: Problem with https://www.erp5.com/ : IP 127.0.0.1 got "
"status code %s instead of 301 ssl_certificate: Certificate for "
"https://www.erp5.com/ will expire on Mon, 13 Jul 2020 12:00:00 "
"-0000, which is more than 15 days, UTC now is Fri, 27 Dec 2019 "
"15:11:12 -0000" % explanation
)
def test_bad_code_explanation_520(self):
self._test_bad_code_explanation(520, '520 (Too many redirects)')
def test_bad_code_explanation_523(self):
self._test_bad_code_explanation(523, '523 (Connection error)')
def test_bad_code_explanation_524(self):
self._test_bad_code_explanation(524, '524 (Connection timeout)')
def test_bad_code_explanation_526(self):
self._test_bad_code_explanation(526, '526 (SSL Error)')
def test_bad_ip(self):
self.writeSurykatkaPromise(
{
......@@ -336,7 +579,8 @@ class TestCheckSurykatkaJSONHttpQuery(CheckSurykatkaJSONMixin):
'json-file': self.json_file,
'url': 'https://www.erp5.com/',
'status-code': '301',
'ip-list': '127.0.0.1 127.0.0.2'
'ip-list': '127.0.0.1 127.0.0.2',
'test-utcnow': 'Fri, 27 Dec 2019 15:11:12 -0000'
}
)
self.writeSurykatkaJson("""{
......@@ -359,6 +603,20 @@ class TestCheckSurykatkaJSONHttpQuery(CheckSurykatkaJSONMixin):
"status_code": 200,
"url": "https://www.erp5.org/"
}
],
"ssl_certificate": [
{
"date": "Fri, 27 Dec 2019 14:43:26 -0000",
"hostname": "www.erp5.com",
"ip": "127.0.0.1",
"not_after": "Mon, 13 Jul 2020 12:00:00 -0000"
},
{
"date": "Fri, 27 Dec 2019 14:43:26 -0000",
"hostname": "www.erp5.com",
"ip": "127.0.0.2",
"not_after": "Mon, 13 Jul 2020 12:00:00 -0000"
}
]
}
""")
......@@ -367,8 +625,11 @@ class TestCheckSurykatkaJSONHttpQuery(CheckSurykatkaJSONMixin):
self.launcher.run()
self.assertFailedMessage(
self.getPromiseResult(self.promise_name),
"http_query: Problem with https://www.erp5.com/: "
"expected IPs 127.0.0.1 127.0.0.2 differes from got 127.0.0.1 127.0.0.4"
"http_query: Problem with https://www.erp5.com/ : expected IPs "
"127.0.0.1 127.0.0.2 differes from got 127.0.0.1 127.0.0.4 "
"ssl_certificate: Certificate for https://www.erp5.com/ will expire "
"on Mon, 13 Jul 2020 12:00:00 -0000, which is more than 15 days, "
"UTC now is Fri, 27 Dec 2019 15:11:12 -0000"
)
def test_bad_ip_status_code(self):
......@@ -378,7 +639,8 @@ class TestCheckSurykatkaJSONHttpQuery(CheckSurykatkaJSONMixin):
'json-file': self.json_file,
'url': 'https://www.erp5.com/',
'status-code': '301',
'ip-list': '127.0.0.1 127.0.0.2'
'ip-list': '127.0.0.1 127.0.0.2',
'test-utcnow': 'Fri, 27 Dec 2019 15:11:12 -0000'
}
)
self.writeSurykatkaJson("""{
......@@ -401,6 +663,20 @@ class TestCheckSurykatkaJSONHttpQuery(CheckSurykatkaJSONMixin):
"status_code": 200,
"url": "https://www.erp5.org/"
}
],
"ssl_certificate": [
{
"date": "Fri, 27 Dec 2019 14:43:26 -0000",
"hostname": "www.erp5.com",
"ip": "127.0.0.1",
"not_after": "Mon, 13 Jul 2020 12:00:00 -0000"
},
{
"date": "Fri, 27 Dec 2019 14:43:26 -0000",
"hostname": "www.erp5.com",
"ip": "127.0.0.2",
"not_after": "Mon, 13 Jul 2020 12:00:00 -0000"
}
]
}
""")
......@@ -409,8 +685,10 @@ class TestCheckSurykatkaJSONHttpQuery(CheckSurykatkaJSONMixin):
self.launcher.run()
self.assertFailedMessage(
self.getPromiseResult(self.promise_name),
"http_query: Problem with https://www.erp5.com/: "
"IP 127.0.0.1 got status code 302 instead of 301, "
"expected IPs 127.0.0.1 127.0.0.2 differes from got "
"127.0.0.1 127.0.0.4"
"http_query: Problem with https://www.erp5.com/ : IP 127.0.0.1 got "
"status code 302 instead of 301, expected IPs 127.0.0.1 127.0.0.2 "
"differes from got 127.0.0.1 127.0.0.4 ssl_certificate: Certificate "
"for https://www.erp5.com/ will expire on Mon, 13 Jul 2020 12:00:00 "
"-0000, which is more than 15 days, UTC now is Fri, 27 Dec 2019 "
"15:11:12 -0000"
)
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment