Skip to content

S
surykatka
Commit 506522e1 authored by Romain Courteaud's avatar Romain Courteaud
Browse files
Options

Store the http response headers

parent 65fd3852
Show whitespace changes
Inline Side-by-side
Showing with 373 additions and 11 deletions
src/surykatka/bot.py
View file @ 506522e1
......@@ -298,6 +298,7 @@ class WebBot:
result_dict["http_query"].append(
{
"status_code": network_change["status_code"],
"http_header_dict": network_change["http_header_dict"],
"total_seconds": network_change["total_seconds"],
"url": network_change["url"],
"ip": network_change["ip"],
......
src/surykatka/db.py
View file @ 506522e1
......@@ -18,7 +18,7 @@
# See https://www.nexedi.com/licensing for rationale and options.
import peewee
from playhouse.sqlite_ext import SqliteExtDatabase
from playhouse.sqlite_ext import SqliteExtDatabase, JSONField
import datetime
from playhouse.migrate import migrate, SqliteMigrator
......@@ -115,6 +115,7 @@ class LogDB:
ip = peewee.TextField()
url = peewee.TextField()
status_code = peewee.IntegerField()
http_header_dict = JSONField(default=dict)
total_seconds = peewee.FloatField(default=0)
class Meta:
......@@ -131,7 +132,7 @@ class LogDB:
def createTables(self):
# http://www.sqlite.org/pragma.html#pragma_user_version
db_version = self._db.pragma("user_version")
expected_version = 3
expected_version = 4
if db_version != expected_version:
with self._db.transaction():
......@@ -152,10 +153,12 @@ class LogDB:
# version 1 without SSL support
self._db.create_tables([self.SslChange])
migrator = SqliteMigrator(self._db)
migration_list = []
if (0 < db_version) and (db_version <= 2):
# version 2 without the http total_seconds column
migrator = SqliteMigrator(self._db)
migrate(
migration_list.append(
migrator.add_column(
"HttpCodeChange",
"total_seconds",
......@@ -163,6 +166,19 @@ class LogDB:
)
)
if (0 < db_version) and (db_version <= 3):
# version 3 without the http header column
migration_list.append(
migrator.add_column(
"HttpCodeChange",
"http_header_dict",
self.HttpCodeChange.http_header_dict,
)
)
if migration_list:
migrate(*migration_list)
if db_version >= expected_version:
raise ValueError("Can not downgrade SQLite DB")
......
src/surykatka/http.py
View file @ 506522e1
......@@ -120,7 +120,17 @@ def calculateSpeedRange(total_seconds, fast, moderate):
return "SLOW"
def logHttpStatus(db, ip, url, code, total_seconds, fast, moderate, status_id):
def logHttpStatus(
db,
ip,
url,
code,
http_header_dict,
total_seconds,
fast,
moderate,
status_id,
):
with db._db.atomic():
try:
......@@ -132,6 +142,7 @@ def logHttpStatus(db, ip, url, code, total_seconds, fast, moderate, status_id):
if (
(previous_entry is None)
or (previous_entry.status_code != code)
or (previous_entry.http_header_dict != http_header_dict)
or (
calculateSpeedRange(
previous_entry.total_seconds, fast, moderate
......@@ -144,6 +155,7 @@ def logHttpStatus(db, ip, url, code, total_seconds, fast, moderate, status_id):
ip=ip,
url=url,
status_code=code,
http_header_dict=http_header_dict,
total_seconds=total_seconds,
)
return previous_entry.status_id
......@@ -182,11 +194,51 @@ def checkHttpStatus(
response = request(
ip_url, headers={"Host": hostname}, version=bot_version, **request_kw
)
# Blacklisted, because of non stability
# 'Date', 'Age', 'Expires'
header_list = [
# Redirect
"Location",
# HTTP Range
"Accept-Ranges",
# HTTP Cache
"Etag",
"Last-Modified",
"Vary",
"Cache-Control",
"Set-Cookie",
"WWW-Authenticate"
# gzip
"Content-Type",
"Content-Encoding",
"Content-Disposition"
# Security
"Content-Security-Policy",
"Referrer-Policy",
"Strict-Transport-Policy",
"Feature-Policy",
"X-Frame-Options",
"X-Content-Type-Options"
# CORS
"Access-Control-Allow-Origin",
"Access-Control-Allow-Methods",
"Access-Control-Allow-Credentials",
"Access-Control-Allow-Headers",
"Access-Control-Expose-Headers",
]
header_dict = {}
for header_key in header_list:
header_value = response.headers.get(header_key, None)
if header_value is not None:
header_dict[header_key] = header_value
logHttpStatus(
db,
ip,
url,
response.status_code,
header_dict,
response.elapsed.total_seconds(),
elapsed_fast,
elapsed_moderate,
......
tests/test_bot.py
View file @ 506522e1
......@@ -115,6 +115,7 @@ class SurykatkaBotTestCase(unittest.TestCase):
"surykatka.http.request"
) as mock_request:
mock_request.return_value.headers = {"Etag": "foobar"}
mock_get_default_resolver.return_value = resolver
mock_query.return_value = [MockAnswer("1.2.3.4")]
mock_create_default_context.return_value.wrap_socket.return_value.getpeercert.side_effect = [
......@@ -178,6 +179,7 @@ class SurykatkaBotTestCase(unittest.TestCase):
"surykatka.http.request"
) as mock_request:
mock_request.return_value.headers = {"Etag": "foobar"}
mock_query.return_value = [MockAnswer("1.2.3.4")]
mock_create_default_context.return_value.wrap_socket.return_value.getpeercert.side_effect = [
b"",
......@@ -244,6 +246,7 @@ class SurykatkaBotTestCase(unittest.TestCase):
"surykatka.http.request"
) as mock_request:
mock_request.return_value.headers = {"Etag": "foobar"}
mock_query.return_value = [MockAnswer("1.2.3.4")]
mock_create_default_context.return_value.wrap_socket.return_value.getpeercert.side_effect = [
b"",
......@@ -312,6 +315,7 @@ class SurykatkaBotTestCase(unittest.TestCase):
"surykatka.http.request"
) as mock_request:
mock_request.return_value.headers = {"Etag": "foobar"}
mock_query.return_value = [
MockAnswer("1.2.3.4"),
MockAnswer("1.2.3.5"),
......@@ -392,6 +396,7 @@ class SurykatkaBotTestCase(unittest.TestCase):
"surykatka.http.request"
) as mock_request:
mock_request.return_value.headers = {"Etag": "foobar"}
mock_query.return_value = [MockAnswer("1.2.3.4")]
mock_create_default_context.return_value.wrap_socket.return_value.getpeercert.side_effect = [
b"",
......@@ -460,6 +465,7 @@ class SurykatkaBotTestCase(unittest.TestCase):
"surykatka.http.request"
) as mock_request:
mock_request.return_value.headers = {"Etag": "foobar"}
mock_query.return_value = [MockAnswer("1.2.3.4")]
mock_create_default_context.return_value.wrap_socket.return_value.getpeercert.side_effect = [
b"",
......@@ -515,6 +521,7 @@ class SurykatkaBotTestCase(unittest.TestCase):
"surykatka.http.request"
) as mock_request:
mock_request.return_value.headers = {"Etag": "foobar"}
mock_query.side_effect = [[MockAnswer("1.2.3.4")], []]
bot.iterateLoop()
......
tests/test_db.py
View file @ 506522e1
......@@ -48,8 +48,10 @@ def validate_schema(model):
for column in intersect:
field = model._meta.columns[column]
db_field = db_model._meta.columns[column]
if (field.field_type != db_field.field_type) and (
not (field.field_type == "BIGINT")
if (
(field.field_type != db_field.field_type)
and (not (field.field_type == "BIGINT"))
and (not field.field_type == "JSON")
):
to_change.append((field, db_field))
......@@ -64,7 +66,7 @@ class SurykatkaDBTestCase(unittest.TestCase):
def test_createTable(self):
assert self.db._db.pragma("user_version") == 0
self.db.createTables()
assert self.db._db.pragma("user_version") == 3
assert self.db._db.pragma("user_version") == 4
def test_downgrade(self):
assert self.db._db.pragma("user_version") == 0
......@@ -97,10 +99,15 @@ class SurykatkaDBTestCase(unittest.TestCase):
"HttpCodeChange", "total_seconds"
),
)
migrate(
SqliteMigrator(self.db._db).drop_column(
"HttpCodeChange", "http_header_dict"
),
)
self.db._db.pragma("user_version", 1)
self.db.createTables()
assert self.db._db.pragma("user_version") == 3
assert self.db._db.pragma("user_version") == 4
assert validate_schema(self.db.SslChange).valid, validate_schema(
self.db.SslChange
)
......@@ -126,10 +133,44 @@ class SurykatkaDBTestCase(unittest.TestCase):
"HttpCodeChange", "total_seconds"
),
)
migrate(
SqliteMigrator(self.db._db).drop_column(
"HttpCodeChange", "http_header_dict"
),
)
self.db._db.pragma("user_version", 2)
self.db.createTables()
assert self.db._db.pragma("user_version") == 3
assert self.db._db.pragma("user_version") == 4
assert validate_schema(self.db.HttpCodeChange).valid, validate_schema(
self.db.HttpCodeChange
)
def test_migrationFromVersion3(self):
assert self.db._db.pragma("user_version") == 0
# Recreate version 3
with self.db._db.transaction():
self.db._db.create_tables(
[
self.db.Status,
self.db.ConfigurationChange,
self.db.HttpCodeChange,
self.db.NetworkChange,
self.db.PlatformChange,
self.db.DnsChange,
self.db.SslChange,
]
)
migrate(
SqliteMigrator(self.db._db).drop_column(
"HttpCodeChange", "http_header_dict"
),
)
self.db._db.pragma("user_version", 3)
self.db.createTables()
assert self.db._db.pragma("user_version") == 4
assert validate_schema(self.db.HttpCodeChange).valid, validate_schema(
self.db.HttpCodeChange
)
......
tests/test_http.py
View file @ 506522e1
......@@ -184,6 +184,7 @@ class SurykatkaHttpTestCase(unittest.TestCase):
ip = "127.0.0.1"
url = "http://example.org"
status_code = 200
http_header_dict = {"a": "b"}
total_seconds = 0.1
fast = 0.2
moderate = 0.5
......@@ -193,6 +194,7 @@ class SurykatkaHttpTestCase(unittest.TestCase):
ip,
url,
status_code,
http_header_dict,
total_seconds,
fast,
moderate,
......@@ -202,6 +204,7 @@ class SurykatkaHttpTestCase(unittest.TestCase):
assert self.db.HttpCodeChange.get().ip == ip
assert self.db.HttpCodeChange.get().url == url
assert self.db.HttpCodeChange.get().status_code == status_code
assert self.db.HttpCodeChange.get().http_header_dict == {"a": "b"}
assert self.db.HttpCodeChange.get().status_id == status_id
assert self.db.HttpCodeChange.get().total_seconds == total_seconds
assert result == status_id
......@@ -210,6 +213,8 @@ class SurykatkaHttpTestCase(unittest.TestCase):
ip = "127.0.0.1"
url = "http://example.org"
status_code = 200
http_header_dict = {"a": "b"}
http_header_dict_2 = {"a2": "b2"}
total_seconds = 0.1
fast = 0.2
moderate = 0.5
......@@ -219,6 +224,7 @@ class SurykatkaHttpTestCase(unittest.TestCase):
ip,
url,
status_code,
http_header_dict,
total_seconds,
fast,
moderate,
......@@ -230,6 +236,7 @@ class SurykatkaHttpTestCase(unittest.TestCase):
ip,
url,
status_code + 1,
http_header_dict_2,
total_seconds + 1,
fast,
moderate,
......@@ -245,6 +252,7 @@ class SurykatkaHttpTestCase(unittest.TestCase):
ip = "127.0.0.1"
url = "http://example.org"
status_code = 200
http_header_dict = {"a": "b"}
total_seconds = 0.1
fast = 0.2
moderate = 0.5
......@@ -254,6 +262,7 @@ class SurykatkaHttpTestCase(unittest.TestCase):
ip,
url,
status_code,
http_header_dict,
total_seconds,
fast,
moderate,
......@@ -265,6 +274,7 @@ class SurykatkaHttpTestCase(unittest.TestCase):
ip,
url,
status_code,
http_header_dict,
total_seconds,
fast,
moderate,
......@@ -275,6 +285,7 @@ class SurykatkaHttpTestCase(unittest.TestCase):
assert self.db.HttpCodeChange.get().ip == ip
assert self.db.HttpCodeChange.get().url == url
assert self.db.HttpCodeChange.get().status_code == status_code
assert self.db.HttpCodeChange.get().http_header_dict == {"a": "b"}
assert self.db.HttpCodeChange.get().total_seconds == total_seconds
assert self.db.HttpCodeChange.get().status_id == status_id
......@@ -282,6 +293,7 @@ class SurykatkaHttpTestCase(unittest.TestCase):
ip = "127.0.0.1"
url = "http://example.org"
status_code = 200
http_header_dict = {"a": "b"}
total_seconds = 0.1
fast = 0.2
moderate = 0.5
......@@ -292,6 +304,7 @@ class SurykatkaHttpTestCase(unittest.TestCase):
ip,
url,
status_code,
http_header_dict,
total_seconds,
fast,
moderate,
......@@ -303,6 +316,7 @@ class SurykatkaHttpTestCase(unittest.TestCase):
ip,
url,
status_code_2,
http_header_dict,
total_seconds,
fast,
moderate,
......@@ -328,6 +342,12 @@ class SurykatkaHttpTestCase(unittest.TestCase):
).status_code
== status_code
)
assert (
self.db.HttpCodeChange.get(
self.db.HttpCodeChange.status == status_id
).http_header_dict
== http_header_dict
)
assert (
self.db.HttpCodeChange.get(
self.db.HttpCodeChange.status == status_id
......@@ -352,6 +372,108 @@ class SurykatkaHttpTestCase(unittest.TestCase):
).status_code
== status_code_2
)
assert (
self.db.HttpCodeChange.get(
self.db.HttpCodeChange.status == status_id_2
).http_header_dict
== http_header_dict
)
assert (
self.db.HttpCodeChange.get(
self.db.HttpCodeChange.status == status_id_2
).total_seconds
== total_seconds
)
def test_logHttpStatus_insertWhenDifferentHttpHeader(self):
ip = "127.0.0.1"
url = "http://example.org"
status_code = 200
http_header_dict = {"a": "b"}
total_seconds = 0.1
fast = 0.2
moderate = 0.5
http_header_dict_2 = {"a2": "b2"}
status_id = logStatus(self.db, "foo")
result = logHttpStatus(
self.db,
ip,
url,
status_code,
http_header_dict,
total_seconds,
fast,
moderate,
status_id,
)
status_id_2 = logStatus(self.db, "foo")
result_2 = logHttpStatus(
self.db,
ip,
url,
status_code,
http_header_dict_2,
total_seconds,
fast,
moderate,
status_id_2,
)
assert result_2 != result
assert self.db.HttpCodeChange.select().count() == 2
assert (
self.db.HttpCodeChange.get(
self.db.HttpCodeChange.status == status_id
).ip
== ip
)
assert (
self.db.HttpCodeChange.get(
self.db.HttpCodeChange.status == status_id
).url
== url
)
assert (
self.db.HttpCodeChange.get(
self.db.HttpCodeChange.status == status_id
).status_code
== status_code
)
assert (
self.db.HttpCodeChange.get(
self.db.HttpCodeChange.status == status_id
).http_header_dict
== http_header_dict
)
assert (
self.db.HttpCodeChange.get(
self.db.HttpCodeChange.status == status_id
).total_seconds
== total_seconds
)
assert (
self.db.HttpCodeChange.get(
self.db.HttpCodeChange.status == status_id_2
).ip
== ip
)
assert (
self.db.HttpCodeChange.get(
self.db.HttpCodeChange.status == status_id_2
).url
== url
)
assert (
self.db.HttpCodeChange.get(
self.db.HttpCodeChange.status == status_id_2
).status_code
== status_code
)
assert (
self.db.HttpCodeChange.get(
self.db.HttpCodeChange.status == status_id_2
).http_header_dict
== http_header_dict_2
)
assert (
self.db.HttpCodeChange.get(
self.db.HttpCodeChange.status == status_id_2
......@@ -363,6 +485,7 @@ class SurykatkaHttpTestCase(unittest.TestCase):
ip = "127.0.0.1"
url = "http://example.org"
status_code = 200
http_header_dict = {"a": "b"}
total_seconds_error = 0
total_seconds_fast = 4
total_seconds_moderate = 5
......@@ -375,6 +498,7 @@ class SurykatkaHttpTestCase(unittest.TestCase):
ip,
url,
status_code,
http_header_dict,
total_seconds_error,
fast,
moderate,
......@@ -386,6 +510,7 @@ class SurykatkaHttpTestCase(unittest.TestCase):
ip,
url,
status_code,
http_header_dict,
total_seconds_fast,
fast,
moderate,
......@@ -397,6 +522,7 @@ class SurykatkaHttpTestCase(unittest.TestCase):
ip,
url,
status_code,
http_header_dict,
total_seconds_moderate,
fast,
moderate,
......@@ -408,6 +534,7 @@ class SurykatkaHttpTestCase(unittest.TestCase):
ip,
url,
status_code,
http_header_dict,
total_seconds_slow,
fast,
moderate,
......@@ -448,6 +575,7 @@ class SurykatkaHttpTestCase(unittest.TestCase):
ip = "127.0.0.1"
url = "http://example.org"
status_code = 200
http_header_dict = {"a": "b"}
total_seconds_error = 0
total_seconds_fast = 0.001
total_seconds_moderate = 0.2
......@@ -460,6 +588,7 @@ class SurykatkaHttpTestCase(unittest.TestCase):
ip,
url,
status_code,
http_header_dict,
total_seconds_error,
fast,
moderate,
......@@ -471,6 +600,7 @@ class SurykatkaHttpTestCase(unittest.TestCase):
ip,
url,
status_code,
http_header_dict,
total_seconds_fast,
fast,
moderate,
......@@ -482,6 +612,7 @@ class SurykatkaHttpTestCase(unittest.TestCase):
ip,
url,
status_code,
http_header_dict,
total_seconds_moderate,
fast,
moderate,
......@@ -493,6 +624,7 @@ class SurykatkaHttpTestCase(unittest.TestCase):
ip,
url,
status_code,
http_header_dict,
total_seconds_slow,
fast,
moderate,
......@@ -533,6 +665,7 @@ class SurykatkaHttpTestCase(unittest.TestCase):
ip = "127.0.0.1"
url = "http://example.org"
status_code = 200
http_header_dict = {"a": "b"}
total_seconds = 0.001
total_seconds_2 = 0.199
fast = 0.2
......@@ -543,6 +676,7 @@ class SurykatkaHttpTestCase(unittest.TestCase):
ip,
url,
status_code,
http_header_dict,
total_seconds,
fast,
moderate,
......@@ -554,6 +688,7 @@ class SurykatkaHttpTestCase(unittest.TestCase):
ip,
url,
status_code,
http_header_dict,
total_seconds_2,
fast,
moderate,
......@@ -564,6 +699,9 @@ class SurykatkaHttpTestCase(unittest.TestCase):
assert self.db.HttpCodeChange.get().ip == ip
assert self.db.HttpCodeChange.get().url == url
assert self.db.HttpCodeChange.get().status_code == status_code
assert (
self.db.HttpCodeChange.get().http_header_dict == http_header_dict
)
assert self.db.HttpCodeChange.get().total_seconds == total_seconds
assert self.db.HttpCodeChange.get().status_id == status_id
......@@ -571,6 +709,7 @@ class SurykatkaHttpTestCase(unittest.TestCase):
ip = "127.0.0.1"
url = "http://example.org"
status_code = 200
http_header_dict = {"a": "b"}
total_seconds = 0.2
total_seconds_2 = 0.499
fast = 0.2
......@@ -581,6 +720,7 @@ class SurykatkaHttpTestCase(unittest.TestCase):
ip,
url,
status_code,
http_header_dict,
total_seconds,
fast,
moderate,
......@@ -592,6 +732,7 @@ class SurykatkaHttpTestCase(unittest.TestCase):
ip,
url,
status_code,
http_header_dict,
total_seconds_2,
fast,
moderate,
......@@ -602,6 +743,9 @@ class SurykatkaHttpTestCase(unittest.TestCase):
assert self.db.HttpCodeChange.get().ip == ip
assert self.db.HttpCodeChange.get().url == url
assert self.db.HttpCodeChange.get().status_code == status_code
assert (
self.db.HttpCodeChange.get().http_header_dict == http_header_dict
)
assert self.db.HttpCodeChange.get().total_seconds == total_seconds
assert self.db.HttpCodeChange.get().status_id == status_id
......@@ -609,6 +753,7 @@ class SurykatkaHttpTestCase(unittest.TestCase):
ip = "127.0.0.1"
url = "http://example.org"
status_code = 200
http_header_dict = {"a": "b"}
total_seconds = 0.5
total_seconds_2 = 99
fast = 0.2
......@@ -619,6 +764,7 @@ class SurykatkaHttpTestCase(unittest.TestCase):
ip,
url,
status_code,
http_header_dict,
total_seconds,
fast,
moderate,
......@@ -630,6 +776,7 @@ class SurykatkaHttpTestCase(unittest.TestCase):
ip,
url,
status_code,
http_header_dict,
total_seconds_2,
fast,
moderate,
......@@ -640,6 +787,9 @@ class SurykatkaHttpTestCase(unittest.TestCase):
assert self.db.HttpCodeChange.get().ip == ip
assert self.db.HttpCodeChange.get().url == url
assert self.db.HttpCodeChange.get().status_code == status_code
assert (
self.db.HttpCodeChange.get().http_header_dict == http_header_dict
)
assert self.db.HttpCodeChange.get().total_seconds == total_seconds
assert self.db.HttpCodeChange.get().status_id == status_id
......@@ -650,6 +800,7 @@ class SurykatkaHttpTestCase(unittest.TestCase):
url_2 = url + "2"
total_seconds = 0.1
status_code = 200
http_header_dict = {"a": "b"}
fast = 0.2
moderate = 0.5
status_id = logStatus(self.db, "foo")
......@@ -658,6 +809,7 @@ class SurykatkaHttpTestCase(unittest.TestCase):
ip,
url,
status_code,
http_header_dict,
total_seconds,
fast,
moderate,
......@@ -668,6 +820,7 @@ class SurykatkaHttpTestCase(unittest.TestCase):
ip_2,
url,
status_code,
http_header_dict,
total_seconds,
fast,
moderate,
......@@ -678,6 +831,7 @@ class SurykatkaHttpTestCase(unittest.TestCase):
ip,
url_2,
status_code,
http_header_dict,
total_seconds,
fast,
moderate,
......@@ -688,6 +842,7 @@ class SurykatkaHttpTestCase(unittest.TestCase):
ip_2,
url_2,
status_code,
http_header_dict,
total_seconds,
fast,
moderate,
......@@ -704,7 +859,10 @@ class SurykatkaHttpTestCase(unittest.TestCase):
url = "http://example.org/foo?bar=1"
bot_version = 1
httpretty.register_uri(
httpretty.GET, "http://127.0.0.1/foo?bar=1", status=418
httpretty.GET,
"http://127.0.0.1/foo?bar=1",
status=418,
adding_headers={"Etag": "bar"},
)
status_id = logStatus(self.db, "foo")
......@@ -726,6 +884,7 @@ class SurykatkaHttpTestCase(unittest.TestCase):
assert self.db.HttpCodeChange.get().ip == ip
assert self.db.HttpCodeChange.get().url == url
assert self.db.HttpCodeChange.get().status_code == 418
assert self.db.HttpCodeChange.get().http_header_dict == {"Etag": "bar"}
assert self.db.HttpCodeChange.get().status_id == status_id
def test_checkHttpStatus_https(self):
......@@ -735,6 +894,8 @@ class SurykatkaHttpTestCase(unittest.TestCase):
status_id = logStatus(self.db, "foo")
with mock.patch("surykatka.http.request") as mock_request:
mock_request.return_value.headers = {"Etag": "foobar"}
checkHttpStatus(self.db, status_id, url, ip, bot_version)
assert mock_request.call_count == 1
......@@ -756,6 +917,9 @@ class SurykatkaHttpTestCase(unittest.TestCase):
assert self.db.HttpCodeChange.get().url == url
# XXX No idea how to mock SSL
assert self.db.HttpCodeChange.get().status_code == 1
assert self.db.HttpCodeChange.get().http_header_dict == {
"Etag": "foobar"
}
assert self.db.HttpCodeChange.get().status_id == status_id
def test_checkHttpStatus_relativeUrl(self):
......@@ -771,6 +935,87 @@ class SurykatkaHttpTestCase(unittest.TestCase):
else:
raise NotImplementedError("Expected NotImplementedError")
def __generateHeaderDict(self, header_list):
result_dict = {}
for header in header_list:
result_dict[header] = header + " bar"
return result_dict
@httpretty.activate
def test_checkHttpStatus_whitelistHttpHeader(self):
ip = "127.0.0.1"
url = "http://example.org/foo?bar=1"
bot_version = 1
whitelist_header_list = [
# Redirect
"Location",
# HTTP Range
"Accept-Ranges",
# HTTP Cache
"Etag",
"Last-Modified",
"Vary",
"Cache-Control",
"Set-Cookie",
"WWW-Authenticate"
# gzip
"Content-Type",
"Content-Encoding",
"Content-Disposition"
# Security
"Content-Security-Policy",
"Referrer-Policy",
"Strict-Transport-Policy",
"Feature-Policy",
"X-Frame-Options",
"X-Content-Type-Options"
# CORS
"Access-Control-Allow-Origin",
"Access-Control-Allow-Methods",
"Access-Control-Allow-Credentials",
"Access-Control-Allow-Headers",
"Access-Control-Expose-Headers",
]
blacklist_header_list = [
"Age",
"Date",
"Expires",
"Foo",
]
header_dict = self.__generateHeaderDict(whitelist_header_list)
header_dict.update(self.__generateHeaderDict(blacklist_header_list))
httpretty.register_uri(
httpretty.GET,
"http://127.0.0.1/foo?bar=1",
status=418,
adding_headers=header_dict,
)
status_id = logStatus(self.db, "foo")
checkHttpStatus(self.db, status_id, url, ip, bot_version)
last_request = httpretty.last_request()
assert len(last_request.headers) == 5, last_request.headers.keys()
assert last_request.headers["Accept"] == "text/html;q=0.9,*/*;q=0.8"
assert last_request.headers["Accept-Encoding"] == "gzip, deflate"
assert last_request.headers["Connection"] == "keep-alive"
assert last_request.headers["Host"] == "example.org"
assert (
last_request.headers["User-Agent"]
== "SURYKATKA/1 (+https://lab.nexedi.com/nexedi/surykatka)"
)
assert len(last_request.body) == 0
assert self.db.HttpCodeChange.select().count() == 1
assert self.db.HttpCodeChange.get().ip == ip
assert self.db.HttpCodeChange.get().url == url
assert self.db.HttpCodeChange.get().status_code == 418
assert self.db.HttpCodeChange.get().http_header_dict == self.__generateHeaderDict(
whitelist_header_list
)
assert self.db.HttpCodeChange.get().status_id == status_id
def suite():
suite = unittest.TestSuite()
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7