Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
S
surykatka
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
Analytics
Analytics
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Commits
Issue Boards
Open sidebar
Romain Courteaud
surykatka
Commits
a25dd45a
Commit
a25dd45a
authored
Feb 16, 2021
by
Romain Courteaud
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Add PUBLIC_SUFFIX parameter
Allow to skip some top domains like .co.uk
parent
fe0adc9f
Changes
5
Hide whitespace changes
Inline
Side-by-side
Showing
5 changed files
with
94 additions
and
5 deletions
+94
-5
src/surykatka/bot.py
src/surykatka/bot.py
+4
-1
src/surykatka/configuration.py
src/surykatka/configuration.py
+2
-0
src/surykatka/dns.py
src/surykatka/dns.py
+7
-2
tests/test_bot.py
tests/test_bot.py
+67
-2
tests/test_dns.py
tests/test_dns.py
+14
-0
No files found.
src/surykatka/bot.py
View file @
a25dd45a
...
...
@@ -76,7 +76,10 @@ class WebBot:
domain_list
=
list
(
set
(
domain_list
))
# Expand with all parent domains
return
expandDomainList
(
domain_list
)
return
expandDomainList
(
domain_list
,
public_suffix_list
=
self
.
config
[
"PUBLIC_SUFFIX"
].
split
(),
)
def
iterateLoop
(
self
):
status_id
=
logStatus
(
self
.
_db
,
"loop"
)
...
...
src/surykatka/configuration.py
View file @
a25dd45a
...
...
@@ -48,6 +48,8 @@ def createConfiguration(
get_default_resolver
().
nameservers
)
if
"PUBLIC_SUFFIX"
not
in
config
[
CONFIG_SECTION
]:
config
[
CONFIG_SECTION
][
"PUBLIC_SUFFIX"
]
=
""
if
"DOMAIN"
not
in
config
[
CONFIG_SECTION
]:
config
[
CONFIG_SECTION
][
"DOMAIN"
]
=
""
if
"URL"
not
in
config
[
CONFIG_SECTION
]:
...
...
src/surykatka/dns.py
View file @
a25dd45a
...
...
@@ -133,12 +133,17 @@ def getReachableResolverList(db, status_id, resolver_ip_list, timeout=TIMEOUT):
return
result_ip_list
def
expandDomainList
(
domain_list
):
def
expandDomainList
(
domain_list
,
public_suffix_list
=
None
):
for
domain_text
in
domain_list
:
dns_name
=
dns
.
name
.
from_text
(
domain_text
)
if
(
len
(
dns_name
.
labels
)
-
1
)
>
2
:
domain_list
.
append
(
dns_name
.
parent
().
to_text
(
omit_final_dot
=
True
))
# https://publicsuffix.org/list/public_suffix_list.dat
parent_domain_text
=
dns_name
.
parent
().
to_text
(
omit_final_dot
=
True
)
if
(
public_suffix_list
is
None
)
or
(
parent_domain_text
not
in
public_suffix_list
):
domain_list
.
append
(
parent_domain_text
)
domain_list
=
list
(
set
(
domain_list
))
domain_list
.
sort
()
...
...
tests/test_bot.py
View file @
a25dd45a
...
...
@@ -56,13 +56,16 @@ def checkSslChange(bot, result_list):
def
checkDnsChange
(
bot
,
result_list
):
assert
bot
.
_db
.
DnsChange
.
select
().
count
()
==
len
(
result_list
)
select_list
=
(
bot
.
_db
.
DnsChange
.
select
()
.
order_by
(
bot
.
_db
.
DnsChange
.
resolver_ip
.
asc
())
.
order_by
(
bot
.
_db
.
DnsChange
.
domain
.
asc
())
)
assert
[(
x
.
resolver_ip
,
x
.
domain
)
for
x
in
select_list
]
==
result_list
db_result_list
=
[(
x
.
resolver_ip
,
x
.
domain
)
for
x
in
select_list
]
assert
bot
.
_db
.
DnsChange
.
select
().
count
()
==
len
(
result_list
),
db_result_list
assert
db_result_list
==
result_list
class
SurykatkaBotTestCase
(
unittest
.
TestCase
):
...
...
@@ -442,6 +445,68 @@ class SurykatkaBotTestCase(unittest.TestCase):
],
)
def
test_oneNameserverOneSubDomainOneIpOnePublicSuffix
(
self
):
resolver_ip
=
"127.0.0.1"
domain
=
"example.com"
sub_domain
=
"foo.%s"
%
domain
bot
=
WebBot
(
mapping
=
{
"SQLITE"
:
":memory:"
,
"DOMAIN"
:
sub_domain
,
"NAMESERVER"
:
resolver_ip
,
"PUBLIC_SUFFIX"
:
domain
,
}
)
bot
.
initDB
()
with
mock
.
patch
(
"surykatka.dns.dns.resolver.Resolver.query"
)
as
mock_query
,
mock
.
patch
(
"surykatka.network.socket.socket"
)
as
mock_socket
,
mock
.
patch
(
"surykatka.ssl.ssl.create_default_context"
)
as
mock_create_default_context
,
mock
.
patch
(
"surykatka.http.request"
)
as
mock_request
:
mock_request
.
return_value
.
headers
=
{
"Etag"
:
"foobar"
}
mock_query
.
return_value
=
[
MockAnswer
(
"1.2.3.4"
)]
mock_create_default_context
.
return_value
.
wrap_socket
.
return_value
.
getpeercert
.
side_effect
=
[
b""
,
{
"notBefore"
:
"Jan 27 04:33:22 2020 GMT"
,
"notAfter"
:
"Jan 27 04:33:22 2020 GMT"
,
"subject"
:
[[(
"commonName"
,
"foo"
)]],
"issuer"
:
[[(
"commonName"
,
"bar"
)]],
},
]
bot
.
iterateLoop
()
assert
mock_query
.
call_count
==
2
assert
mock_socket
.
call_count
==
3
assert
mock_create_default_context
.
call_count
==
1
assert
mock_request
.
call_count
==
2
checkNetworkChange
(
bot
,
[(
resolver_ip
,
53
),
(
"1.2.3.4"
,
80
),
(
"1.2.3.4"
,
443
)]
)
checkDnsChange
(
bot
,
[(
resolver_ip
,
"example.org"
),
(
resolver_ip
,
sub_domain
)]
)
checkSslChange
(
bot
,
[(
"1.2.3.4"
,
443
,
sub_domain
)])
checkHttpCodeChange
(
bot
,
[
(
"1.2.3.4"
,
"http://%s"
%
sub_domain
),
(
"1.2.3.4"
,
"https://%s"
%
sub_domain
),
],
)
def
test_oneNameserverOneUrlOneIp
(
self
):
resolver_ip
=
"127.0.0.1"
domain
=
"example.org"
...
...
tests/test_dns.py
View file @
a25dd45a
...
...
@@ -50,6 +50,20 @@ class SurykatkaDNSTestCase(unittest.TestCase):
result
=
expandDomainList
([
"c"
,
"b.a.a"
,
"a.a.a"
,
"a.a"
])
assert
result
==
[
"a.a"
,
"a.a.a"
,
"b.a.a"
,
"c"
]
def
test_expandDomainList_skipPublicSuffix
(
self
):
result
=
expandDomainList
([
"doo.foo.bar.co.uk"
])
assert
result
==
[
"bar.co.uk"
,
"co.uk"
,
"doo.foo.bar.co.uk"
,
"foo.bar.co.uk"
,
]
result
=
expandDomainList
(
[
"doo.foo.bar.co.uk"
],
public_suffix_list
=
[
"bar.co.uk"
]
)
assert
result
==
[
"doo.foo.bar.co.uk"
,
"foo.bar.co.uk"
]
################################################
# logDnsQuery
################################################
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment