Commit 64e95fa9 authored by Peter Müller's avatar Peter Müller Committed by Michael Tremer

importer: Import raw sources for inetnum's again

The extended feeds do not have enough detailed information
for us, so that we need to import inetnums from RIRs where
possible. Filtering private networks is necessary as RIR data
may contain 0.0.0.0/0 or similar entries for administrative
purposes or due to misfilings.

Special thanks goes to Michael for spending numerous hours
on this, setting up a testing environment and providing helpful
advice while debugging.

Partially fixes: #12458

Cc: Michael Tremer <michael.tremer@ipfire.org>
Signed-off-by: default avatarPeter Müller <peter.mueller@ipfire.org>
Signed-off-by: default avatarMichael Tremer <michael.tremer@ipfire.org>
parent 2cd2e342
......@@ -30,8 +30,8 @@ WHOIS_SOURCES = (
"https://ftp.afrinic.net/pub/pub/dbase/afrinic.db.gz",
# Asia Pacific Network Information Centre
#"https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz",
#"https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz",
"https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz",
"https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz",
#"https://ftp.apnic.net/apnic/whois/apnic.db.route6.gz",
#"https://ftp.apnic.net/apnic/whois/apnic.db.route.gz",
"https://ftp.apnic.net/apnic/whois/apnic.db.aut-num.gz",
......@@ -45,8 +45,8 @@ WHOIS_SOURCES = (
# XXX ???
# Réseaux IP Européens
#"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz",
#"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz",
"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz",
"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz",
#"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route6.gz",
#"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route.gz",
"https://ftp.ripe.net/ripe/dbase/split/ripe.db.aut-num.gz",
......@@ -55,10 +55,10 @@ WHOIS_SOURCES = (
EXTENDED_SOURCES = (
# African Network Information Centre
"https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest",
#"https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest",
# Asia Pacific Network Information Centre
"https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest",
#"https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest",
# American Registry for Internet Numbers
"https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest",
......@@ -67,7 +67,7 @@ EXTENDED_SOURCES = (
"http://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest",
# Réseaux IP Européens
"https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest",
#"https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest",
)
class Downloader(object):
......
......@@ -393,6 +393,10 @@ class CLI(object):
if line.startswith("aut-num:"):
return self._parse_autnum_block(block)
# inetnum
if line.startswith("inet6num:") or line.startswith("inetnum:"):
return self._parse_inetnum_block(block)
# organisation
elif line.startswith("organisation:"):
return self._parse_org_block(block)
......@@ -422,6 +426,65 @@ class CLI(object):
autnum.get("asn"), autnum.get("org"),
)
def _parse_inetnum_block(self, block):
logging.debug("Parsing inetnum block:")
inetnum = {}
for line in block:
logging.debug(line)
# Split line
key, val = split_line(line)
if key == "inetnum":
start_address, delim, end_address = val.partition("-")
# Strip any excess space
start_address, end_address = start_address.rstrip(), end_address.strip()
# Convert to IP address
try:
start_address = ipaddress.ip_address(start_address)
end_address = ipaddress.ip_address(end_address)
except ValueError:
logging.warning("Could not parse line: %s" % line)
return
# Set prefix to default
prefix = 32
# Count number of addresses in this subnet
num_addresses = int(end_address) - int(start_address)
if num_addresses:
prefix -= math.log(num_addresses, 2)
inetnum["inetnum"] = "%s/%.0f" % (start_address, prefix)
elif key == "inet6num":
inetnum[key] = val
elif key == "country":
if val == "UNITED STATES":
val = "US"
inetnum[key] = val.upper()
# Skip empty objects
if not inetnum:
return
network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False)
# Bail out in case we have processed a non-public IP network
if network.is_private:
logging.warning("Skipping non-globally routable network: %s" % network)
return
self.db.execute("INSERT INTO networks(network, country) \
VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country",
"%s" % network, inetnum.get("country"),
)
def _parse_org_block(self, block):
org = {}
for line in block:
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment