export: Skip writing any subnets

It is unnecessary to write any subnets in the output, because
they might already be included in a bigger parent network.

Since the tree is ordered, we only need to check if the network
to be written is a subnet of the previous one. If so, we skip it.
Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>

src/libloc.sym

@@ -83,6 +83,7 @@ global:
 	loc_network_get_asn;
 	loc_network_get_country_code;
 	loc_network_has_flag;
+	loc_network_is_subnet_of;
 	loc_network_match_asn;
 	loc_network_match_country_code;
 	loc_network_match_flag;

src/loc/network.h

@@ -51,6 +51,8 @@ int loc_network_has_flag(struct loc_network* network, uint32_t flag);
 int loc_network_set_flag(struct loc_network* network, uint32_t flag);
 int loc_network_match_flag(struct loc_network* network, uint32_t flag);
 
+int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other);
+
 #ifdef LIBLOC_PRIVATE
 
 int loc_network_to_database_v1(struct loc_network* network, struct loc_database_network_v1* dbobj);

src/network.c

@@ -346,6 +346,24 @@ LOC_EXPORT int loc_network_match_flag(struct loc_network* network, uint32_t flag
 	return loc_network_has_flag(network, flag);
 }
 
+LOC_EXPORT int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other) {
+	// If the start address of the other network is smaller than this network,
+	// it cannot be a subnet.
+	if (in6_addr_cmp(&self->start_address, &other->start_address) < 0)
+		return 0;
+
+	// Get the end addresses
+	struct in6_addr last_address_self  = make_last_address(&self->start_address,  self->prefix);
+	struct in6_addr last_address_other = make_last_address(&other->start_address, other->prefix);
+
+	// If the end address of the other network is greater than this network,
+	// it cannot be a subnet.
+	if (in6_addr_cmp(&last_address_self, &last_address_other) > 0)
+		return 0;
+
+	return 1;
+}
+
 LOC_EXPORT int loc_network_to_database_v1(struct loc_network* network, struct loc_database_network_v1* dbobj) {
 	// Add country code
 	loc_country_code_copy(dbobj->country_code, network->country_code);

src/python/export.py

@@ -39,8 +39,11 @@ class OutputWriter(object):
 	suffix = "networks"
 	mode = "w"
 
-	def __init__(self, f, prefix=None):
-		self.f, self.prefix = f, prefix
+	def __init__(self, f, prefix=None, flatten=True):
+		self.f, self.prefix, self.flatten = f, prefix, flatten
+
+		# The previously written network
+		self._last_network = None
 
 		# Immediately write the header
 		self._write_header()
@@ -57,6 +60,18 @@ class OutputWriter(object):
 	def __repr__(self):
 		return "<%s f=%s>" % (self.__class__.__name__, self.f)
 
+	def _flatten(self, network):
+		"""
+			Checks if the given network needs to be written to file,
+			or if it is a subnet of the previously written network.
+		"""
+		if self._last_network and network.is_subnet_of(self._last_network):
+			return True
+
+		# Remember this network for the next call
+		self._last_network = network
+		return False
+
 	def _write_header(self):
 		"""
 			The header of the file
@@ -69,9 +84,17 @@ class OutputWriter(object):
 		"""
 		pass
 
-	def write(self, network):
+	def _write_network(self, network):
 		self.f.write("%s\n" % network)
 
+	def write(self, network):
+		if self.flatten and self._flatten(network):
+			log.debug("Skipping writing network %s" % network)
+			return
+
+		# Write the network to file
+		self._write_network(network)
+
 	def finish(self):
 		"""
 			Called when all data has been written
@@ -91,7 +114,7 @@ class IpsetOutputWriter(OutputWriter):
 	def _write_header(self):
 		self.f.write("create %s hash:net family inet hashsize 1024 maxelem 65536\n" % self.prefix)
 
-	def write(self, network):
+	def _write_network(self, network):
 		self.f.write("add %s %s\n" % (self.prefix, network))
 
 
@@ -107,7 +130,7 @@ class NftablesOutputWriter(OutputWriter):
 	def _write_footer(self):
 		self.f.write("}\n")
 
-	def write(self, network):
+	def _write_network(self, network):
 		self.f.write("	%s,\n" % network)
 
 
@@ -119,7 +142,7 @@ class XTGeoIPOutputWriter(OutputWriter):
 	suffix = "iv"
 	mode = "wb"
 
-	def write(self, network):
+	def _write_network(self, network):
 		n = ipaddress.ip_network("%s" % network)
 
 		for address in (n.network_address, n.broadcast_address):

src/python/network.c

@@ -154,6 +154,18 @@ static PyObject* Network_set_flag(NetworkObject* self, PyObject* args) {
 	Py_RETURN_NONE;
 }
 
+static PyObject* Network_is_subnet_of(NetworkObject* self, PyObject* args) {
+	NetworkObject* other = NULL;
+
+	if (!PyArg_ParseTuple(args, "O!", &NetworkType, &other))
+		return NULL;
+
+	if (loc_network_is_subnet_of(self->network, other->network))
+		Py_RETURN_TRUE;
+
+	Py_RETURN_FALSE;
+}
+
 static struct PyMethodDef Network_methods[] = {
 	{
 		"has_flag",
@@ -161,6 +173,12 @@ static struct PyMethodDef Network_methods[] = {
 		METH_VARARGS,
 		NULL,
 	},
+	{
+		"is_subnet_of",
+		(PyCFunction)Network_is_subnet_of,
+		METH_VARARGS,
+		NULL,
+	},
 	{
 		"set_flag",
 		(PyCFunction)Network_set_flag,

src/test-network.c

@@ -95,6 +95,19 @@ int main(int argc, char** argv) {
 	size_t nodes = loc_network_tree_count_nodes(tree);
 	printf("The tree has %zu nodes\n", nodes);
 
+	// Check subnet function
+	err = loc_network_is_subnet_of(network1, network2);
+	if (err != 0) {
+		fprintf(stderr, "Subnet check 1 failed: %d\n", err);
+		exit(EXIT_FAILURE);
+	}
+
+	err = loc_network_is_subnet_of(network2, network1);
+	if (err != 1) {
+		fprintf(stderr, "Subnet check 2 failed: %d\n", err);
+		exit(EXIT_FAILURE);
+	}
+
 	// Create a database
 	struct loc_writer* writer;
 	err = loc_writer_new(ctx, &writer, NULL, NULL);