Skip to content

L
libloc
Commit 975f280f authored by Michael Tremer's avatar Michael Tremer
Browse files
Options

database: Read all data from the large mmap() 

Signed-off-by: default avatarMichael Tremer <michael.tremer@ipfire.org>
parent 6c27aec2
Hide whitespace changes
Inline Side-by-side
Showing with 120 additions and 210 deletions
src/database.c
View file @ 975f280f
......@@ -142,35 +142,53 @@ struct loc_database_enumerator {
struct in6_addr gap4_start;
};
#define loc_database_read_object(db, buffer, objects, pos) \
__loc_database_read_object(db, buffer, sizeof(*buffer), objects, pos)
/*
Reads an object into memory (used when mmap() isn't available)
Checks if it is safe to read the buffer of size length starting at p.
*/
static void* __loc_database_read_object(struct loc_database* db, void* buffer,
const size_t length, const struct loc_database_objects* objects, const off_t pos) {
// Calculate offset
const off_t offset = pos * length;
#define loc_database_check_boundaries(db, p) \
__loc_database_check_boundaries(db, (const char*)p, sizeof(*p))
// Map the object first if possible
if (objects->data)
return objects->data + offset;
static int __loc_database_check_boundaries(struct loc_database* db,
const char* p, const size_t length) {
size_t offset = p - db->data;
// Otherwise fall back and read the object into the buffer
const int fd = fileno(db->f);
// Return if everything is within the boundary
if (offset <= db->length - length)
return 1;
// Read object
ssize_t bytes_read = pread(fd, buffer, length, objects->offset + offset);
DEBUG(db->ctx, "Database read check failed at %p for %zu byte(s)\n", p, length);
DEBUG(db->ctx, " p = %p (offset = %jd, length = %zu)\n", p, offset, length);
DEBUG(db->ctx, " data = %p (length = %zu)\n", db->data, db->length);
DEBUG(db->ctx, " end = %p\n", db->data + db->length);
DEBUG(db->ctx, " overflow of %zu byte(s)\n", offset + length - db->length);
// Check if we could read what we wanted
if (bytes_read < (ssize_t)length) {
ERROR(db->ctx, "Error reading object from database: %m\n");
// Otherwise raise EFAULT
errno = EFAULT;
return 0;
}
/*
Returns a pointer to the n-th object
*/
static char* loc_database_object(struct loc_database* db,
const struct loc_database_objects* objects, const size_t length, const off_t n) {
// Return NULL if objects were not initialized
if (!objects->data) {
errno = EFAULT;
return NULL;
}
// Success!
return buffer;
// Calculate offset
const off_t offset = n * length;
// Return a pointer to where the object lies
char* object = objects->data + offset;
// Check if the object is part of the memory
if (!__loc_database_check_boundaries(db, object, length))
return NULL;
return object;
}
static int loc_database_version_supported(struct loc_database* db, uint8_t version) {
......@@ -285,25 +303,6 @@ static int loc_database_read_signature(struct loc_database* db,
return 0;
}
/*
Checks if it is safe to read the buffer of size length starting at p.
*/
#define loc_database_check_boundaries(db, p) \
__loc_database_check_boundaries(db, (const char*)p, sizeof(*p))
static int __loc_database_check_boundaries(struct loc_database* db,
const char* p, const size_t length) {
size_t offset = p - db->data;
// Return if everything is within the boundary
if (offset < db->length - length)
return 1;
// Otherwise raise EFAULT
errno = EFAULT;
return 0;
}
static int loc_database_read_header_v1(struct loc_database* db) {
const struct loc_database_header_v1* header =
(const struct loc_database_header_v1*)(db->data + LOC_DATABASE_MAGIC_SIZE);
......@@ -337,9 +336,15 @@ static int loc_database_read_header_v1(struct loc_database* db) {
if (r)
return r;
const char* stringpool_start = db->data + be32toh(header->pool_offset);
size_t stringpool_length = be32toh(header->pool_length);
// Check if the stringpool is part of the mapped area
if (!__loc_database_check_boundaries(db, stringpool_start, stringpool_length))
return 1;
// Open the stringpool
r = loc_stringpool_open(db->ctx, &db->pool, db->f,
be32toh(header->pool_length), be32toh(header->pool_offset));
r = loc_stringpool_open(db->ctx, &db->pool, stringpool_start, stringpool_length);
if (r)
return r;
......@@ -709,8 +714,7 @@ LOC_EXPORT size_t loc_database_count_as(struct loc_database* db) {
// Returns the AS at position pos
static int loc_database_fetch_as(struct loc_database* db, struct loc_as** as, off_t pos) {
struct loc_database_as_v1 as_v1;
struct loc_database_as_v1* p_v1;
struct loc_database_as_v1* as_v1 = NULL;
int r;
if ((size_t)pos >= db->as_objects.count) {
......@@ -722,12 +726,13 @@ static int loc_database_fetch_as(struct loc_database* db, struct loc_as** as, of
switch (db->version) {
case LOC_DATABASE_VERSION_1:
// Read the object
p_v1 = loc_database_read_object(db, &as_v1, &db->as_objects, pos);
if (!p_v1)
// Find the object
as_v1 = (struct loc_database_as_v1*)loc_database_object(db,
&db->as_objects, sizeof(*as_v1), pos);
if (!as_v1)
return 1;
r = loc_as_new_from_database_v1(db->ctx, db->pool, as, p_v1);
r = loc_as_new_from_database_v1(db->ctx, db->pool, as, as_v1);
break;
default:
......@@ -792,8 +797,7 @@ LOC_EXPORT int loc_database_get_as(struct loc_database* db, struct loc_as** as,
// Returns the network at position pos
static int loc_database_fetch_network(struct loc_database* db, struct loc_network** network,
struct in6_addr* address, unsigned int prefix, off_t pos) {
struct loc_database_network_v1 network_v1;
struct loc_database_network_v1* p_v1;
struct loc_database_network_v1* network_v1 = NULL;
int r;
if ((size_t)pos >= db->network_objects.count) {
......@@ -808,11 +812,12 @@ static int loc_database_fetch_network(struct loc_database* db, struct loc_networ
switch (db->version) {
case LOC_DATABASE_VERSION_1:
// Read the object
p_v1 = loc_database_read_object(db, &network_v1, &db->network_objects, pos);
if (!p_v1)
network_v1 = (struct loc_database_network_v1*)loc_database_object(db,
&db->network_objects, sizeof(*network_v1), pos);
if (!network_v1)
return 1;
r = loc_network_new_from_database_v1(db->ctx, network, address, prefix, p_v1);
r = loc_network_new_from_database_v1(db->ctx, network, address, prefix, network_v1);
break;
default:
......@@ -862,14 +867,14 @@ static int __loc_database_lookup_handle_leaf(struct loc_database* db, const stru
static int __loc_database_lookup(struct loc_database* db, const struct in6_addr* address,
struct loc_network** network, struct in6_addr* network_address,
off_t node_index, unsigned int level) {
struct loc_database_network_node_v1 node_v1;
struct loc_database_network_node_v1* p_v1;
struct loc_database_network_node_v1* node_v1 = NULL;
int r;
// Fetch the next node
p_v1 = loc_database_read_object(db, &node_v1, &db->network_node_objects, node_index);
if (!p_v1)
node_v1 = (struct loc_database_network_node_v1*)loc_database_object(db,
&db->network_node_objects, sizeof(*node_v1), node_index);
if (!node_v1)
return 1;
// Follow the path
......@@ -877,9 +882,9 @@ static int __loc_database_lookup(struct loc_database* db, const struct in6_addr*
loc_address_set_bit(network_address, level, bit);
if (bit == 0)
node_index = be32toh(p_v1->zero);
node_index = be32toh(node_v1->zero);
else
node_index = be32toh(p_v1->one);
node_index = be32toh(node_v1->one);
// If the node index is zero, the tree ends here
// and we cannot descend any further
......@@ -907,8 +912,8 @@ static int __loc_database_lookup(struct loc_database* db, const struct in6_addr*
}
// If this node has a leaf, we will check if it matches
if (__loc_database_node_is_leaf(p_v1)) {
r = __loc_database_lookup_handle_leaf(db, address, network, network_address, level, p_v1);
if (__loc_database_node_is_leaf(node_v1)) {
r = __loc_database_lookup_handle_leaf(db, address, network, network_address, level, node_v1);
if (r <= 0)
return r;
}
......@@ -955,8 +960,7 @@ LOC_EXPORT int loc_database_lookup_from_string(struct loc_database* db,
// Returns the country at position pos
static int loc_database_fetch_country(struct loc_database* db,
struct loc_country** country, off_t pos) {
struct loc_database_country_v1 country_v1;
struct loc_database_country_v1* p_v1;
struct loc_database_country_v1* country_v1 = NULL;
int r;
// Check if the country is within range
......@@ -970,11 +974,12 @@ static int loc_database_fetch_country(struct loc_database* db,
switch (db->version) {
case LOC_DATABASE_VERSION_1:
// Read the object
p_v1 = loc_database_read_object(db, &country_v1, &db->country_objects, pos);
if (!p_v1)
country_v1 = (struct loc_database_country_v1*)loc_database_object(db,
&db->country_objects, sizeof(*country_v1), pos);
if (!country_v1)
return 1;
r = loc_country_new_from_database_v1(db->ctx, db->pool, country, p_v1);
r = loc_country_new_from_database_v1(db->ctx, db->pool, country, country_v1);
break;
default:
......@@ -1315,8 +1320,6 @@ static int loc_database_enumerator_match_network(
static int __loc_database_enumerator_next_network(
struct loc_database_enumerator* enumerator, struct loc_network** network, int filter) {
struct loc_database_network_node_v1 node_v1;
// Return top element from the stack
while (1) {
*network = loc_network_list_pop_first(enumerator->stack);
......@@ -1360,8 +1363,9 @@ static int __loc_database_enumerator_next_network(
enumerator->networks_visited[node->offset]++;
// Pop node from top of the stack
struct loc_database_network_node_v1* n = loc_database_read_object(enumerator->db,
&node_v1, &enumerator->db->network_node_objects, node->offset);
struct loc_database_network_node_v1* n =
(struct loc_database_network_node_v1*)loc_database_object(enumerator->db,
&enumerator->db->network_node_objects, sizeof(*n), node->offset);
if (!n)
return 1;
......
src/libloc/stringpool.h
View file @ 975f280f
......@@ -27,7 +27,7 @@
struct loc_stringpool;
int loc_stringpool_new(struct loc_ctx* ctx, struct loc_stringpool** pool);
int loc_stringpool_open(struct loc_ctx* ctx, struct loc_stringpool** pool,
FILE* f, size_t length, off_t offset);
const char* data, const size_t length);
struct loc_stringpool* loc_stringpool_ref(struct loc_stringpool* pool);
struct loc_stringpool* loc_stringpool_unref(struct loc_stringpool* pool);
......
src/stringpool.c
View file @ 975f280f
......@@ -27,89 +27,36 @@
#include <libloc/private.h>
#include <libloc/stringpool.h>
#define LOC_STRINGPOOL_BLOCK_SIZE (512 * 1024)
struct loc_stringpool {
struct loc_ctx* ctx;
int refcount;
// A file descriptor when we open an existing stringpool
int fd;
off_t offset;
// Reference to any mapped data
const char* data;
ssize_t length;
// Mapped data (from mmap())
char* mmapped_data;
char* data;
char* pos;
char buffer[LOC_DATABASE_PAGE_SIZE];
// Reference to own storage
char* blocks;
size_t size;
};
static off_t loc_stringpool_get_offset(struct loc_stringpool* pool, const char* pos) {
if (pos < pool->data) {
errno = EFAULT;
return -1;
}
if (pos > (pool->data + pool->length)) {
errno = EFAULT;
return -1;
}
static int loc_stringpool_grow(struct loc_stringpool* pool, const size_t size) {
DEBUG(pool->ctx, "Growing string pool by %zu byte(s)\n", size);
return pos - pool->data;
}
static char* __loc_stringpool_get(struct loc_stringpool* pool, off_t offset) {
ssize_t bytes_read;
// Check boundaries
if (offset < 0 || offset >= pool->length) {
errno = ERANGE;
return NULL;
}
// Increment size
pool->size += size;
// Return any data that we have in memory
if (pool->data)
return pool->data + offset;
// Otherwise read a block from file
bytes_read = pread(pool->fd, pool->buffer, sizeof(pool->buffer),
pool->offset + offset);
// Break on error
if (bytes_read < 0) {
ERROR(pool->ctx, "Could not read from string pool: %m\n");
return NULL;
}
// It is okay, if we did not read as much as we wanted, since we might be reading
// the last block which might be of an unknown size.
// Search for a complete string. If there is no NULL byte, the block is garbage.
char* end = memchr(pool->buffer, bytes_read, '\0');
if (!end)
return NULL;
// Return what's in the buffer
return pool->buffer;
}
static int loc_stringpool_grow(struct loc_stringpool* pool, size_t length) {
DEBUG(pool->ctx, "Growing string pool to %zu bytes\n", length);
// Save pos pointer
off_t pos = loc_stringpool_get_offset(pool, pool->pos);
// Reallocate data section
pool->data = realloc(pool->data, length);
if (!pool->data)
// Reallocate blocks
pool->blocks = realloc(pool->blocks, pool->size);
if (!pool->blocks) {
ERROR(pool->ctx, "Could not grow string pool: %m\n");
return 1;
}
pool->length = length;
// Restore pos
pool->pos = __loc_stringpool_get(pool, pos);
// Update data pointer
pool->data = pool->blocks;
return 0;
}
......@@ -122,44 +69,33 @@ static off_t loc_stringpool_append(struct loc_stringpool* pool, const char* stri
DEBUG(pool->ctx, "Appending '%s' to string pool at %p\n", string, pool);
// How much space to we need?
const size_t length = strlen(string) + 1;
// Make sure we have enough space
int r = loc_stringpool_grow(pool, pool->length + strlen(string) + 1);
if (r)
return -1;
if (pool->length + length > pool->size) {
int r = loc_stringpool_grow(pool, LOC_STRINGPOOL_BLOCK_SIZE);
if (r)
return r;
}
off_t offset = loc_stringpool_get_offset(pool, pool->pos);
off_t offset = pool->length;
// Copy string byte by byte
while (*string)
*pool->pos++ = *string++;
// Copy the string
memcpy(pool->blocks + offset, string, length);
// Terminate the string
*pool->pos++ = '\0';
// Update the length of the pool
pool->length += length;
return offset;
}
static void loc_stringpool_free(struct loc_stringpool* pool) {
DEBUG(pool->ctx, "Releasing string pool %p\n", pool);
int r;
// Close file
if (pool->fd > 0)
close(pool->fd);
// Unmap any mapped memory
if (pool->mmapped_data) {
r = munmap(pool->mmapped_data, pool->length);
if (r)
ERROR(pool->ctx, "Error unmapping string pool: %m\n");
if (pool->mmapped_data == pool->data)
pool->data = NULL;
}
// Free any data
if (pool->data)
free(pool->data);
if (pool->blocks)
free(pool->blocks);
loc_unref(pool->ctx);
free(pool);
......@@ -178,27 +114,8 @@ int loc_stringpool_new(struct loc_ctx* ctx, struct loc_stringpool** pool) {
return 0;
}
static int loc_stringpool_mmap(struct loc_stringpool* pool) {
// Try mmap()
char* p = mmap(NULL, pool->length, PROT_READ, MAP_PRIVATE, pool->fd, pool->offset);
if (p == MAP_FAILED) {
// Ignore if data hasn't been aligned correctly
if (errno == EINVAL)
return 0;
ERROR(pool->ctx, "Could not mmap stringpool: %m\n");
return 1;
}
// Store mapped memory area
pool->data = pool->mmapped_data = pool->pos = p;
return 0;
}
int loc_stringpool_open(struct loc_ctx* ctx, struct loc_stringpool** pool,
FILE* f, size_t length, off_t offset) {
const char* data, const size_t length) {
struct loc_stringpool* p = NULL;
// Allocate a new stringpool
......@@ -206,29 +123,11 @@ int loc_stringpool_open(struct loc_ctx* ctx, struct loc_stringpool** pool,
if (r)
goto ERROR;
// Store offset and length
p->offset = offset;
// Store data and length
p->data = data;
p->length = length;
DEBUG(p->ctx, "Reading string pool starting from %jd (%zu bytes)\n",
(intmax_t)p->offset, p->length);
int fd = fileno(f);
// Copy the file descriptor
p->fd = dup(fd);
if (p->fd < 0) {
ERROR(ctx, "Could not duplicate file the file descriptor: %m\n");
r = 1;
goto ERROR;
}
// Map data into memory
if (p->length > 0) {
r = loc_stringpool_mmap(p);
if (r)
goto ERROR;
}
DEBUG(p->ctx, "Opened string pool at %p (%zu bytes)\n", p->data, p->length);
*pool = p;
return 0;
......@@ -256,11 +155,18 @@ struct loc_stringpool* loc_stringpool_unref(struct loc_stringpool* pool) {
}
const char* loc_stringpool_get(struct loc_stringpool* pool, off_t offset) {
return __loc_stringpool_get(pool, offset);
// Check boundaries
if (offset < 0 || offset >= pool->length) {
errno = ERANGE;
return NULL;
}
// Return any data that we have in memory
return pool->data + offset;
}
size_t loc_stringpool_get_size(struct loc_stringpool* pool) {
return loc_stringpool_get_offset(pool, pool->pos);
return pool->length;
}
static off_t loc_stringpool_find(struct loc_stringpool* pool, const char* s) {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7