Commit a098ecd2 authored by Stephen Boyd's avatar Stephen Boyd Committed by Linus Torvalds

firmware: support loading into a pre-allocated buffer

Some systems are memory constrained but they need to load very large
firmwares.  The firmware subsystem allows drivers to request this
firmware be loaded from the filesystem, but this requires that the
entire firmware be loaded into kernel memory first before it's provided
to the driver.  This can lead to a situation where we map the firmware
twice, once to load the firmware into kernel memory and once to copy the
firmware into the final resting place.

This creates needless memory pressure and delays loading because we have
to copy from kernel memory to somewhere else.  Let's add a
request_firmware_into_buf() API that allows drivers to request firmware
be loaded directly into a pre-allocated buffer.  This skips the
intermediate step of allocating a buffer in kernel memory to hold the
firmware image while it's read from the filesystem.  It also requires
that drivers know how much memory they'll require before requesting the
firmware and negates any benefits of firmware caching because the
firmware layer doesn't manage the buffer lifetime.

For a 16MB buffer, about half the time is spent performing a memcpy from
the buffer to the final resting place.  I see loading times go from
0.081171 seconds to 0.047696 seconds after applying this patch.  Plus
the vmalloc pressure is reduced.

This is based on a patch from Vikram Mulukutla on codeaurora.org:
  https://www.codeaurora.org/cgit/quic/la/kernel/msm-3.18/commit/drivers/base/firmware_class.c?h=rel/msm-3.18&id=0a328c5f6cd999f5c591f172216835636f39bcb5

Link: http://lkml.kernel.org/r/20160607164741.31849-4-stephen.boyd@linaro.orgSigned-off-by: default avatarStephen Boyd <stephen.boyd@linaro.org>
Cc: Mimi Zohar <zohar@linux.vnet.ibm.com>
Cc: Vikram Mulukutla <markivx@codeaurora.org>
Cc: Mark Brown <broonie@kernel.org>
Cc: Ming Lei <ming.lei@canonical.com>
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 0e742e92
...@@ -46,7 +46,8 @@ MODULE_LICENSE("GPL"); ...@@ -46,7 +46,8 @@ MODULE_LICENSE("GPL");
extern struct builtin_fw __start_builtin_fw[]; extern struct builtin_fw __start_builtin_fw[];
extern struct builtin_fw __end_builtin_fw[]; extern struct builtin_fw __end_builtin_fw[];
static bool fw_get_builtin_firmware(struct firmware *fw, const char *name) static bool fw_get_builtin_firmware(struct firmware *fw, const char *name,
void *buf, size_t size)
{ {
struct builtin_fw *b_fw; struct builtin_fw *b_fw;
...@@ -54,6 +55,9 @@ static bool fw_get_builtin_firmware(struct firmware *fw, const char *name) ...@@ -54,6 +55,9 @@ static bool fw_get_builtin_firmware(struct firmware *fw, const char *name)
if (strcmp(name, b_fw->name) == 0) { if (strcmp(name, b_fw->name) == 0) {
fw->size = b_fw->size; fw->size = b_fw->size;
fw->data = b_fw->data; fw->data = b_fw->data;
if (buf && fw->size <= size)
memcpy(buf, fw->data, fw->size);
return true; return true;
} }
} }
...@@ -74,7 +78,9 @@ static bool fw_is_builtin_firmware(const struct firmware *fw) ...@@ -74,7 +78,9 @@ static bool fw_is_builtin_firmware(const struct firmware *fw)
#else /* Module case - no builtin firmware support */ #else /* Module case - no builtin firmware support */
static inline bool fw_get_builtin_firmware(struct firmware *fw, const char *name) static inline bool fw_get_builtin_firmware(struct firmware *fw,
const char *name, void *buf,
size_t size)
{ {
return false; return false;
} }
...@@ -144,6 +150,7 @@ struct firmware_buf { ...@@ -144,6 +150,7 @@ struct firmware_buf {
unsigned long status; unsigned long status;
void *data; void *data;
size_t size; size_t size;
size_t allocated_size;
#ifdef CONFIG_FW_LOADER_USER_HELPER #ifdef CONFIG_FW_LOADER_USER_HELPER
bool is_paged_buf; bool is_paged_buf;
bool need_uevent; bool need_uevent;
...@@ -179,7 +186,8 @@ static DEFINE_MUTEX(fw_lock); ...@@ -179,7 +186,8 @@ static DEFINE_MUTEX(fw_lock);
static struct firmware_cache fw_cache; static struct firmware_cache fw_cache;
static struct firmware_buf *__allocate_fw_buf(const char *fw_name, static struct firmware_buf *__allocate_fw_buf(const char *fw_name,
struct firmware_cache *fwc) struct firmware_cache *fwc,
void *dbuf, size_t size)
{ {
struct firmware_buf *buf; struct firmware_buf *buf;
...@@ -195,6 +203,8 @@ static struct firmware_buf *__allocate_fw_buf(const char *fw_name, ...@@ -195,6 +203,8 @@ static struct firmware_buf *__allocate_fw_buf(const char *fw_name,
kref_init(&buf->ref); kref_init(&buf->ref);
buf->fwc = fwc; buf->fwc = fwc;
buf->data = dbuf;
buf->allocated_size = size;
init_completion(&buf->completion); init_completion(&buf->completion);
#ifdef CONFIG_FW_LOADER_USER_HELPER #ifdef CONFIG_FW_LOADER_USER_HELPER
INIT_LIST_HEAD(&buf->pending_list); INIT_LIST_HEAD(&buf->pending_list);
...@@ -218,7 +228,8 @@ static struct firmware_buf *__fw_lookup_buf(const char *fw_name) ...@@ -218,7 +228,8 @@ static struct firmware_buf *__fw_lookup_buf(const char *fw_name)
static int fw_lookup_and_allocate_buf(const char *fw_name, static int fw_lookup_and_allocate_buf(const char *fw_name,
struct firmware_cache *fwc, struct firmware_cache *fwc,
struct firmware_buf **buf) struct firmware_buf **buf, void *dbuf,
size_t size)
{ {
struct firmware_buf *tmp; struct firmware_buf *tmp;
...@@ -230,7 +241,7 @@ static int fw_lookup_and_allocate_buf(const char *fw_name, ...@@ -230,7 +241,7 @@ static int fw_lookup_and_allocate_buf(const char *fw_name,
*buf = tmp; *buf = tmp;
return 1; return 1;
} }
tmp = __allocate_fw_buf(fw_name, fwc); tmp = __allocate_fw_buf(fw_name, fwc, dbuf, size);
if (tmp) if (tmp)
list_add(&tmp->list, &fwc->head); list_add(&tmp->list, &fwc->head);
spin_unlock(&fwc->lock); spin_unlock(&fwc->lock);
...@@ -262,6 +273,7 @@ static void __fw_free_buf(struct kref *ref) ...@@ -262,6 +273,7 @@ static void __fw_free_buf(struct kref *ref)
vfree(buf->pages); vfree(buf->pages);
} else } else
#endif #endif
if (!buf->allocated_size)
vfree(buf->data); vfree(buf->data);
kfree_const(buf->fw_id); kfree_const(buf->fw_id);
kfree(buf); kfree(buf);
...@@ -302,13 +314,21 @@ static void fw_finish_direct_load(struct device *device, ...@@ -302,13 +314,21 @@ static void fw_finish_direct_load(struct device *device,
mutex_unlock(&fw_lock); mutex_unlock(&fw_lock);
} }
static int fw_get_filesystem_firmware(struct device *device, static int
struct firmware_buf *buf) fw_get_filesystem_firmware(struct device *device, struct firmware_buf *buf)
{ {
loff_t size; loff_t size;
int i, len; int i, len;
int rc = -ENOENT; int rc = -ENOENT;
char *path; char *path;
enum kernel_read_file_id id = READING_FIRMWARE;
size_t msize = INT_MAX;
/* Already populated data member means we're loading into a buffer */
if (buf->data) {
id = READING_FIRMWARE_PREALLOC_BUFFER;
msize = buf->allocated_size;
}
path = __getname(); path = __getname();
if (!path) if (!path)
...@@ -327,8 +347,8 @@ static int fw_get_filesystem_firmware(struct device *device, ...@@ -327,8 +347,8 @@ static int fw_get_filesystem_firmware(struct device *device,
} }
buf->size = 0; buf->size = 0;
rc = kernel_read_file_from_path(path, &buf->data, &size, rc = kernel_read_file_from_path(path, &buf->data, &size, msize,
INT_MAX, READING_FIRMWARE); id);
if (rc) { if (rc) {
if (rc == -ENOENT) if (rc == -ENOENT)
dev_dbg(device, "loading %s failed with error %d\n", dev_dbg(device, "loading %s failed with error %d\n",
...@@ -692,6 +712,15 @@ static ssize_t firmware_loading_store(struct device *dev, ...@@ -692,6 +712,15 @@ static ssize_t firmware_loading_store(struct device *dev,
static DEVICE_ATTR(loading, 0644, firmware_loading_show, firmware_loading_store); static DEVICE_ATTR(loading, 0644, firmware_loading_show, firmware_loading_store);
static void firmware_rw_buf(struct firmware_buf *buf, char *buffer,
loff_t offset, size_t count, bool read)
{
if (read)
memcpy(buffer, buf->data + offset, count);
else
memcpy(buf->data + offset, buffer, count);
}
static void firmware_rw(struct firmware_buf *buf, char *buffer, static void firmware_rw(struct firmware_buf *buf, char *buffer,
loff_t offset, size_t count, bool read) loff_t offset, size_t count, bool read)
{ {
...@@ -739,6 +768,9 @@ static ssize_t firmware_data_read(struct file *filp, struct kobject *kobj, ...@@ -739,6 +768,9 @@ static ssize_t firmware_data_read(struct file *filp, struct kobject *kobj,
ret_count = count; ret_count = count;
if (buf->data)
firmware_rw_buf(buf, buffer, offset, count, true);
else
firmware_rw(buf, buffer, offset, count, true); firmware_rw(buf, buffer, offset, count, true);
out: out:
...@@ -815,12 +847,21 @@ static ssize_t firmware_data_write(struct file *filp, struct kobject *kobj, ...@@ -815,12 +847,21 @@ static ssize_t firmware_data_write(struct file *filp, struct kobject *kobj,
goto out; goto out;
} }
if (buf->data) {
if (offset + count > buf->allocated_size) {
retval = -ENOMEM;
goto out;
}
firmware_rw_buf(buf, buffer, offset, count, false);
retval = count;
} else {
retval = fw_realloc_buffer(fw_priv, offset + count); retval = fw_realloc_buffer(fw_priv, offset + count);
if (retval) if (retval)
goto out; goto out;
retval = count; retval = count;
firmware_rw(buf, buffer, offset, count, false); firmware_rw(buf, buffer, offset, count, false);
}
buf->size = max_t(size_t, offset + count, buf->size); buf->size = max_t(size_t, offset + count, buf->size);
out: out:
...@@ -890,6 +931,7 @@ static int _request_firmware_load(struct firmware_priv *fw_priv, ...@@ -890,6 +931,7 @@ static int _request_firmware_load(struct firmware_priv *fw_priv,
struct firmware_buf *buf = fw_priv->buf; struct firmware_buf *buf = fw_priv->buf;
/* fall back on userspace loading */ /* fall back on userspace loading */
if (!buf->data)
buf->is_paged_buf = true; buf->is_paged_buf = true;
dev_set_uevent_suppress(f_dev, true); dev_set_uevent_suppress(f_dev, true);
...@@ -925,7 +967,7 @@ static int _request_firmware_load(struct firmware_priv *fw_priv, ...@@ -925,7 +967,7 @@ static int _request_firmware_load(struct firmware_priv *fw_priv,
if (is_fw_load_aborted(buf)) if (is_fw_load_aborted(buf))
retval = -EAGAIN; retval = -EAGAIN;
else if (!buf->data) else if (buf->is_paged_buf && !buf->data)
retval = -ENOMEM; retval = -ENOMEM;
device_del(f_dev); device_del(f_dev);
...@@ -1008,7 +1050,7 @@ static int sync_cached_firmware_buf(struct firmware_buf *buf) ...@@ -1008,7 +1050,7 @@ static int sync_cached_firmware_buf(struct firmware_buf *buf)
*/ */
static int static int
_request_firmware_prepare(struct firmware **firmware_p, const char *name, _request_firmware_prepare(struct firmware **firmware_p, const char *name,
struct device *device) struct device *device, void *dbuf, size_t size)
{ {
struct firmware *firmware; struct firmware *firmware;
struct firmware_buf *buf; struct firmware_buf *buf;
...@@ -1021,12 +1063,12 @@ _request_firmware_prepare(struct firmware **firmware_p, const char *name, ...@@ -1021,12 +1063,12 @@ _request_firmware_prepare(struct firmware **firmware_p, const char *name,
return -ENOMEM; return -ENOMEM;
} }
if (fw_get_builtin_firmware(firmware, name)) { if (fw_get_builtin_firmware(firmware, name, dbuf, size)) {
dev_dbg(device, "using built-in %s\n", name); dev_dbg(device, "using built-in %s\n", name);
return 0; /* assigned */ return 0; /* assigned */
} }
ret = fw_lookup_and_allocate_buf(name, &fw_cache, &buf); ret = fw_lookup_and_allocate_buf(name, &fw_cache, &buf, dbuf, size);
/* /*
* bind with 'buf' now to avoid warning in failure path * bind with 'buf' now to avoid warning in failure path
...@@ -1089,7 +1131,8 @@ static int assign_firmware_buf(struct firmware *fw, struct device *device, ...@@ -1089,7 +1131,8 @@ static int assign_firmware_buf(struct firmware *fw, struct device *device,
/* called from request_firmware() and request_firmware_work_func() */ /* called from request_firmware() and request_firmware_work_func() */
static int static int
_request_firmware(const struct firmware **firmware_p, const char *name, _request_firmware(const struct firmware **firmware_p, const char *name,
struct device *device, unsigned int opt_flags) struct device *device, void *buf, size_t size,
unsigned int opt_flags)
{ {
struct firmware *fw = NULL; struct firmware *fw = NULL;
long timeout; long timeout;
...@@ -1103,7 +1146,7 @@ _request_firmware(const struct firmware **firmware_p, const char *name, ...@@ -1103,7 +1146,7 @@ _request_firmware(const struct firmware **firmware_p, const char *name,
goto out; goto out;
} }
ret = _request_firmware_prepare(&fw, name, device); ret = _request_firmware_prepare(&fw, name, device, buf, size);
if (ret <= 0) /* error or already assigned */ if (ret <= 0) /* error or already assigned */
goto out; goto out;
...@@ -1182,7 +1225,7 @@ request_firmware(const struct firmware **firmware_p, const char *name, ...@@ -1182,7 +1225,7 @@ request_firmware(const struct firmware **firmware_p, const char *name,
/* Need to pin this module until return */ /* Need to pin this module until return */
__module_get(THIS_MODULE); __module_get(THIS_MODULE);
ret = _request_firmware(firmware_p, name, device, ret = _request_firmware(firmware_p, name, device, NULL, 0,
FW_OPT_UEVENT | FW_OPT_FALLBACK); FW_OPT_UEVENT | FW_OPT_FALLBACK);
module_put(THIS_MODULE); module_put(THIS_MODULE);
return ret; return ret;
...@@ -1206,13 +1249,43 @@ int request_firmware_direct(const struct firmware **firmware_p, ...@@ -1206,13 +1249,43 @@ int request_firmware_direct(const struct firmware **firmware_p,
int ret; int ret;
__module_get(THIS_MODULE); __module_get(THIS_MODULE);
ret = _request_firmware(firmware_p, name, device, ret = _request_firmware(firmware_p, name, device, NULL, 0,
FW_OPT_UEVENT | FW_OPT_NO_WARN); FW_OPT_UEVENT | FW_OPT_NO_WARN);
module_put(THIS_MODULE); module_put(THIS_MODULE);
return ret; return ret;
} }
EXPORT_SYMBOL_GPL(request_firmware_direct); EXPORT_SYMBOL_GPL(request_firmware_direct);
/**
* request_firmware_into_buf - load firmware into a previously allocated buffer
* @firmware_p: pointer to firmware image
* @name: name of firmware file
* @device: device for which firmware is being loaded and DMA region allocated
* @buf: address of buffer to load firmware into
* @size: size of buffer
*
* This function works pretty much like request_firmware(), but it doesn't
* allocate a buffer to hold the firmware data. Instead, the firmware
* is loaded directly into the buffer pointed to by @buf and the @firmware_p
* data member is pointed at @buf.
*
* This function doesn't cache firmware either.
*/
int
request_firmware_into_buf(const struct firmware **firmware_p, const char *name,
struct device *device, void *buf, size_t size)
{
int ret;
__module_get(THIS_MODULE);
ret = _request_firmware(firmware_p, name, device, buf, size,
FW_OPT_UEVENT | FW_OPT_FALLBACK |
FW_OPT_NOCACHE);
module_put(THIS_MODULE);
return ret;
}
EXPORT_SYMBOL(request_firmware_into_buf);
/** /**
* release_firmware: - release the resource associated with a firmware image * release_firmware: - release the resource associated with a firmware image
* @fw: firmware resource to release * @fw: firmware resource to release
...@@ -1245,7 +1318,7 @@ static void request_firmware_work_func(struct work_struct *work) ...@@ -1245,7 +1318,7 @@ static void request_firmware_work_func(struct work_struct *work)
fw_work = container_of(work, struct firmware_work, work); fw_work = container_of(work, struct firmware_work, work);
_request_firmware(&fw, fw_work->name, fw_work->device, _request_firmware(&fw, fw_work->name, fw_work->device, NULL, 0,
fw_work->opt_flags); fw_work->opt_flags);
fw_work->cont(fw, fw_work->context); fw_work->cont(fw, fw_work->context);
put_device(fw_work->device); /* taken in request_firmware_nowait() */ put_device(fw_work->device); /* taken in request_firmware_nowait() */
...@@ -1378,7 +1451,7 @@ static int uncache_firmware(const char *fw_name) ...@@ -1378,7 +1451,7 @@ static int uncache_firmware(const char *fw_name)
pr_debug("%s: %s\n", __func__, fw_name); pr_debug("%s: %s\n", __func__, fw_name);
if (fw_get_builtin_firmware(&fw, fw_name)) if (fw_get_builtin_firmware(&fw, fw_name, NULL, 0))
return 0; return 0;
buf = fw_lookup_buf(fw_name); buf = fw_lookup_buf(fw_name);
......
...@@ -866,6 +866,7 @@ int kernel_read_file(struct file *file, void **buf, loff_t *size, ...@@ -866,6 +866,7 @@ int kernel_read_file(struct file *file, void **buf, loff_t *size,
goto out; goto out;
} }
if (id != READING_FIRMWARE_PREALLOC_BUFFER)
*buf = vmalloc(i_size); *buf = vmalloc(i_size);
if (!*buf) { if (!*buf) {
ret = -ENOMEM; ret = -ENOMEM;
...@@ -897,9 +898,11 @@ int kernel_read_file(struct file *file, void **buf, loff_t *size, ...@@ -897,9 +898,11 @@ int kernel_read_file(struct file *file, void **buf, loff_t *size,
out_free: out_free:
if (ret < 0) { if (ret < 0) {
if (id != READING_FIRMWARE_PREALLOC_BUFFER) {
vfree(*buf); vfree(*buf);
*buf = NULL; *buf = NULL;
} }
}
out: out:
allow_write_access(file); allow_write_access(file);
......
...@@ -47,6 +47,8 @@ int request_firmware_nowait( ...@@ -47,6 +47,8 @@ int request_firmware_nowait(
void (*cont)(const struct firmware *fw, void *context)); void (*cont)(const struct firmware *fw, void *context));
int request_firmware_direct(const struct firmware **fw, const char *name, int request_firmware_direct(const struct firmware **fw, const char *name,
struct device *device); struct device *device);
int request_firmware_into_buf(const struct firmware **firmware_p,
const char *name, struct device *device, void *buf, size_t size);
void release_firmware(const struct firmware *fw); void release_firmware(const struct firmware *fw);
#else #else
...@@ -75,5 +77,11 @@ static inline int request_firmware_direct(const struct firmware **fw, ...@@ -75,5 +77,11 @@ static inline int request_firmware_direct(const struct firmware **fw,
return -EINVAL; return -EINVAL;
} }
static inline int request_firmware_into_buf(const struct firmware **firmware_p,
const char *name, struct device *device, void *buf, size_t size)
{
return -EINVAL;
}
#endif #endif
#endif #endif
...@@ -2652,6 +2652,7 @@ extern int do_pipe_flags(int *, int); ...@@ -2652,6 +2652,7 @@ extern int do_pipe_flags(int *, int);
#define __kernel_read_file_id(id) \ #define __kernel_read_file_id(id) \
id(UNKNOWN, unknown) \ id(UNKNOWN, unknown) \
id(FIRMWARE, firmware) \ id(FIRMWARE, firmware) \
id(FIRMWARE_PREALLOC_BUFFER, firmware) \
id(MODULE, kernel-module) \ id(MODULE, kernel-module) \
id(KEXEC_IMAGE, kexec-image) \ id(KEXEC_IMAGE, kexec-image) \
id(KEXEC_INITRAMFS, kexec-initramfs) \ id(KEXEC_INITRAMFS, kexec-initramfs) \
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment