Skip to content

L
linux
Commit bc385447 authored by Tom Lendacky's avatar Tom Lendacky Committed by Herbert Xu
Browse files
Options

crypto: ccp - Use a single queue for proper ordering of tfm requests 

Move to a single queue to serialize requests within a tfm. When
testing using IPSec with a large number of network connections
the per cpu tfm queuing logic was not working properly.
Signed-off-by: default avatarTom Lendacky <thomas.lendacky@amd.com>
Signed-off-by: default avatarHerbert Xu <herbert@gondor.apana.org.au>
parent c11baa02
Hide whitespace changes
Inline Side-by-side
Showing with 48 additions and 116 deletions
drivers/crypto/ccp/ccp-crypto-main.c
View file @ bc385447
...@@ -38,23 +38,20 @@ MODULE_PARM_DESC(sha_disable, "Disable use of SHA - any non-zero value"); ...@@ -38,23 +38,20 @@ MODULE_PARM_DESC(sha_disable, "Disable use of SHA - any non-zero value");
static LIST_HEAD(hash_algs); static LIST_HEAD(hash_algs);
static LIST_HEAD(cipher_algs); static LIST_HEAD(cipher_algs);
/* For any tfm, requests for that tfm on the same CPU must be returned /* For any tfm, requests for that tfm must be returned on the order
* in the order received. With multiple queues available, the CCP can * received. With multiple queues available, the CCP can process more
* process more than one cmd at a time. Therefore we must maintain * than one cmd at a time. Therefore we must maintain a cmd list to insure
* a cmd list to insure the proper ordering of requests on a given tfm/cpu * the proper ordering of requests on a given tfm.
* combination.
*/ */
struct ccp_crypto_cpu_queue { struct ccp_crypto_queue {
struct list_head cmds; struct list_head cmds;
struct list_head *backlog; struct list_head *backlog;
unsigned int cmd_count; unsigned int cmd_count;
}; };
#define CCP_CRYPTO_MAX_QLEN 50 #define CCP_CRYPTO_MAX_QLEN 100
struct ccp_crypto_percpu_queue { static struct ccp_crypto_queue req_queue;
struct ccp_crypto_cpu_queue __percpu *cpu_queue; static spinlock_t req_queue_lock;
};
static struct ccp_crypto_percpu_queue req_queue;
struct ccp_crypto_cmd { struct ccp_crypto_cmd {
struct list_head entry; struct list_head entry;
...@@ -71,8 +68,6 @@ struct ccp_crypto_cmd { ...@@ -71,8 +68,6 @@ struct ccp_crypto_cmd {
/* Used for held command processing to determine state */ /* Used for held command processing to determine state */
int ret; int ret;
int cpu;
}; };
struct ccp_crypto_cpu { struct ccp_crypto_cpu {
...@@ -91,25 +86,21 @@ static inline bool ccp_crypto_success(int err) ...@@ -91,25 +86,21 @@ static inline bool ccp_crypto_success(int err)
return true; return true;
} }
/*
* ccp_crypto_cmd_complete must be called while running on the appropriate
* cpu and the caller must have done a get_cpu to disable preemption
*/
static struct ccp_crypto_cmd *ccp_crypto_cmd_complete( static struct ccp_crypto_cmd *ccp_crypto_cmd_complete(
struct ccp_crypto_cmd *crypto_cmd, struct ccp_crypto_cmd **backlog) struct ccp_crypto_cmd *crypto_cmd, struct ccp_crypto_cmd **backlog)
{ {
struct ccp_crypto_cpu_queue *cpu_queue;
struct ccp_crypto_cmd *held = NULL, *tmp; struct ccp_crypto_cmd *held = NULL, *tmp;
unsigned long flags;
*backlog = NULL; *backlog = NULL;
cpu_queue = this_cpu_ptr(req_queue.cpu_queue); spin_lock_irqsave(&req_queue_lock, flags);
/* Held cmds will be after the current cmd in the queue so start /* Held cmds will be after the current cmd in the queue so start
* searching for a cmd with a matching tfm for submission. * searching for a cmd with a matching tfm for submission.
*/ */
tmp = crypto_cmd; tmp = crypto_cmd;
list_for_each_entry_continue(tmp, &cpu_queue->cmds, entry) { list_for_each_entry_continue(tmp, &req_queue.cmds, entry) {
if (crypto_cmd->tfm != tmp->tfm) if (crypto_cmd->tfm != tmp->tfm)
continue; continue;
held = tmp; held = tmp;
...@@ -120,47 +111,45 @@ static struct ccp_crypto_cmd *ccp_crypto_cmd_complete( ...@@ -120,47 +111,45 @@ static struct ccp_crypto_cmd *ccp_crypto_cmd_complete(
* Because cmds can be executed from any point in the cmd list * Because cmds can be executed from any point in the cmd list
* special precautions have to be taken when handling the backlog. * special precautions have to be taken when handling the backlog.
*/ */
if (cpu_queue->backlog != &cpu_queue->cmds) { if (req_queue.backlog != &req_queue.cmds) {
/* Skip over this cmd if it is the next backlog cmd */ /* Skip over this cmd if it is the next backlog cmd */
if (cpu_queue->backlog == &crypto_cmd->entry) if (req_queue.backlog == &crypto_cmd->entry)
cpu_queue->backlog = crypto_cmd->entry.next; req_queue.backlog = crypto_cmd->entry.next;
*backlog = container_of(cpu_queue->backlog, *backlog = container_of(req_queue.backlog,
struct ccp_crypto_cmd, entry); struct ccp_crypto_cmd, entry);
cpu_queue->backlog = cpu_queue->backlog->next; req_queue.backlog = req_queue.backlog->next;
/* Skip over this cmd if it is now the next backlog cmd */ /* Skip over this cmd if it is now the next backlog cmd */
if (cpu_queue->backlog == &crypto_cmd->entry) if (req_queue.backlog == &crypto_cmd->entry)
cpu_queue->backlog = crypto_cmd->entry.next; req_queue.backlog = crypto_cmd->entry.next;
} }
/* Remove the cmd entry from the list of cmds */ /* Remove the cmd entry from the list of cmds */
cpu_queue->cmd_count--; req_queue.cmd_count--;
list_del(&crypto_cmd->entry); list_del(&crypto_cmd->entry);
spin_unlock_irqrestore(&req_queue_lock, flags);
return held; return held;
} }
static void ccp_crypto_complete_on_cpu(struct work_struct *work) static void ccp_crypto_complete(void *data, int err)
{ {
struct ccp_crypto_cpu *cpu_work = struct ccp_crypto_cmd *crypto_cmd = data;
container_of(work, struct ccp_crypto_cpu, work);
struct ccp_crypto_cmd *crypto_cmd = cpu_work->crypto_cmd;
struct ccp_crypto_cmd *held, *next, *backlog; struct ccp_crypto_cmd *held, *next, *backlog;
struct crypto_async_request *req = crypto_cmd->req; struct crypto_async_request *req = crypto_cmd->req;
struct ccp_ctx *ctx = crypto_tfm_ctx(req->tfm); struct ccp_ctx *ctx = crypto_tfm_ctx(req->tfm);
int cpu, ret; int ret;
cpu = get_cpu();
if (cpu_work->err == -EINPROGRESS) { if (err == -EINPROGRESS) {
/* Only propogate the -EINPROGRESS if necessary */ /* Only propogate the -EINPROGRESS if necessary */
if (crypto_cmd->ret == -EBUSY) { if (crypto_cmd->ret == -EBUSY) {
crypto_cmd->ret = -EINPROGRESS; crypto_cmd->ret = -EINPROGRESS;
req->complete(req, -EINPROGRESS); req->complete(req, -EINPROGRESS);
} }
goto e_cpu; return;
} }
/* Operation has completed - update the queue before invoking /* Operation has completed - update the queue before invoking
...@@ -178,7 +167,7 @@ static void ccp_crypto_complete_on_cpu(struct work_struct *work) ...@@ -178,7 +167,7 @@ static void ccp_crypto_complete_on_cpu(struct work_struct *work)
req->complete(req, -EINPROGRESS); req->complete(req, -EINPROGRESS);
/* Completion callbacks */ /* Completion callbacks */
ret = cpu_work->err; ret = err;
if (ctx->complete) if (ctx->complete)
ret = ctx->complete(req, ret); ret = ctx->complete(req, ret);
req->complete(req, ret); req->complete(req, ret);
...@@ -203,52 +192,28 @@ static void ccp_crypto_complete_on_cpu(struct work_struct *work) ...@@ -203,52 +192,28 @@ static void ccp_crypto_complete_on_cpu(struct work_struct *work)
} }
kfree(crypto_cmd); kfree(crypto_cmd);
e_cpu:
put_cpu();
complete(&cpu_work->completion);
}
static void ccp_crypto_complete(void *data, int err)
{
struct ccp_crypto_cmd *crypto_cmd = data;
struct ccp_crypto_cpu cpu_work;
INIT_WORK(&cpu_work.work, ccp_crypto_complete_on_cpu);
init_completion(&cpu_work.completion);
cpu_work.crypto_cmd = crypto_cmd;
cpu_work.err = err;
schedule_work_on(crypto_cmd->cpu, &cpu_work.work);
/* Keep the completion call synchronous */
wait_for_completion(&cpu_work.completion);
} }
static int ccp_crypto_enqueue_cmd(struct ccp_crypto_cmd *crypto_cmd) static int ccp_crypto_enqueue_cmd(struct ccp_crypto_cmd *crypto_cmd)
{ {
struct ccp_crypto_cpu_queue *cpu_queue;
struct ccp_crypto_cmd *active = NULL, *tmp; struct ccp_crypto_cmd *active = NULL, *tmp;
int cpu, ret; unsigned long flags;
int ret;
cpu = get_cpu();
crypto_cmd->cpu = cpu;
cpu_queue = this_cpu_ptr(req_queue.cpu_queue); spin_lock_irqsave(&req_queue_lock, flags);
/* Check if the cmd can/should be queued */ /* Check if the cmd can/should be queued */
if (cpu_queue->cmd_count >= CCP_CRYPTO_MAX_QLEN) { if (req_queue.cmd_count >= CCP_CRYPTO_MAX_QLEN) {
ret = -EBUSY; ret = -EBUSY;
if (!(crypto_cmd->cmd->flags & CCP_CMD_MAY_BACKLOG)) if (!(crypto_cmd->cmd->flags & CCP_CMD_MAY_BACKLOG))
goto e_cpu; goto e_lock;
} }
/* Look for an entry with the same tfm. If there is a cmd /* Look for an entry with the same tfm. If there is a cmd
* with the same tfm in the list for this cpu then the current * with the same tfm in the list then the current cmd cannot
* cmd cannot be submitted to the CCP yet. * be submitted to the CCP yet.
*/ */
list_for_each_entry(tmp, &cpu_queue->cmds, entry) { list_for_each_entry(tmp, &req_queue.cmds, entry) {
if (crypto_cmd->tfm != tmp->tfm) if (crypto_cmd->tfm != tmp->tfm)
continue; continue;
active = tmp; active = tmp;
...@@ -259,21 +224,21 @@ static int ccp_crypto_enqueue_cmd(struct ccp_crypto_cmd *crypto_cmd) ...@@ -259,21 +224,21 @@ static int ccp_crypto_enqueue_cmd(struct ccp_crypto_cmd *crypto_cmd)
if (!active) { if (!active) {
ret = ccp_enqueue_cmd(crypto_cmd->cmd); ret = ccp_enqueue_cmd(crypto_cmd->cmd);
if (!ccp_crypto_success(ret)) if (!ccp_crypto_success(ret))
goto e_cpu; goto e_lock;
} }
if (cpu_queue->cmd_count >= CCP_CRYPTO_MAX_QLEN) { if (req_queue.cmd_count >= CCP_CRYPTO_MAX_QLEN) {
ret = -EBUSY; ret = -EBUSY;
if (cpu_queue->backlog == &cpu_queue->cmds) if (req_queue.backlog == &req_queue.cmds)
cpu_queue->backlog = &crypto_cmd->entry; req_queue.backlog = &crypto_cmd->entry;
} }
crypto_cmd->ret = ret; crypto_cmd->ret = ret;
cpu_queue->cmd_count++; req_queue.cmd_count++;
list_add_tail(&crypto_cmd->entry, &cpu_queue->cmds); list_add_tail(&crypto_cmd->entry, &req_queue.cmds);
e_cpu: e_lock:
put_cpu(); spin_unlock_irqrestore(&req_queue_lock, flags);
return ret; return ret;
} }
...@@ -387,50 +352,18 @@ static void ccp_unregister_algs(void) ...@@ -387,50 +352,18 @@ static void ccp_unregister_algs(void)
} }
} }
static int ccp_init_queues(void)
{
struct ccp_crypto_cpu_queue *cpu_queue;
int cpu;
req_queue.cpu_queue = alloc_percpu(struct ccp_crypto_cpu_queue);
if (!req_queue.cpu_queue)
return -ENOMEM;
for_each_possible_cpu(cpu) {
cpu_queue = per_cpu_ptr(req_queue.cpu_queue, cpu);
INIT_LIST_HEAD(&cpu_queue->cmds);
cpu_queue->backlog = &cpu_queue->cmds;
cpu_queue->cmd_count = 0;
}
return 0;
}
static void ccp_fini_queue(void)
{
struct ccp_crypto_cpu_queue *cpu_queue;
int cpu;
for_each_possible_cpu(cpu) {
cpu_queue = per_cpu_ptr(req_queue.cpu_queue, cpu);
BUG_ON(!list_empty(&cpu_queue->cmds));
}
free_percpu(req_queue.cpu_queue);
}
static int ccp_crypto_init(void) static int ccp_crypto_init(void)
{ {
int ret; int ret;
ret = ccp_init_queues(); spin_lock_init(&req_queue_lock);
if (ret) INIT_LIST_HEAD(&req_queue.cmds);
return ret; req_queue.backlog = &req_queue.cmds;
req_queue.cmd_count = 0;
ret = ccp_register_algs(); ret = ccp_register_algs();
if (ret) { if (ret)
ccp_unregister_algs(); ccp_unregister_algs();
ccp_fini_queue();
}
return ret; return ret;
} }
...@@ -438,7 +371,6 @@ static int ccp_crypto_init(void) ...@@ -438,7 +371,6 @@ static int ccp_crypto_init(void)
static void ccp_crypto_exit(void) static void ccp_crypto_exit(void)
{ {
ccp_unregister_algs(); ccp_unregister_algs();
ccp_fini_queue();
} }
module_init(ccp_crypto_init); module_init(ccp_crypto_init);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7