Commit 4b5b7999 authored by Christian Lamparter's avatar Christian Lamparter Committed by Herbert Xu

crypto: crypto4xx - fix stalls under heavy load

If the crypto4xx device is continuously loaded by dm-crypt
and ipsec work, it will start to work intermittent after a
few (between 20-30) seconds, hurting throughput and latency.

This patch contains various stability improvements in order
to fix this issue. So far, the hardware has survived more
than a day without suffering any stalls under the continuous
load.
Signed-off-by: default avatarChristian Lamparter <chunkeey@gmail.com>
Signed-off-by: default avatarHerbert Xu <herbert@gondor.apana.org.au>
parent 64e1062b
...@@ -280,17 +280,20 @@ static u32 crypto4xx_get_pd_from_pdr_nolock(struct crypto4xx_device *dev) ...@@ -280,17 +280,20 @@ static u32 crypto4xx_get_pd_from_pdr_nolock(struct crypto4xx_device *dev)
static u32 crypto4xx_put_pd_to_pdr(struct crypto4xx_device *dev, u32 idx) static u32 crypto4xx_put_pd_to_pdr(struct crypto4xx_device *dev, u32 idx)
{ {
struct pd_uinfo *pd_uinfo = &dev->pdr_uinfo[idx]; struct pd_uinfo *pd_uinfo = &dev->pdr_uinfo[idx];
u32 tail;
unsigned long flags; unsigned long flags;
spin_lock_irqsave(&dev->core_dev->lock, flags); spin_lock_irqsave(&dev->core_dev->lock, flags);
pd_uinfo->state = PD_ENTRY_FREE;
if (dev->pdr_tail != PPC4XX_LAST_PD) if (dev->pdr_tail != PPC4XX_LAST_PD)
dev->pdr_tail++; dev->pdr_tail++;
else else
dev->pdr_tail = 0; dev->pdr_tail = 0;
pd_uinfo->state = PD_ENTRY_FREE; tail = dev->pdr_tail;
spin_unlock_irqrestore(&dev->core_dev->lock, flags); spin_unlock_irqrestore(&dev->core_dev->lock, flags);
return 0; return tail;
} }
/** /**
...@@ -854,16 +857,16 @@ int crypto4xx_build_pd(struct crypto_async_request *req, ...@@ -854,16 +857,16 @@ int crypto4xx_build_pd(struct crypto_async_request *req,
} }
} }
sa->sa_command_1.bf.hash_crypto_offset = 0; pd->pd_ctl.w = PD_CTL_HOST_READY |
pd->pd_ctl.w = 0; ((crypto_tfm_alg_type(req->tfm) == CRYPTO_ALG_TYPE_AHASH) |
pd->pd_ctl.bf.hash_final = (crypto_tfm_alg_type(req->tfm) == CRYPTO_ALG_TYPE_AEAD) ?
(crypto_tfm_alg_type(req->tfm) == CRYPTO_ALG_TYPE_AHASH); PD_CTL_HASH_FINAL : 0);
pd->pd_ctl.bf.host_ready = 1;
pd->pd_ctl_len.w = 0x00400000 | datalen; pd->pd_ctl_len.w = 0x00400000 | datalen;
pd_uinfo->state = PD_ENTRY_INUSE | (is_busy ? PD_ENTRY_BUSY : 0); pd_uinfo->state = PD_ENTRY_INUSE | (is_busy ? PD_ENTRY_BUSY : 0);
wmb(); wmb();
/* write any value to push engine to read a pd */ /* write any value to push engine to read a pd */
writel(0, dev->ce_base + CRYPTO4XX_INT_DESCR_RD);
writel(1, dev->ce_base + CRYPTO4XX_INT_DESCR_RD); writel(1, dev->ce_base + CRYPTO4XX_INT_DESCR_RD);
return is_busy ? -EBUSY : -EINPROGRESS; return is_busy ? -EBUSY : -EINPROGRESS;
} }
...@@ -964,23 +967,23 @@ static void crypto4xx_bh_tasklet_cb(unsigned long data) ...@@ -964,23 +967,23 @@ static void crypto4xx_bh_tasklet_cb(unsigned long data)
struct crypto4xx_core_device *core_dev = dev_get_drvdata(dev); struct crypto4xx_core_device *core_dev = dev_get_drvdata(dev);
struct pd_uinfo *pd_uinfo; struct pd_uinfo *pd_uinfo;
struct ce_pd *pd; struct ce_pd *pd;
u32 tail; u32 tail = core_dev->dev->pdr_tail;
u32 head = core_dev->dev->pdr_head;
while (core_dev->dev->pdr_head != core_dev->dev->pdr_tail) { do {
tail = core_dev->dev->pdr_tail;
pd_uinfo = &core_dev->dev->pdr_uinfo[tail]; pd_uinfo = &core_dev->dev->pdr_uinfo[tail];
pd = &core_dev->dev->pdr[tail]; pd = &core_dev->dev->pdr[tail];
if ((pd_uinfo->state & PD_ENTRY_INUSE) && if ((pd_uinfo->state & PD_ENTRY_INUSE) &&
pd->pd_ctl.bf.pe_done && ((READ_ONCE(pd->pd_ctl.w) &
!pd->pd_ctl.bf.host_ready) { (PD_CTL_PE_DONE | PD_CTL_HOST_READY)) ==
pd->pd_ctl.bf.pe_done = 0; PD_CTL_PE_DONE)) {
crypto4xx_pd_done(core_dev->dev, tail); crypto4xx_pd_done(core_dev->dev, tail);
crypto4xx_put_pd_to_pdr(core_dev->dev, tail); tail = crypto4xx_put_pd_to_pdr(core_dev->dev, tail);
} else { } else {
/* if tail not done, break */ /* if tail not done, break */
break; break;
} }
} } while (head != tail);
} }
/** /**
......
...@@ -261,6 +261,9 @@ union ce_pd_ctl { ...@@ -261,6 +261,9 @@ union ce_pd_ctl {
} bf; } bf;
u32 w; u32 w;
} __attribute__((packed)); } __attribute__((packed));
#define PD_CTL_HASH_FINAL BIT(4)
#define PD_CTL_PE_DONE BIT(1)
#define PD_CTL_HOST_READY BIT(0)
union ce_pd_ctl_len { union ce_pd_ctl_len {
struct { struct {
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment