Commit 147a1c06 authored by David S. Miller's avatar David S. Miller

Merge branch 'ionic-diet'

Shannon Nelson says:

====================
ionic: putting ionic on a diet

Building on the performance work done in the previous patchset
    [Link] https://lore.kernel.org/netdev/20240229193935.14197-1-shannon.nelson@amd.com/
this patchset puts the ionic driver on a diet, decreasing the memory
requirements per queue, and simplifies a few more bits of logic.

We trimmed the queue management structs and gained some ground, but
the most savings came from trimming the individual buffer descriptors.
The original design used a single generic buffer descriptor for Tx, Rx and
Adminq needs, but the Rx and Adminq descriptors really don't need all the
info that the Tx descriptors track.  By splitting up the descriptor types
we can significantly reduce the descriptor sizes for Rx and Adminq use.

There is a small reduction in the queue management structs, saving about
3 cachelines per queuepair:

    ionic_qcq:
	Before:	/* size: 2176, cachelines: 34, members: 23 */
	After:	/* size: 2048, cachelines: 32, members: 23 */

We also remove an array of completion descriptor pointers, or about
8 Kbytes per queue.

But the biggest savings came from splitting the desc_info struct into
queue specific structs and trimming out what was unnecessary.

    Before:
	ionic_desc_info:
		/* size: 496, cachelines: 8, members: 10 */
    After:
	ionic_tx_desc_info:
		/* size: 496, cachelines: 8, members: 6 */
	ionic_rx_desc_info:
		/* size: 224, cachelines: 4, members: 2 */
	ionic_admin_desc_info:
		/* size: 8, cachelines: 1, members: 1 */

In a 64 core host the ionic driver will default to 64 queuepairs of
1024 descriptors for Rx, 1024 for Tx, and 80 for Adminq and Notifyq.

The total memory usage for 64 queues:
    Before:
	  65 * sizeof(ionic_qcq)			   141,440
	+ 64 * 1024 * sizeof(ionic_desc_info)		32,505,856
	+ 64 * 1024 * sizeof(ionic_desc_info)		32,505,856
	+ 64 * 1024 * 2 * sizeof(ionic_qc_info)		    16,384
	+  1 *   80 * sizeof(ionic_desc_info)		    39,690
							----------
							65,201,038

    After:
	  65 * sizeof(ionic_qcq)			   133,120
	+ 64 * 1024 * sizeof(ionic_tx_desc_info)	32,505,856
	+ 64 * 1024 * sizeof(ionic_rx_desc_info)	14,680,064
	+                           (removed)		         0
	+  1 *   80 * sizeof(ionic_admin desc_info)	       640
							----------
							47,319,680

This saves us approximately 18 Mbytes per port in a 64 core machine,
a 28% savings in our memory needs.

In addition, this improves our simple single thread / single queue
iperf case on a 9100 MTU connection from 86.7 to 95 Gbits/sec.
====================
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parents e3eec349 2854242d
......@@ -76,6 +76,8 @@ int ionic_adminq_post_wait(struct ionic_lif *lif, struct ionic_admin_ctx *ctx);
int ionic_adminq_post_wait_nomsg(struct ionic_lif *lif, struct ionic_admin_ctx *ctx);
void ionic_adminq_netdev_err_print(struct ionic_lif *lif, u8 opcode,
u8 status, int err);
bool ionic_notifyq_service(struct ionic_cq *cq);
bool ionic_adminq_service(struct ionic_cq *cq);
int ionic_dev_cmd_wait(struct ionic *ionic, unsigned long max_wait);
int ionic_dev_cmd_wait_nomsg(struct ionic *ionic, unsigned long max_wait);
......
......@@ -629,43 +629,25 @@ int ionic_cq_init(struct ionic_lif *lif, struct ionic_cq *cq,
cq->desc_size = desc_size;
cq->tail_idx = 0;
cq->done_color = 1;
cq->idev = &lif->ionic->idev;
return 0;
}
void ionic_cq_map(struct ionic_cq *cq, void *base, dma_addr_t base_pa)
{
struct ionic_cq_info *cur;
unsigned int i;
cq->base = base;
cq->base_pa = base_pa;
for (i = 0, cur = cq->info; i < cq->num_descs; i++, cur++)
cur->cq_desc = base + (i * cq->desc_size);
}
void ionic_cq_bind(struct ionic_cq *cq, struct ionic_queue *q)
{
cq->bound_q = q;
}
unsigned int ionic_cq_service(struct ionic_cq *cq, unsigned int work_to_do,
ionic_cq_cb cb, ionic_cq_done_cb done_cb,
void *done_arg)
{
struct ionic_cq_info *cq_info;
unsigned int work_done = 0;
if (work_to_do == 0)
return 0;
cq_info = &cq->info[cq->tail_idx];
while (cb(cq, cq_info)) {
while (cb(cq)) {
if (cq->tail_idx == cq->num_descs - 1)
cq->done_color = !cq->done_color;
cq->tail_idx = (cq->tail_idx + 1) & (cq->num_descs - 1);
cq_info = &cq->info[cq->tail_idx];
if (++work_done >= work_to_do)
break;
......@@ -692,7 +674,6 @@ int ionic_q_init(struct ionic_lif *lif, struct ionic_dev *idev,
return -EINVAL;
q->lif = lif;
q->idev = idev;
q->index = index;
q->num_descs = num_descs;
q->desc_size = desc_size;
......@@ -706,53 +687,11 @@ int ionic_q_init(struct ionic_lif *lif, struct ionic_dev *idev,
return 0;
}
void ionic_q_map(struct ionic_queue *q, void *base, dma_addr_t base_pa)
void ionic_q_post(struct ionic_queue *q, bool ring_doorbell)
{
struct ionic_desc_info *cur;
unsigned int i;
q->base = base;
q->base_pa = base_pa;
for (i = 0, cur = q->info; i < q->num_descs; i++, cur++)
cur->desc = base + (i * q->desc_size);
}
void ionic_q_cmb_map(struct ionic_queue *q, void __iomem *base, dma_addr_t base_pa)
{
struct ionic_desc_info *cur;
unsigned int i;
q->cmb_base = base;
q->cmb_base_pa = base_pa;
for (i = 0, cur = q->info; i < q->num_descs; i++, cur++)
cur->cmb_desc = base + (i * q->desc_size);
}
void ionic_q_sg_map(struct ionic_queue *q, void *base, dma_addr_t base_pa)
{
struct ionic_desc_info *cur;
unsigned int i;
q->sg_base = base;
q->sg_base_pa = base_pa;
for (i = 0, cur = q->info; i < q->num_descs; i++, cur++)
cur->sg_desc = base + (i * q->sg_desc_size);
}
void ionic_q_post(struct ionic_queue *q, bool ring_doorbell, ionic_desc_cb cb,
void *cb_arg)
{
struct ionic_desc_info *desc_info;
struct ionic_lif *lif = q->lif;
struct device *dev = q->dev;
desc_info = &q->info[q->head_idx];
desc_info->cb = cb;
desc_info->cb_arg = cb_arg;
q->head_idx = (q->head_idx + 1) & (q->num_descs - 1);
dev_dbg(dev, "lif=%d qname=%s qid=%d qtype=%d p_index=%d ringdb=%d\n",
......@@ -771,7 +710,7 @@ void ionic_q_post(struct ionic_queue *q, bool ring_doorbell, ionic_desc_cb cb,
}
}
static bool ionic_q_is_posted(struct ionic_queue *q, unsigned int pos)
bool ionic_q_is_posted(struct ionic_queue *q, unsigned int pos)
{
unsigned int mask, tail, head;
......@@ -781,37 +720,3 @@ static bool ionic_q_is_posted(struct ionic_queue *q, unsigned int pos)
return ((pos - tail) & mask) < ((head - tail) & mask);
}
void ionic_q_service(struct ionic_queue *q, struct ionic_cq_info *cq_info,
unsigned int stop_index)
{
struct ionic_desc_info *desc_info;
ionic_desc_cb cb;
void *cb_arg;
u16 index;
/* check for empty queue */
if (q->tail_idx == q->head_idx)
return;
/* stop index must be for a descriptor that is not yet completed */
if (unlikely(!ionic_q_is_posted(q, stop_index)))
dev_err(q->dev,
"ionic stop is not posted %s stop %u tail %u head %u\n",
q->name, stop_index, q->tail_idx, q->head_idx);
do {
desc_info = &q->info[q->tail_idx];
index = q->tail_idx;
q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1);
cb = desc_info->cb;
cb_arg = desc_info->cb_arg;
desc_info->cb = NULL;
desc_info->cb_arg = NULL;
if (cb)
cb(q, desc_info, cq_info, cb_arg);
} while (index != stop_index);
}
......@@ -122,11 +122,13 @@ static_assert(sizeof(struct ionic_log_event) == 64);
/* I/O */
static_assert(sizeof(struct ionic_txq_desc) == 16);
static_assert(sizeof(struct ionic_txq_sg_desc) == 128);
static_assert(sizeof(struct ionic_txq_sg_desc_v1) == 256);
static_assert(sizeof(struct ionic_txq_comp) == 16);
static_assert(sizeof(struct ionic_rxq_desc) == 16);
static_assert(sizeof(struct ionic_rxq_sg_desc) == 128);
static_assert(sizeof(struct ionic_rxq_comp) == 16);
static_assert(sizeof(struct ionic_rxq_comp) == sizeof(struct ionic_txq_comp));
/* SR/IOV */
static_assert(sizeof(struct ionic_vf_setattr_cmd) == 64);
......@@ -175,21 +177,8 @@ struct ionic_dev {
struct ionic_devinfo dev_info;
};
struct ionic_cq_info {
union {
void *cq_desc;
struct ionic_admin_comp *admincq;
struct ionic_notifyq_event *notifyq;
};
};
struct ionic_queue;
struct ionic_qcq;
struct ionic_desc_info;
typedef void (*ionic_desc_cb)(struct ionic_queue *q,
struct ionic_desc_info *desc_info,
struct ionic_cq_info *cq_info, void *cb_arg);
#define IONIC_MAX_BUF_LEN ((u16)-1)
#define IONIC_PAGE_SIZE PAGE_SIZE
......@@ -209,28 +198,25 @@ struct ionic_buf_info {
u32 len;
};
#define IONIC_MAX_FRAGS (1 + IONIC_TX_MAX_SG_ELEMS_V1)
#define IONIC_TX_MAX_FRAGS (1 + IONIC_TX_MAX_SG_ELEMS_V1)
#define IONIC_RX_MAX_FRAGS (1 + IONIC_RX_MAX_SG_ELEMS)
struct ionic_desc_info {
union {
void *desc;
struct ionic_txq_desc *txq_desc;
struct ionic_rxq_desc *rxq_desc;
struct ionic_admin_cmd *adminq_desc;
};
void __iomem *cmb_desc;
union {
void *sg_desc;
struct ionic_txq_sg_desc *txq_sg_desc;
struct ionic_rxq_sg_desc *rxq_sgl_desc;
};
struct ionic_tx_desc_info {
unsigned int bytes;
unsigned int nbufs;
struct ionic_buf_info bufs[MAX_SKB_FRAGS + 1];
ionic_desc_cb cb;
void *cb_arg;
struct sk_buff *skb;
struct xdp_frame *xdpf;
enum xdp_action act;
struct ionic_buf_info bufs[MAX_SKB_FRAGS + 1];
};
struct ionic_rx_desc_info {
unsigned int nbufs;
struct ionic_buf_info bufs[IONIC_RX_MAX_FRAGS];
};
struct ionic_admin_desc_info {
void *ctx;
};
#define IONIC_QUEUE_NAME_MAX_SZ 16
......@@ -238,7 +224,12 @@ struct ionic_desc_info {
struct ionic_queue {
struct device *dev;
struct ionic_lif *lif;
struct ionic_desc_info *info;
union {
void *info;
struct ionic_tx_desc_info *tx_info;
struct ionic_rx_desc_info *rx_info;
struct ionic_admin_desc_info *admin_info;
};
u64 dbval;
unsigned long dbell_deadline;
unsigned long dbell_jiffies;
......@@ -248,29 +239,33 @@ struct ionic_queue {
unsigned int num_descs;
unsigned int max_sg_elems;
u64 features;
u64 drop;
struct ionic_dev *idev;
unsigned int type;
unsigned int hw_index;
unsigned int hw_type;
bool xdp_flush;
union {
void *base;
struct ionic_txq_desc *txq;
struct ionic_rxq_desc *rxq;
struct ionic_admin_cmd *adminq;
};
union {
void __iomem *cmb_base;
struct ionic_txq_desc __iomem *cmb_txq;
struct ionic_rxq_desc __iomem *cmb_rxq;
};
union {
void *sg_base;
struct ionic_txq_sg_desc *txq_sgl;
struct ionic_txq_sg_desc_v1 *txq_sgl_v1;
struct ionic_rxq_sg_desc *rxq_sgl;
};
struct xdp_rxq_info *xdp_rxq_info;
struct ionic_queue *partner;
bool xdp_flush;
dma_addr_t base_pa;
dma_addr_t cmb_base_pa;
dma_addr_t sg_base_pa;
u64 drop;
unsigned int desc_size;
unsigned int sg_desc_size;
unsigned int pid;
......@@ -292,7 +287,6 @@ struct ionic_intr_info {
struct ionic_cq {
struct ionic_lif *lif;
struct ionic_cq_info *info;
struct ionic_queue *bound_q;
struct ionic_intr_info *bound_intr;
u16 tail_idx;
......@@ -301,6 +295,7 @@ struct ionic_cq {
unsigned int desc_size;
void *base;
dma_addr_t base_pa;
struct ionic_dev *idev;
} ____cacheline_aligned_in_smp;
struct ionic;
......@@ -375,7 +370,7 @@ int ionic_cq_init(struct ionic_lif *lif, struct ionic_cq *cq,
unsigned int num_descs, size_t desc_size);
void ionic_cq_map(struct ionic_cq *cq, void *base, dma_addr_t base_pa);
void ionic_cq_bind(struct ionic_cq *cq, struct ionic_queue *q);
typedef bool (*ionic_cq_cb)(struct ionic_cq *cq, struct ionic_cq_info *cq_info);
typedef bool (*ionic_cq_cb)(struct ionic_cq *cq);
typedef void (*ionic_cq_done_cb)(void *done_arg);
unsigned int ionic_cq_service(struct ionic_cq *cq, unsigned int work_to_do,
ionic_cq_cb cb, ionic_cq_done_cb done_cb,
......@@ -386,13 +381,9 @@ int ionic_q_init(struct ionic_lif *lif, struct ionic_dev *idev,
struct ionic_queue *q, unsigned int index, const char *name,
unsigned int num_descs, size_t desc_size,
size_t sg_desc_size, unsigned int pid);
void ionic_q_map(struct ionic_queue *q, void *base, dma_addr_t base_pa);
void ionic_q_cmb_map(struct ionic_queue *q, void __iomem *base, dma_addr_t base_pa);
void ionic_q_sg_map(struct ionic_queue *q, void *base, dma_addr_t base_pa);
void ionic_q_post(struct ionic_queue *q, bool ring_doorbell, ionic_desc_cb cb,
void *cb_arg);
void ionic_q_service(struct ionic_queue *q, struct ionic_cq_info *cq_info,
unsigned int stop_index);
void ionic_q_post(struct ionic_queue *q, bool ring_doorbell);
bool ionic_q_is_posted(struct ionic_queue *q, unsigned int pos);
int ionic_heartbeat_check(struct ionic *ionic);
bool ionic_is_fw_running(struct ionic_dev *idev);
......
......@@ -71,25 +71,25 @@ struct ionic_qcq {
void *q_base;
dma_addr_t q_base_pa;
u32 q_size;
u32 cq_size;
void *cq_base;
dma_addr_t cq_base_pa;
u32 cq_size;
void *sg_base;
dma_addr_t sg_base_pa;
u32 sg_size;
unsigned int flags;
void __iomem *cmb_q_base;
phys_addr_t cmb_q_base_pa;
u32 cmb_q_size;
u32 cmb_pgid;
u32 cmb_order;
struct dim dim;
struct timer_list napi_deadline;
struct ionic_queue q;
struct ionic_cq cq;
struct ionic_intr_info intr;
struct timer_list napi_deadline;
struct napi_struct napi;
unsigned int flags;
struct ionic_qcq *napi_qcq;
struct ionic_intr_info intr;
struct dentry *dentry;
};
......
......@@ -190,7 +190,8 @@ static const char *ionic_opcode_to_str(enum ionic_cmd_opcode opcode)
static void ionic_adminq_flush(struct ionic_lif *lif)
{
struct ionic_desc_info *desc_info;
struct ionic_admin_desc_info *desc_info;
struct ionic_admin_cmd *desc;
unsigned long irqflags;
struct ionic_queue *q;
......@@ -203,10 +204,10 @@ static void ionic_adminq_flush(struct ionic_lif *lif)
q = &lif->adminqcq->q;
while (q->tail_idx != q->head_idx) {
desc_info = &q->info[q->tail_idx];
memset(desc_info->desc, 0, sizeof(union ionic_adminq_cmd));
desc_info->cb = NULL;
desc_info->cb_arg = NULL;
desc = &q->adminq[q->tail_idx];
desc_info = &q->admin_info[q->tail_idx];
memset(desc, 0, sizeof(union ionic_adminq_cmd));
desc_info->ctx = NULL;
q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1);
}
spin_unlock_irqrestore(&lif->adminq_lock, irqflags);
......@@ -246,25 +247,93 @@ static int ionic_adminq_check_err(struct ionic_lif *lif,
return err;
}
static void ionic_adminq_cb(struct ionic_queue *q,
struct ionic_desc_info *desc_info,
struct ionic_cq_info *cq_info, void *cb_arg)
bool ionic_notifyq_service(struct ionic_cq *cq)
{
struct ionic_admin_ctx *ctx = cb_arg;
struct ionic_deferred_work *work;
union ionic_notifyq_comp *comp;
struct net_device *netdev;
struct ionic_queue *q;
struct ionic_lif *lif;
u64 eid;
comp = &((union ionic_notifyq_comp *)cq->base)[cq->tail_idx];
q = cq->bound_q;
lif = q->admin_info[0].ctx;
netdev = lif->netdev;
eid = le64_to_cpu(comp->event.eid);
/* Have we run out of new completions to process? */
if ((s64)(eid - lif->last_eid) <= 0)
return false;
lif->last_eid = eid;
dev_dbg(lif->ionic->dev, "notifyq event:\n");
dynamic_hex_dump("event ", DUMP_PREFIX_OFFSET, 16, 1,
comp, sizeof(*comp), true);
switch (le16_to_cpu(comp->event.ecode)) {
case IONIC_EVENT_LINK_CHANGE:
ionic_link_status_check_request(lif, CAN_NOT_SLEEP);
break;
case IONIC_EVENT_RESET:
if (lif->ionic->idev.fw_status_ready &&
!test_bit(IONIC_LIF_F_FW_RESET, lif->state) &&
!test_and_set_bit(IONIC_LIF_F_FW_STOPPING, lif->state)) {
work = kzalloc(sizeof(*work), GFP_ATOMIC);
if (!work) {
netdev_err(lif->netdev, "Reset event dropped\n");
clear_bit(IONIC_LIF_F_FW_STOPPING, lif->state);
} else {
work->type = IONIC_DW_TYPE_LIF_RESET;
ionic_lif_deferred_enqueue(&lif->deferred, work);
}
}
break;
default:
netdev_warn(netdev, "Notifyq event ecode=%d eid=%lld\n",
comp->event.ecode, eid);
break;
}
return true;
}
bool ionic_adminq_service(struct ionic_cq *cq)
{
struct ionic_admin_desc_info *desc_info;
struct ionic_queue *q = cq->bound_q;
struct ionic_admin_comp *comp;
u16 index;
if (!ctx)
return;
comp = &((struct ionic_admin_comp *)cq->base)[cq->tail_idx];
if (!color_match(comp->color, cq->done_color))
return false;
/* check for empty queue */
if (q->tail_idx == q->head_idx)
return false;
comp = cq_info->cq_desc;
do {
desc_info = &q->admin_info[q->tail_idx];
index = q->tail_idx;
q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1);
if (likely(desc_info->ctx)) {
struct ionic_admin_ctx *ctx = desc_info->ctx;
memcpy(&ctx->comp, comp, sizeof(*comp));
dev_dbg(q->dev, "comp admin queue command:\n");
dynamic_hex_dump("comp ", DUMP_PREFIX_OFFSET, 16, 1,
&ctx->comp, sizeof(ctx->comp), true);
complete_all(&ctx->work);
desc_info->ctx = NULL;
}
} while (index != le16_to_cpu(comp->comp_index));
return true;
}
bool ionic_adminq_poke_doorbell(struct ionic_queue *q)
......@@ -298,7 +367,8 @@ bool ionic_adminq_poke_doorbell(struct ionic_queue *q)
int ionic_adminq_post(struct ionic_lif *lif, struct ionic_admin_ctx *ctx)
{
struct ionic_desc_info *desc_info;
struct ionic_admin_desc_info *desc_info;
struct ionic_admin_cmd *desc;
unsigned long irqflags;
struct ionic_queue *q;
int err = 0;
......@@ -320,14 +390,17 @@ int ionic_adminq_post(struct ionic_lif *lif, struct ionic_admin_ctx *ctx)
if (err)
goto err_out;
desc_info = &q->info[q->head_idx];
memcpy(desc_info->desc, &ctx->cmd, sizeof(ctx->cmd));
desc_info = &q->admin_info[q->head_idx];
desc_info->ctx = ctx;
desc = &q->adminq[q->head_idx];
memcpy(desc, &ctx->cmd, sizeof(ctx->cmd));
dev_dbg(&lif->netdev->dev, "post admin queue command:\n");
dynamic_hex_dump("cmd ", DUMP_PREFIX_OFFSET, 16, 1,
&ctx->cmd, sizeof(ctx->cmd), true);
ionic_q_post(q, true, ionic_adminq_cb, ctx);
ionic_q_post(q, true);
err_out:
spin_unlock_irqrestore(&lif->adminq_lock, irqflags);
......
......@@ -14,7 +14,7 @@ int ionic_tx_napi(struct napi_struct *napi, int budget);
int ionic_txrx_napi(struct napi_struct *napi, int budget);
netdev_tx_t ionic_start_xmit(struct sk_buff *skb, struct net_device *netdev);
bool ionic_rx_service(struct ionic_cq *cq, struct ionic_cq_info *cq_info);
bool ionic_rx_service(struct ionic_cq *cq);
int ionic_xdp_xmit(struct net_device *netdev, int n, struct xdp_frame **xdp, u32 flags);
#endif /* _IONIC_TXRX_H_ */
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment