Skip to content

L
linux
Commit 48ae409a authored by David S. Miller's avatar David S. Miller
Browse files
Options

Merge branch 'gve-desc' 

Rushil Gupta says:

====================
gve: Add QPL mode for DQO descriptor format

GVE supports QPL ("queue-page-list") mode where
all data is communicated through a set of pre-registered
pages. Adding this mode to DQO.
====================
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parents 16fd7539 5a3f8d12
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 649 additions and 121 deletions
Documentation/networking/device_drivers/ethernet/google/gve.rst
View file @ 48ae409a
...@@ -52,6 +52,15 @@ Descriptor Formats ...@@ -52,6 +52,15 @@ Descriptor Formats
GVE supports two descriptor formats: GQI and DQO. These two formats have GVE supports two descriptor formats: GQI and DQO. These two formats have
entirely different descriptors, which will be described below. entirely different descriptors, which will be described below.
Addressing Mode
------------------
GVE supports two addressing modes: QPL and RDA.
QPL ("queue-page-list") mode communicates data through a set of
pre-registered pages.
For RDA ("raw DMA addressing") mode, the set of pages is dynamic.
Therefore, the packet buffers can be anywhere in guest memory.
Registers Registers
--------- ---------
All registers are MMIO. All registers are MMIO.
......
drivers/net/ethernet/google/gve/gve.h
View file @ 48ae409a
...@@ -52,6 +52,26 @@ ...@@ -52,6 +52,26 @@
#define GVE_GQ_TX_MIN_PKT_DESC_BYTES 182 #define GVE_GQ_TX_MIN_PKT_DESC_BYTES 182
#define DQO_QPL_DEFAULT_TX_PAGES 512
#define DQO_QPL_DEFAULT_RX_PAGES 2048
/* Maximum TSO size supported on DQO */
#define GVE_DQO_TX_MAX 0x3FFFF
#define GVE_TX_BUF_SHIFT_DQO 11
/* 2K buffers for DQO-QPL */
#define GVE_TX_BUF_SIZE_DQO BIT(GVE_TX_BUF_SHIFT_DQO)
#define GVE_TX_BUFS_PER_PAGE_DQO (PAGE_SIZE >> GVE_TX_BUF_SHIFT_DQO)
#define GVE_MAX_TX_BUFS_PER_PKT (DIV_ROUND_UP(GVE_DQO_TX_MAX, GVE_TX_BUF_SIZE_DQO))
/* If number of free/recyclable buffers are less than this threshold; driver
* allocs and uses a non-qpl page on the receive path of DQO QPL to free
* up buffers.
* Value is set big enough to post at least 3 64K LRO packet via 2K buffer to NIC.
*/
#define GVE_DQO_QPL_ONDEMAND_ALLOC_THRESHOLD 96
/* Each slot in the desc ring has a 1:1 mapping to a slot in the data ring */ /* Each slot in the desc ring has a 1:1 mapping to a slot in the data ring */
struct gve_rx_desc_queue { struct gve_rx_desc_queue {
struct gve_rx_desc *desc_ring; /* the descriptor ring */ struct gve_rx_desc *desc_ring; /* the descriptor ring */
...@@ -218,6 +238,15 @@ struct gve_rx_ring { ...@@ -218,6 +238,15 @@ struct gve_rx_ring {
* which cannot be reused yet. * which cannot be reused yet.
*/ */
struct gve_index_list used_buf_states; struct gve_index_list used_buf_states;
/* qpl assigned to this queue */
struct gve_queue_page_list *qpl;
/* index into queue page list */
u32 next_qpl_page_idx;
/* track number of used buffers */
u16 used_buf_states_cnt;
} dqo; } dqo;
}; };
...@@ -329,8 +358,14 @@ struct gve_tx_pending_packet_dqo { ...@@ -329,8 +358,14 @@ struct gve_tx_pending_packet_dqo {
* All others correspond to `skb`'s frags and should be unmapped with * All others correspond to `skb`'s frags and should be unmapped with
* `dma_unmap_page`. * `dma_unmap_page`.
*/ */
DEFINE_DMA_UNMAP_ADDR(dma[MAX_SKB_FRAGS + 1]); union {
DEFINE_DMA_UNMAP_LEN(len[MAX_SKB_FRAGS + 1]); struct {
DEFINE_DMA_UNMAP_ADDR(dma[MAX_SKB_FRAGS + 1]);
DEFINE_DMA_UNMAP_LEN(len[MAX_SKB_FRAGS + 1]);
};
s16 tx_qpl_buf_ids[GVE_MAX_TX_BUFS_PER_PKT];
};
u16 num_bufs; u16 num_bufs;
/* Linked list index to next element in the list, or -1 if none */ /* Linked list index to next element in the list, or -1 if none */
...@@ -385,6 +420,32 @@ struct gve_tx_ring { ...@@ -385,6 +420,32 @@ struct gve_tx_ring {
* set. * set.
*/ */
u32 last_re_idx; u32 last_re_idx;
/* free running number of packet buf descriptors posted */
u16 posted_packet_desc_cnt;
/* free running number of packet buf descriptors completed */
u16 completed_packet_desc_cnt;
/* QPL fields */
struct {
/* Linked list of gve_tx_buf_dqo. Index into
* tx_qpl_buf_next, or -1 if empty.
*
* This is a consumer list owned by the TX path. When it
* runs out, the producer list is stolen from the
* completion handling path
* (dqo_compl.free_tx_qpl_buf_head).
*/
s16 free_tx_qpl_buf_head;
/* Free running count of the number of QPL tx buffers
* allocated
*/
u32 alloc_tx_qpl_buf_cnt;
/* Cached value of `dqo_compl.free_tx_qpl_buf_cnt` */
u32 free_tx_qpl_buf_cnt;
};
} dqo_tx; } dqo_tx;
}; };
...@@ -428,6 +489,24 @@ struct gve_tx_ring { ...@@ -428,6 +489,24 @@ struct gve_tx_ring {
* reached a specified timeout. * reached a specified timeout.
*/ */
struct gve_index_list timed_out_completions; struct gve_index_list timed_out_completions;
/* QPL fields */
struct {
/* Linked list of gve_tx_buf_dqo. Index into
* tx_qpl_buf_next, or -1 if empty.
*
* This is the producer list, owned by the completion
* handling path. When the consumer list
* (dqo_tx.free_tx_qpl_buf_head) is runs out, this list
* will be stolen.
*/
atomic_t free_tx_qpl_buf_head;
/* Free running count of the number of tx buffers
* freed
*/
atomic_t free_tx_qpl_buf_cnt;
};
} dqo_compl; } dqo_compl;
} ____cacheline_aligned; } ____cacheline_aligned;
u64 pkt_done; /* free-running - total packets completed */ u64 pkt_done; /* free-running - total packets completed */
...@@ -454,6 +533,21 @@ struct gve_tx_ring { ...@@ -454,6 +533,21 @@ struct gve_tx_ring {
s16 num_pending_packets; s16 num_pending_packets;
u32 complq_mask; /* complq size is complq_mask + 1 */ u32 complq_mask; /* complq size is complq_mask + 1 */
/* QPL fields */
struct {
/* qpl assigned to this queue */
struct gve_queue_page_list *qpl;
/* Each QPL page is divided into TX bounce buffers
* of size GVE_TX_BUF_SIZE_DQO. tx_qpl_buf_next is
* an array to manage linked lists of TX buffers.
* An entry j at index i implies that j'th buffer
* is next on the list after i
*/
s16 *tx_qpl_buf_next;
u32 num_tx_qpl_bufs;
};
} dqo; } dqo;
} ____cacheline_aligned; } ____cacheline_aligned;
struct netdev_queue *netdev_txq; struct netdev_queue *netdev_txq;
...@@ -532,6 +626,7 @@ enum gve_queue_format { ...@@ -532,6 +626,7 @@ enum gve_queue_format {
GVE_GQI_RDA_FORMAT = 0x1, GVE_GQI_RDA_FORMAT = 0x1,
GVE_GQI_QPL_FORMAT = 0x2, GVE_GQI_QPL_FORMAT = 0x2,
GVE_DQO_RDA_FORMAT = 0x3, GVE_DQO_RDA_FORMAT = 0x3,
GVE_DQO_QPL_FORMAT = 0x4,
}; };
struct gve_priv { struct gve_priv {
...@@ -551,7 +646,8 @@ struct gve_priv { ...@@ -551,7 +646,8 @@ struct gve_priv {
u16 num_event_counters; u16 num_event_counters;
u16 tx_desc_cnt; /* num desc per ring */ u16 tx_desc_cnt; /* num desc per ring */
u16 rx_desc_cnt; /* num desc per ring */ u16 rx_desc_cnt; /* num desc per ring */
u16 tx_pages_per_qpl; /* tx buffer length */ u16 tx_pages_per_qpl; /* Suggested number of pages per qpl for TX queues by NIC */
u16 rx_pages_per_qpl; /* Suggested number of pages per qpl for RX queues by NIC */
u16 rx_data_slot_cnt; /* rx buffer length */ u16 rx_data_slot_cnt; /* rx buffer length */
u64 max_registered_pages; u64 max_registered_pages;
u64 num_registered_pages; /* num pages registered with NIC */ u64 num_registered_pages; /* num pages registered with NIC */
...@@ -809,11 +905,17 @@ static inline u32 gve_rx_idx_to_ntfy(struct gve_priv *priv, u32 queue_idx) ...@@ -809,11 +905,17 @@ static inline u32 gve_rx_idx_to_ntfy(struct gve_priv *priv, u32 queue_idx)
return (priv->num_ntfy_blks / 2) + queue_idx; return (priv->num_ntfy_blks / 2) + queue_idx;
} }
static inline bool gve_is_qpl(struct gve_priv *priv)
{
return priv->queue_format == GVE_GQI_QPL_FORMAT ||
priv->queue_format == GVE_DQO_QPL_FORMAT;
}
/* Returns the number of tx queue page lists /* Returns the number of tx queue page lists
*/ */
static inline u32 gve_num_tx_qpls(struct gve_priv *priv) static inline u32 gve_num_tx_qpls(struct gve_priv *priv)
{ {
if (priv->queue_format != GVE_GQI_QPL_FORMAT) if (!gve_is_qpl(priv))
return 0; return 0;
return priv->tx_cfg.num_queues + priv->num_xdp_queues; return priv->tx_cfg.num_queues + priv->num_xdp_queues;
...@@ -833,7 +935,7 @@ static inline u32 gve_num_xdp_qpls(struct gve_priv *priv) ...@@ -833,7 +935,7 @@ static inline u32 gve_num_xdp_qpls(struct gve_priv *priv)
*/ */
static inline u32 gve_num_rx_qpls(struct gve_priv *priv) static inline u32 gve_num_rx_qpls(struct gve_priv *priv)
{ {
if (priv->queue_format != GVE_GQI_QPL_FORMAT) if (!gve_is_qpl(priv))
return 0; return 0;
return priv->rx_cfg.num_queues; return priv->rx_cfg.num_queues;
......
drivers/net/ethernet/google/gve/gve_adminq.c
View file @ 48ae409a
...@@ -39,7 +39,8 @@ void gve_parse_device_option(struct gve_priv *priv, ...@@ -39,7 +39,8 @@ void gve_parse_device_option(struct gve_priv *priv,
struct gve_device_option_gqi_rda **dev_op_gqi_rda, struct gve_device_option_gqi_rda **dev_op_gqi_rda,
struct gve_device_option_gqi_qpl **dev_op_gqi_qpl, struct gve_device_option_gqi_qpl **dev_op_gqi_qpl,
struct gve_device_option_dqo_rda **dev_op_dqo_rda, struct gve_device_option_dqo_rda **dev_op_dqo_rda,
struct gve_device_option_jumbo_frames **dev_op_jumbo_frames) struct gve_device_option_jumbo_frames **dev_op_jumbo_frames,
struct gve_device_option_dqo_qpl **dev_op_dqo_qpl)
{ {
u32 req_feat_mask = be32_to_cpu(option->required_features_mask); u32 req_feat_mask = be32_to_cpu(option->required_features_mask);
u16 option_length = be16_to_cpu(option->option_length); u16 option_length = be16_to_cpu(option->option_length);
...@@ -112,6 +113,22 @@ void gve_parse_device_option(struct gve_priv *priv, ...@@ -112,6 +113,22 @@ void gve_parse_device_option(struct gve_priv *priv,
} }
*dev_op_dqo_rda = (void *)(option + 1); *dev_op_dqo_rda = (void *)(option + 1);
break; break;
case GVE_DEV_OPT_ID_DQO_QPL:
if (option_length < sizeof(**dev_op_dqo_qpl) ||
req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_DQO_QPL) {
dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT,
"DQO QPL", (int)sizeof(**dev_op_dqo_qpl),
GVE_DEV_OPT_REQ_FEAT_MASK_DQO_QPL,
option_length, req_feat_mask);
break;
}
if (option_length > sizeof(**dev_op_dqo_qpl)) {
dev_warn(&priv->pdev->dev,
GVE_DEVICE_OPTION_TOO_BIG_FMT, "DQO QPL");
}
*dev_op_dqo_qpl = (void *)(option + 1);
break;
case GVE_DEV_OPT_ID_JUMBO_FRAMES: case GVE_DEV_OPT_ID_JUMBO_FRAMES:
if (option_length < sizeof(**dev_op_jumbo_frames) || if (option_length < sizeof(**dev_op_jumbo_frames) ||
req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_JUMBO_FRAMES) { req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_JUMBO_FRAMES) {
...@@ -146,7 +163,8 @@ gve_process_device_options(struct gve_priv *priv, ...@@ -146,7 +163,8 @@ gve_process_device_options(struct gve_priv *priv,
struct gve_device_option_gqi_rda **dev_op_gqi_rda, struct gve_device_option_gqi_rda **dev_op_gqi_rda,
struct gve_device_option_gqi_qpl **dev_op_gqi_qpl, struct gve_device_option_gqi_qpl **dev_op_gqi_qpl,
struct gve_device_option_dqo_rda **dev_op_dqo_rda, struct gve_device_option_dqo_rda **dev_op_dqo_rda,
struct gve_device_option_jumbo_frames **dev_op_jumbo_frames) struct gve_device_option_jumbo_frames **dev_op_jumbo_frames,
struct gve_device_option_dqo_qpl **dev_op_dqo_qpl)
{ {
const int num_options = be16_to_cpu(descriptor->num_device_options); const int num_options = be16_to_cpu(descriptor->num_device_options);
struct gve_device_option *dev_opt; struct gve_device_option *dev_opt;
...@@ -166,7 +184,8 @@ gve_process_device_options(struct gve_priv *priv, ...@@ -166,7 +184,8 @@ gve_process_device_options(struct gve_priv *priv,
gve_parse_device_option(priv, descriptor, dev_opt, gve_parse_device_option(priv, descriptor, dev_opt,
dev_op_gqi_rda, dev_op_gqi_qpl, dev_op_gqi_rda, dev_op_gqi_qpl,
dev_op_dqo_rda, dev_op_jumbo_frames); dev_op_dqo_rda, dev_op_jumbo_frames,
dev_op_dqo_qpl);
dev_opt = next_opt; dev_opt = next_opt;
} }
...@@ -505,12 +524,24 @@ static int gve_adminq_create_tx_queue(struct gve_priv *priv, u32 queue_index) ...@@ -505,12 +524,24 @@ static int gve_adminq_create_tx_queue(struct gve_priv *priv, u32 queue_index)
cmd.create_tx_queue.queue_page_list_id = cpu_to_be32(qpl_id); cmd.create_tx_queue.queue_page_list_id = cpu_to_be32(qpl_id);
} else { } else {
u16 comp_ring_size;
u32 qpl_id = 0;
if (priv->queue_format == GVE_DQO_RDA_FORMAT) {
qpl_id = GVE_RAW_ADDRESSING_QPL_ID;
comp_ring_size =
priv->options_dqo_rda.tx_comp_ring_entries;
} else {
qpl_id = tx->dqo.qpl->id;
comp_ring_size = priv->tx_desc_cnt;
}
cmd.create_tx_queue.queue_page_list_id = cpu_to_be32(qpl_id);
cmd.create_tx_queue.tx_ring_size = cmd.create_tx_queue.tx_ring_size =
cpu_to_be16(priv->tx_desc_cnt); cpu_to_be16(priv->tx_desc_cnt);
cmd.create_tx_queue.tx_comp_ring_addr = cmd.create_tx_queue.tx_comp_ring_addr =
cpu_to_be64(tx->complq_bus_dqo); cpu_to_be64(tx->complq_bus_dqo);
cmd.create_tx_queue.tx_comp_ring_size = cmd.create_tx_queue.tx_comp_ring_size =
cpu_to_be16(priv->options_dqo_rda.tx_comp_ring_entries); cpu_to_be16(comp_ring_size);
} }
return gve_adminq_issue_cmd(priv, &cmd); return gve_adminq_issue_cmd(priv, &cmd);
...@@ -555,6 +586,18 @@ static int gve_adminq_create_rx_queue(struct gve_priv *priv, u32 queue_index) ...@@ -555,6 +586,18 @@ static int gve_adminq_create_rx_queue(struct gve_priv *priv, u32 queue_index)
cmd.create_rx_queue.queue_page_list_id = cpu_to_be32(qpl_id); cmd.create_rx_queue.queue_page_list_id = cpu_to_be32(qpl_id);
cmd.create_rx_queue.packet_buffer_size = cpu_to_be16(rx->packet_buffer_size); cmd.create_rx_queue.packet_buffer_size = cpu_to_be16(rx->packet_buffer_size);
} else { } else {
u16 rx_buff_ring_entries;
u32 qpl_id = 0;
if (priv->queue_format == GVE_DQO_RDA_FORMAT) {
qpl_id = GVE_RAW_ADDRESSING_QPL_ID;
rx_buff_ring_entries =
priv->options_dqo_rda.rx_buff_ring_entries;
} else {
qpl_id = rx->dqo.qpl->id;
rx_buff_ring_entries = priv->rx_desc_cnt;
}
cmd.create_rx_queue.queue_page_list_id = cpu_to_be32(qpl_id);
cmd.create_rx_queue.rx_ring_size = cmd.create_rx_queue.rx_ring_size =
cpu_to_be16(priv->rx_desc_cnt); cpu_to_be16(priv->rx_desc_cnt);
cmd.create_rx_queue.rx_desc_ring_addr = cmd.create_rx_queue.rx_desc_ring_addr =
...@@ -564,7 +607,7 @@ static int gve_adminq_create_rx_queue(struct gve_priv *priv, u32 queue_index) ...@@ -564,7 +607,7 @@ static int gve_adminq_create_rx_queue(struct gve_priv *priv, u32 queue_index)
cmd.create_rx_queue.packet_buffer_size = cmd.create_rx_queue.packet_buffer_size =
cpu_to_be16(priv->data_buffer_size_dqo); cpu_to_be16(priv->data_buffer_size_dqo);
cmd.create_rx_queue.rx_buff_ring_size = cmd.create_rx_queue.rx_buff_ring_size =
cpu_to_be16(priv->options_dqo_rda.rx_buff_ring_entries); cpu_to_be16(rx_buff_ring_entries);
cmd.create_rx_queue.enable_rsc = cmd.create_rx_queue.enable_rsc =
!!(priv->dev->features & NETIF_F_LRO); !!(priv->dev->features & NETIF_F_LRO);
} }
...@@ -675,9 +718,13 @@ gve_set_desc_cnt_dqo(struct gve_priv *priv, ...@@ -675,9 +718,13 @@ gve_set_desc_cnt_dqo(struct gve_priv *priv,
const struct gve_device_option_dqo_rda *dev_op_dqo_rda) const struct gve_device_option_dqo_rda *dev_op_dqo_rda)
{ {
priv->tx_desc_cnt = be16_to_cpu(descriptor->tx_queue_entries); priv->tx_desc_cnt = be16_to_cpu(descriptor->tx_queue_entries);
priv->rx_desc_cnt = be16_to_cpu(descriptor->rx_queue_entries);
if (priv->queue_format == GVE_DQO_QPL_FORMAT)
return 0;
priv->options_dqo_rda.tx_comp_ring_entries = priv->options_dqo_rda.tx_comp_ring_entries =
be16_to_cpu(dev_op_dqo_rda->tx_comp_ring_entries); be16_to_cpu(dev_op_dqo_rda->tx_comp_ring_entries);
priv->rx_desc_cnt = be16_to_cpu(descriptor->rx_queue_entries);
priv->options_dqo_rda.rx_buff_ring_entries = priv->options_dqo_rda.rx_buff_ring_entries =
be16_to_cpu(dev_op_dqo_rda->rx_buff_ring_entries); be16_to_cpu(dev_op_dqo_rda->rx_buff_ring_entries);
...@@ -687,7 +734,9 @@ gve_set_desc_cnt_dqo(struct gve_priv *priv, ...@@ -687,7 +734,9 @@ gve_set_desc_cnt_dqo(struct gve_priv *priv,
static void gve_enable_supported_features(struct gve_priv *priv, static void gve_enable_supported_features(struct gve_priv *priv,
u32 supported_features_mask, u32 supported_features_mask,
const struct gve_device_option_jumbo_frames const struct gve_device_option_jumbo_frames
*dev_op_jumbo_frames) *dev_op_jumbo_frames,
const struct gve_device_option_dqo_qpl
*dev_op_dqo_qpl)
{ {
/* Before control reaches this point, the page-size-capped max MTU from /* Before control reaches this point, the page-size-capped max MTU from
* the gve_device_descriptor field has already been stored in * the gve_device_descriptor field has already been stored in
...@@ -699,6 +748,18 @@ static void gve_enable_supported_features(struct gve_priv *priv, ...@@ -699,6 +748,18 @@ static void gve_enable_supported_features(struct gve_priv *priv,
"JUMBO FRAMES device option enabled.\n"); "JUMBO FRAMES device option enabled.\n");
priv->dev->max_mtu = be16_to_cpu(dev_op_jumbo_frames->max_mtu); priv->dev->max_mtu = be16_to_cpu(dev_op_jumbo_frames->max_mtu);
} }
/* Override pages for qpl for DQO-QPL */
if (dev_op_dqo_qpl) {
priv->tx_pages_per_qpl =
be16_to_cpu(dev_op_dqo_qpl->tx_pages_per_qpl);
priv->rx_pages_per_qpl =
be16_to_cpu(dev_op_dqo_qpl->rx_pages_per_qpl);
if (priv->tx_pages_per_qpl == 0)
priv->tx_pages_per_qpl = DQO_QPL_DEFAULT_TX_PAGES;
if (priv->rx_pages_per_qpl == 0)
priv->rx_pages_per_qpl = DQO_QPL_DEFAULT_RX_PAGES;
}
} }
int gve_adminq_describe_device(struct gve_priv *priv) int gve_adminq_describe_device(struct gve_priv *priv)
...@@ -707,6 +768,7 @@ int gve_adminq_describe_device(struct gve_priv *priv) ...@@ -707,6 +768,7 @@ int gve_adminq_describe_device(struct gve_priv *priv)
struct gve_device_option_gqi_rda *dev_op_gqi_rda = NULL; struct gve_device_option_gqi_rda *dev_op_gqi_rda = NULL;
struct gve_device_option_gqi_qpl *dev_op_gqi_qpl = NULL; struct gve_device_option_gqi_qpl *dev_op_gqi_qpl = NULL;
struct gve_device_option_dqo_rda *dev_op_dqo_rda = NULL; struct gve_device_option_dqo_rda *dev_op_dqo_rda = NULL;
struct gve_device_option_dqo_qpl *dev_op_dqo_qpl = NULL;
struct gve_device_descriptor *descriptor; struct gve_device_descriptor *descriptor;
u32 supported_features_mask = 0; u32 supported_features_mask = 0;
union gve_adminq_command cmd; union gve_adminq_command cmd;
...@@ -733,13 +795,14 @@ int gve_adminq_describe_device(struct gve_priv *priv) ...@@ -733,13 +795,14 @@ int gve_adminq_describe_device(struct gve_priv *priv)
err = gve_process_device_options(priv, descriptor, &dev_op_gqi_rda, err = gve_process_device_options(priv, descriptor, &dev_op_gqi_rda,
&dev_op_gqi_qpl, &dev_op_dqo_rda, &dev_op_gqi_qpl, &dev_op_dqo_rda,
&dev_op_jumbo_frames); &dev_op_jumbo_frames,
&dev_op_dqo_qpl);
if (err) if (err)
goto free_device_descriptor; goto free_device_descriptor;
/* If the GQI_RAW_ADDRESSING option is not enabled and the queue format /* If the GQI_RAW_ADDRESSING option is not enabled and the queue format
* is not set to GqiRda, choose the queue format in a priority order: * is not set to GqiRda, choose the queue format in a priority order:
* DqoRda, GqiRda, GqiQpl. Use GqiQpl as default. * DqoRda, DqoQpl, GqiRda, GqiQpl. Use GqiQpl as default.
*/ */
if (dev_op_dqo_rda) { if (dev_op_dqo_rda) {
priv->queue_format = GVE_DQO_RDA_FORMAT; priv->queue_format = GVE_DQO_RDA_FORMAT;
...@@ -747,7 +810,11 @@ int gve_adminq_describe_device(struct gve_priv *priv) ...@@ -747,7 +810,11 @@ int gve_adminq_describe_device(struct gve_priv *priv)
"Driver is running with DQO RDA queue format.\n"); "Driver is running with DQO RDA queue format.\n");
supported_features_mask = supported_features_mask =
be32_to_cpu(dev_op_dqo_rda->supported_features_mask); be32_to_cpu(dev_op_dqo_rda->supported_features_mask);
} else if (dev_op_gqi_rda) { } else if (dev_op_dqo_qpl) {
priv->queue_format = GVE_DQO_QPL_FORMAT;
supported_features_mask =
be32_to_cpu(dev_op_dqo_qpl->supported_features_mask);
} else if (dev_op_gqi_rda) {
priv->queue_format = GVE_GQI_RDA_FORMAT; priv->queue_format = GVE_GQI_RDA_FORMAT;
dev_info(&priv->pdev->dev, dev_info(&priv->pdev->dev,
"Driver is running with GQI RDA queue format.\n"); "Driver is running with GQI RDA queue format.\n");
...@@ -798,7 +865,7 @@ int gve_adminq_describe_device(struct gve_priv *priv) ...@@ -798,7 +865,7 @@ int gve_adminq_describe_device(struct gve_priv *priv)
priv->default_num_queues = be16_to_cpu(descriptor->default_num_queues); priv->default_num_queues = be16_to_cpu(descriptor->default_num_queues);
gve_enable_supported_features(priv, supported_features_mask, gve_enable_supported_features(priv, supported_features_mask,
dev_op_jumbo_frames); dev_op_jumbo_frames, dev_op_dqo_qpl);
free_device_descriptor: free_device_descriptor:
dma_free_coherent(&priv->pdev->dev, PAGE_SIZE, descriptor, dma_free_coherent(&priv->pdev->dev, PAGE_SIZE, descriptor,
......
drivers/net/ethernet/google/gve/gve_adminq.h
View file @ 48ae409a
...@@ -109,6 +109,14 @@ struct gve_device_option_dqo_rda { ...@@ -109,6 +109,14 @@ struct gve_device_option_dqo_rda {
static_assert(sizeof(struct gve_device_option_dqo_rda) == 8); static_assert(sizeof(struct gve_device_option_dqo_rda) == 8);
struct gve_device_option_dqo_qpl {
__be32 supported_features_mask;
__be16 tx_pages_per_qpl;
__be16 rx_pages_per_qpl;
};
static_assert(sizeof(struct gve_device_option_dqo_qpl) == 8);
struct gve_device_option_jumbo_frames { struct gve_device_option_jumbo_frames {
__be32 supported_features_mask; __be32 supported_features_mask;
__be16 max_mtu; __be16 max_mtu;
...@@ -130,6 +138,7 @@ enum gve_dev_opt_id { ...@@ -130,6 +138,7 @@ enum gve_dev_opt_id {
GVE_DEV_OPT_ID_GQI_RDA = 0x2, GVE_DEV_OPT_ID_GQI_RDA = 0x2,
GVE_DEV_OPT_ID_GQI_QPL = 0x3, GVE_DEV_OPT_ID_GQI_QPL = 0x3,
GVE_DEV_OPT_ID_DQO_RDA = 0x4, GVE_DEV_OPT_ID_DQO_RDA = 0x4,
GVE_DEV_OPT_ID_DQO_QPL = 0x7,
GVE_DEV_OPT_ID_JUMBO_FRAMES = 0x8, GVE_DEV_OPT_ID_JUMBO_FRAMES = 0x8,
}; };
...@@ -139,6 +148,7 @@ enum gve_dev_opt_req_feat_mask { ...@@ -139,6 +148,7 @@ enum gve_dev_opt_req_feat_mask {
GVE_DEV_OPT_REQ_FEAT_MASK_GQI_QPL = 0x0, GVE_DEV_OPT_REQ_FEAT_MASK_GQI_QPL = 0x0,
GVE_DEV_OPT_REQ_FEAT_MASK_DQO_RDA = 0x0, GVE_DEV_OPT_REQ_FEAT_MASK_DQO_RDA = 0x0,
GVE_DEV_OPT_REQ_FEAT_MASK_JUMBO_FRAMES = 0x0, GVE_DEV_OPT_REQ_FEAT_MASK_JUMBO_FRAMES = 0x0,
GVE_DEV_OPT_REQ_FEAT_MASK_DQO_QPL = 0x0,
}; };
enum gve_sup_feature_mask { enum gve_sup_feature_mask {
......
drivers/net/ethernet/google/gve/gve_main.c
View file @ 48ae409a
...@@ -31,7 +31,6 @@ ...@@ -31,7 +31,6 @@
// Minimum amount of time between queue kicks in msec (10 seconds) // Minimum amount of time between queue kicks in msec (10 seconds)
#define MIN_TX_TIMEOUT_GAP (1000 * 10) #define MIN_TX_TIMEOUT_GAP (1000 * 10)
#define DQO_TX_MAX 0x3FFFF
char gve_driver_name[] = "gve"; char gve_driver_name[] = "gve";
const char gve_version_str[] = GVE_VERSION; const char gve_version_str[] = GVE_VERSION;
...@@ -494,7 +493,7 @@ static int gve_setup_device_resources(struct gve_priv *priv) ...@@ -494,7 +493,7 @@ static int gve_setup_device_resources(struct gve_priv *priv)
goto abort_with_stats_report; goto abort_with_stats_report;
} }
if (priv->queue_format == GVE_DQO_RDA_FORMAT) { if (!gve_is_gqi(priv)) {
priv->ptype_lut_dqo = kvzalloc(sizeof(*priv->ptype_lut_dqo), priv->ptype_lut_dqo = kvzalloc(sizeof(*priv->ptype_lut_dqo),
GFP_KERNEL); GFP_KERNEL);
if (!priv->ptype_lut_dqo) { if (!priv->ptype_lut_dqo) {
...@@ -1083,11 +1082,12 @@ static int gve_alloc_xdp_qpls(struct gve_priv *priv) ...@@ -1083,11 +1082,12 @@ static int gve_alloc_xdp_qpls(struct gve_priv *priv)
static int gve_alloc_qpls(struct gve_priv *priv) static int gve_alloc_qpls(struct gve_priv *priv)
{ {
int max_queues = priv->tx_cfg.max_queues + priv->rx_cfg.max_queues; int max_queues = priv->tx_cfg.max_queues + priv->rx_cfg.max_queues;
int page_count;
int start_id; int start_id;
int i, j; int i, j;
int err; int err;
if (priv->queue_format != GVE_GQI_QPL_FORMAT) if (!gve_is_qpl(priv))
return 0; return 0;
priv->qpls = kvcalloc(max_queues, sizeof(*priv->qpls), GFP_KERNEL); priv->qpls = kvcalloc(max_queues, sizeof(*priv->qpls), GFP_KERNEL);
...@@ -1095,17 +1095,25 @@ static int gve_alloc_qpls(struct gve_priv *priv) ...@@ -1095,17 +1095,25 @@ static int gve_alloc_qpls(struct gve_priv *priv)
return -ENOMEM; return -ENOMEM;
start_id = gve_tx_start_qpl_id(priv); start_id = gve_tx_start_qpl_id(priv);
page_count = priv->tx_pages_per_qpl;
for (i = start_id; i < start_id + gve_num_tx_qpls(priv); i++) { for (i = start_id; i < start_id + gve_num_tx_qpls(priv); i++) {
err = gve_alloc_queue_page_list(priv, i, err = gve_alloc_queue_page_list(priv, i,
priv->tx_pages_per_qpl); page_count);
if (err) if (err)
goto free_qpls; goto free_qpls;
} }
start_id = gve_rx_start_qpl_id(priv); start_id = gve_rx_start_qpl_id(priv);
/* For GQI_QPL number of pages allocated have 1:1 relationship with
* number of descriptors. For DQO, number of pages required are
* more than descriptors (because of out of order completions).
*/
page_count = priv->queue_format == GVE_GQI_QPL_FORMAT ?
priv->rx_data_slot_cnt : priv->rx_pages_per_qpl;
for (i = start_id; i < start_id + gve_num_rx_qpls(priv); i++) { for (i = start_id; i < start_id + gve_num_rx_qpls(priv); i++) {
err = gve_alloc_queue_page_list(priv, i, err = gve_alloc_queue_page_list(priv, i,
priv->rx_data_slot_cnt); page_count);
if (err) if (err)
goto free_qpls; goto free_qpls;
} }
...@@ -2051,7 +2059,7 @@ static int gve_init_priv(struct gve_priv *priv, bool skip_describe_device) ...@@ -2051,7 +2059,7 @@ static int gve_init_priv(struct gve_priv *priv, bool skip_describe_device)
/* Big TCP is only supported on DQ*/ /* Big TCP is only supported on DQ*/
if (!gve_is_gqi(priv)) if (!gve_is_gqi(priv))
netif_set_tso_max_size(priv->dev, DQO_TX_MAX); netif_set_tso_max_size(priv->dev, GVE_DQO_TX_MAX);
priv->num_registered_pages = 0; priv->num_registered_pages = 0;
priv->rx_copybreak = GVE_DEFAULT_RX_COPYBREAK; priv->rx_copybreak = GVE_DEFAULT_RX_COPYBREAK;
......
drivers/net/ethernet/google/gve/gve_rx_dqo.c
View file @ 48ae409a
...@@ -22,11 +22,13 @@ static int gve_buf_ref_cnt(struct gve_rx_buf_state_dqo *bs) ...@@ -22,11 +22,13 @@ static int gve_buf_ref_cnt(struct gve_rx_buf_state_dqo *bs)
} }
static void gve_free_page_dqo(struct gve_priv *priv, static void gve_free_page_dqo(struct gve_priv *priv,
struct gve_rx_buf_state_dqo *bs) struct gve_rx_buf_state_dqo *bs,
bool free_page)
{ {
page_ref_sub(bs->page_info.page, bs->page_info.pagecnt_bias - 1); page_ref_sub(bs->page_info.page, bs->page_info.pagecnt_bias - 1);
gve_free_page(&priv->pdev->dev, bs->page_info.page, bs->addr, if (free_page)
DMA_FROM_DEVICE); gve_free_page(&priv->pdev->dev, bs->page_info.page, bs->addr,
DMA_FROM_DEVICE);
bs->page_info.page = NULL; bs->page_info.page = NULL;
} }
...@@ -130,12 +132,20 @@ gve_get_recycled_buf_state(struct gve_rx_ring *rx) ...@@ -130,12 +132,20 @@ gve_get_recycled_buf_state(struct gve_rx_ring *rx)
*/ */
for (i = 0; i < 5; i++) { for (i = 0; i < 5; i++) {
buf_state = gve_dequeue_buf_state(rx, &rx->dqo.used_buf_states); buf_state = gve_dequeue_buf_state(rx, &rx->dqo.used_buf_states);
if (gve_buf_ref_cnt(buf_state) == 0) if (gve_buf_ref_cnt(buf_state) == 0) {
rx->dqo.used_buf_states_cnt--;
return buf_state; return buf_state;
}
gve_enqueue_buf_state(rx, &rx->dqo.used_buf_states, buf_state); gve_enqueue_buf_state(rx, &rx->dqo.used_buf_states, buf_state);
} }
/* For QPL, we cannot allocate any new buffers and must
* wait for the existing ones to be available.
*/
if (rx->dqo.qpl)
return NULL;
/* If there are no free buf states discard an entry from /* If there are no free buf states discard an entry from
* `used_buf_states` so it can be used. * `used_buf_states` so it can be used.
*/ */
...@@ -144,23 +154,39 @@ gve_get_recycled_buf_state(struct gve_rx_ring *rx) ...@@ -144,23 +154,39 @@ gve_get_recycled_buf_state(struct gve_rx_ring *rx)
if (gve_buf_ref_cnt(buf_state) == 0) if (gve_buf_ref_cnt(buf_state) == 0)
return buf_state; return buf_state;
gve_free_page_dqo(rx->gve, buf_state); gve_free_page_dqo(rx->gve, buf_state, true);
gve_free_buf_state(rx, buf_state); gve_free_buf_state(rx, buf_state);
} }
return NULL; return NULL;
} }
static int gve_alloc_page_dqo(struct gve_priv *priv, static int gve_alloc_page_dqo(struct gve_rx_ring *rx,
struct gve_rx_buf_state_dqo *buf_state) struct gve_rx_buf_state_dqo *buf_state)
{ {
int err; struct gve_priv *priv = rx->gve;
u32 idx;
err = gve_alloc_page(priv, &priv->pdev->dev, &buf_state->page_info.page, if (!rx->dqo.qpl) {
&buf_state->addr, DMA_FROM_DEVICE, GFP_ATOMIC); int err;
if (err)
return err;
err = gve_alloc_page(priv, &priv->pdev->dev,
&buf_state->page_info.page,
&buf_state->addr,
DMA_FROM_DEVICE, GFP_ATOMIC);
if (err)
return err;
} else {
idx = rx->dqo.next_qpl_page_idx;
if (idx >= priv->rx_pages_per_qpl) {
net_err_ratelimited("%s: Out of QPL pages\n",
priv->dev->name);
return -ENOMEM;
}
buf_state->page_info.page = rx->dqo.qpl->pages[idx];
buf_state->addr = rx->dqo.qpl->page_buses[idx];
rx->dqo.next_qpl_page_idx++;
}
buf_state->page_info.page_offset = 0; buf_state->page_info.page_offset = 0;
buf_state->page_info.page_address = buf_state->page_info.page_address =
page_address(buf_state->page_info.page); page_address(buf_state->page_info.page);
...@@ -195,9 +221,13 @@ static void gve_rx_free_ring_dqo(struct gve_priv *priv, int idx) ...@@ -195,9 +221,13 @@ static void gve_rx_free_ring_dqo(struct gve_priv *priv, int idx)
for (i = 0; i < rx->dqo.num_buf_states; i++) { for (i = 0; i < rx->dqo.num_buf_states; i++) {
struct gve_rx_buf_state_dqo *bs = &rx->dqo.buf_states[i]; struct gve_rx_buf_state_dqo *bs = &rx->dqo.buf_states[i];
/* Only free page for RDA. QPL pages are freed in gve_main. */
if (bs->page_info.page) if (bs->page_info.page)
gve_free_page_dqo(priv, bs); gve_free_page_dqo(priv, bs, !rx->dqo.qpl);
}
if (rx->dqo.qpl) {
gve_unassign_qpl(priv, rx->dqo.qpl->id);
rx->dqo.qpl = NULL;
} }
if (rx->dqo.bufq.desc_ring) { if (rx->dqo.bufq.desc_ring) {
...@@ -229,7 +259,8 @@ static int gve_rx_alloc_ring_dqo(struct gve_priv *priv, int idx) ...@@ -229,7 +259,8 @@ static int gve_rx_alloc_ring_dqo(struct gve_priv *priv, int idx)
int i; int i;
const u32 buffer_queue_slots = const u32 buffer_queue_slots =
priv->options_dqo_rda.rx_buff_ring_entries; priv->queue_format == GVE_DQO_RDA_FORMAT ?
priv->options_dqo_rda.rx_buff_ring_entries : priv->rx_desc_cnt;
const u32 completion_queue_slots = priv->rx_desc_cnt; const u32 completion_queue_slots = priv->rx_desc_cnt;
netif_dbg(priv, drv, priv->dev, "allocating rx ring DQO\n"); netif_dbg(priv, drv, priv->dev, "allocating rx ring DQO\n");
...@@ -243,7 +274,9 @@ static int gve_rx_alloc_ring_dqo(struct gve_priv *priv, int idx) ...@@ -243,7 +274,9 @@ static int gve_rx_alloc_ring_dqo(struct gve_priv *priv, int idx)
rx->ctx.skb_head = NULL; rx->ctx.skb_head = NULL;
rx->ctx.skb_tail = NULL; rx->ctx.skb_tail = NULL;
rx->dqo.num_buf_states = min_t(s16, S16_MAX, buffer_queue_slots * 4); rx->dqo.num_buf_states = priv->queue_format == GVE_DQO_RDA_FORMAT ?
min_t(s16, S16_MAX, buffer_queue_slots * 4) :
priv->rx_pages_per_qpl;
rx->dqo.buf_states = kvcalloc(rx->dqo.num_buf_states, rx->dqo.buf_states = kvcalloc(rx->dqo.num_buf_states,
sizeof(rx->dqo.buf_states[0]), sizeof(rx->dqo.buf_states[0]),
GFP_KERNEL); GFP_KERNEL);
...@@ -275,6 +308,13 @@ static int gve_rx_alloc_ring_dqo(struct gve_priv *priv, int idx) ...@@ -275,6 +308,13 @@ static int gve_rx_alloc_ring_dqo(struct gve_priv *priv, int idx)
if (!rx->dqo.bufq.desc_ring) if (!rx->dqo.bufq.desc_ring)
goto err; goto err;
if (priv->queue_format != GVE_DQO_RDA_FORMAT) {
rx->dqo.qpl = gve_assign_rx_qpl(priv, rx->q_num);
if (!rx->dqo.qpl)
goto err;
rx->dqo.next_qpl_page_idx = 0;
}
rx->q_resources = dma_alloc_coherent(hdev, sizeof(*rx->q_resources), rx->q_resources = dma_alloc_coherent(hdev, sizeof(*rx->q_resources),
&rx->q_resources_bus, GFP_KERNEL); &rx->q_resources_bus, GFP_KERNEL);
if (!rx->q_resources) if (!rx->q_resources)
...@@ -352,7 +392,7 @@ void gve_rx_post_buffers_dqo(struct gve_rx_ring *rx) ...@@ -352,7 +392,7 @@ void gve_rx_post_buffers_dqo(struct gve_rx_ring *rx)
if (unlikely(!buf_state)) if (unlikely(!buf_state))
break; break;
if (unlikely(gve_alloc_page_dqo(priv, buf_state))) { if (unlikely(gve_alloc_page_dqo(rx, buf_state))) {
u64_stats_update_begin(&rx->statss); u64_stats_update_begin(&rx->statss);
rx->rx_buf_alloc_fail++; rx->rx_buf_alloc_fail++;
u64_stats_update_end(&rx->statss); u64_stats_update_end(&rx->statss);
...@@ -415,6 +455,7 @@ static void gve_try_recycle_buf(struct gve_priv *priv, struct gve_rx_ring *rx, ...@@ -415,6 +455,7 @@ static void gve_try_recycle_buf(struct gve_priv *priv, struct gve_rx_ring *rx,
mark_used: mark_used:
gve_enqueue_buf_state(rx, &rx->dqo.used_buf_states, buf_state); gve_enqueue_buf_state(rx, &rx->dqo.used_buf_states, buf_state);
rx->dqo.used_buf_states_cnt++;
} }
static void gve_rx_skb_csum(struct sk_buff *skb, static void gve_rx_skb_csum(struct sk_buff *skb,
...@@ -475,6 +516,43 @@ static void gve_rx_free_skb(struct gve_rx_ring *rx) ...@@ -475,6 +516,43 @@ static void gve_rx_free_skb(struct gve_rx_ring *rx)
rx->ctx.skb_tail = NULL; rx->ctx.skb_tail = NULL;
} }
static bool gve_rx_should_trigger_copy_ondemand(struct gve_rx_ring *rx)
{
if (!rx->dqo.qpl)
return false;
if (rx->dqo.used_buf_states_cnt <
(rx->dqo.num_buf_states -
GVE_DQO_QPL_ONDEMAND_ALLOC_THRESHOLD))
return false;
return true;
}
static int gve_rx_copy_ondemand(struct gve_rx_ring *rx,
struct gve_rx_buf_state_dqo *buf_state,
u16 buf_len)
{
struct page *page = alloc_page(GFP_ATOMIC);
int num_frags;
if (!page)
return -ENOMEM;
memcpy(page_address(page),
buf_state->page_info.page_address +
buf_state->page_info.page_offset,
buf_len);
num_frags = skb_shinfo(rx->ctx.skb_tail)->nr_frags;
skb_add_rx_frag(rx->ctx.skb_tail, num_frags, page,
0, buf_len, PAGE_SIZE);
u64_stats_update_begin(&rx->statss);
rx->rx_frag_alloc_cnt++;
u64_stats_update_end(&rx->statss);
/* Return unused buffer. */
gve_enqueue_buf_state(rx, &rx->dqo.recycled_buf_states, buf_state);
return 0;
}
/* Chains multi skbs for single rx packet. /* Chains multi skbs for single rx packet.
* Returns 0 if buffer is appended, -1 otherwise. * Returns 0 if buffer is appended, -1 otherwise.
*/ */
...@@ -502,12 +580,20 @@ static int gve_rx_append_frags(struct napi_struct *napi, ...@@ -502,12 +580,20 @@ static int gve_rx_append_frags(struct napi_struct *napi,
rx->ctx.skb_head->truesize += priv->data_buffer_size_dqo; rx->ctx.skb_head->truesize += priv->data_buffer_size_dqo;
} }
/* Trigger ondemand page allocation if we are running low on buffers */
if (gve_rx_should_trigger_copy_ondemand(rx))
return gve_rx_copy_ondemand(rx, buf_state, buf_len);
skb_add_rx_frag(rx->ctx.skb_tail, num_frags, skb_add_rx_frag(rx->ctx.skb_tail, num_frags,
buf_state->page_info.page, buf_state->page_info.page,
buf_state->page_info.page_offset, buf_state->page_info.page_offset,
buf_len, priv->data_buffer_size_dqo); buf_len, priv->data_buffer_size_dqo);
gve_dec_pagecnt_bias(&buf_state->page_info); gve_dec_pagecnt_bias(&buf_state->page_info);
/* Advances buffer page-offset if page is partially used.
* Marks buffer as used if page is full.
*/
gve_try_recycle_buf(priv, rx, buf_state);
return 0; return 0;
} }
...@@ -561,8 +647,6 @@ static int gve_rx_dqo(struct napi_struct *napi, struct gve_rx_ring *rx, ...@@ -561,8 +647,6 @@ static int gve_rx_dqo(struct napi_struct *napi, struct gve_rx_ring *rx,
priv)) != 0) { priv)) != 0) {
goto error; goto error;
} }
gve_try_recycle_buf(priv, rx, buf_state);
return 0; return 0;
} }
...@@ -588,6 +672,12 @@ static int gve_rx_dqo(struct napi_struct *napi, struct gve_rx_ring *rx, ...@@ -588,6 +672,12 @@ static int gve_rx_dqo(struct napi_struct *napi, struct gve_rx_ring *rx,
goto error; goto error;
rx->ctx.skb_tail = rx->ctx.skb_head; rx->ctx.skb_tail = rx->ctx.skb_head;
if (gve_rx_should_trigger_copy_ondemand(rx)) {
if (gve_rx_copy_ondemand(rx, buf_state, buf_len) < 0)
goto error;
return 0;
}
skb_add_rx_frag(rx->ctx.skb_head, 0, buf_state->page_info.page, skb_add_rx_frag(rx->ctx.skb_head, 0, buf_state->page_info.page,
buf_state->page_info.page_offset, buf_len, buf_state->page_info.page_offset, buf_len,
priv->data_buffer_size_dqo); priv->data_buffer_size_dqo);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7