Commit 1f41dc5e authored by David S. Miller's avatar David S. Miller

Merge branch 'ena-next'

Arthur Kiyanovski says:

====================
Improving performance and reducing latencies, by using latest capabilities exposed in ENA device

This patchset introduces the following:
1. A new placement policy of Tx headers and descriptors, which takes
advantage of an option to place headers + descriptors in device memory
space. This is sometimes referred to as LLQ - low latency queue.
The patch set defines the admin capability, maps the device memory as
write-combined, and adds a mode in transmit datapath to do header +
descriptor placement on the device.
2. Support for RX checksum offloading
3. Miscelaneous small improvements and code cleanups

Note: V1 of this patchset was created as if patches e2a322a0 248ab773
from net were applied to net-next before applying the patchset. This V2
version does not assume this, and should be applyed directly on net-next
without the aformentioned patches.
====================
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parents 7b68b836 be26667c
This diff is collapsed.
......@@ -37,6 +37,7 @@
#include <linux/delay.h>
#include <linux/dma-mapping.h>
#include <linux/gfp.h>
#include <linux/io.h>
#include <linux/sched.h>
#include <linux/sizes.h>
#include <linux/spinlock.h>
......@@ -108,6 +109,14 @@ enum ena_intr_moder_level {
ENA_INTR_MAX_NUM_OF_LEVELS,
};
struct ena_llq_configurations {
enum ena_admin_llq_header_location llq_header_location;
enum ena_admin_llq_ring_entry_size llq_ring_entry_size;
enum ena_admin_llq_stride_ctrl llq_stride_ctrl;
enum ena_admin_llq_num_descs_before_header llq_num_decs_before_header;
u16 llq_ring_entry_size_value;
};
struct ena_intr_moder_entry {
unsigned int intr_moder_interval;
unsigned int pkts_per_interval;
......@@ -142,6 +151,15 @@ struct ena_com_tx_meta {
u16 l4_hdr_len; /* In words */
};
struct ena_com_llq_info {
u16 header_location_ctrl;
u16 desc_stride_ctrl;
u16 desc_list_entry_size_ctrl;
u16 desc_list_entry_size;
u16 descs_num_before_header;
u16 descs_per_entry;
};
struct ena_com_io_cq {
struct ena_com_io_desc_addr cdesc_addr;
......@@ -179,6 +197,20 @@ struct ena_com_io_cq {
} ____cacheline_aligned;
struct ena_com_io_bounce_buffer_control {
u8 *base_buffer;
u16 next_to_use;
u16 buffer_size;
u16 buffers_num; /* Must be a power of 2 */
};
/* This struct is to keep tracking the current location of the next llq entry */
struct ena_com_llq_pkt_ctrl {
u8 *curr_bounce_buf;
u16 idx;
u16 descs_left_in_line;
};
struct ena_com_io_sq {
struct ena_com_io_desc_addr desc_addr;
......@@ -190,6 +222,9 @@ struct ena_com_io_sq {
u32 msix_vector;
struct ena_com_tx_meta cached_tx_meta;
struct ena_com_llq_info llq_info;
struct ena_com_llq_pkt_ctrl llq_buf_ctrl;
struct ena_com_io_bounce_buffer_control bounce_buf_ctrl;
u16 q_depth;
u16 qid;
......@@ -197,6 +232,7 @@ struct ena_com_io_sq {
u16 idx;
u16 tail;
u16 next_to_comp;
u16 llq_last_copy_tail;
u32 tx_max_header_size;
u8 phase;
u8 desc_entry_size;
......@@ -334,6 +370,8 @@ struct ena_com_dev {
u16 intr_delay_resolution;
u32 intr_moder_tx_interval;
struct ena_intr_moder_entry *intr_moder_tbl;
struct ena_com_llq_info llq_info;
};
struct ena_com_dev_get_features_ctx {
......@@ -342,6 +380,7 @@ struct ena_com_dev_get_features_ctx {
struct ena_admin_feature_aenq_desc aenq;
struct ena_admin_feature_offload_desc offload;
struct ena_admin_ena_hw_hints hw_hints;
struct ena_admin_feature_llq_desc llq;
};
struct ena_com_create_io_ctx {
......@@ -397,8 +436,6 @@ void ena_com_mmio_reg_read_request_destroy(struct ena_com_dev *ena_dev);
/* ena_com_admin_init - Init the admin and the async queues
* @ena_dev: ENA communication layer struct
* @aenq_handlers: Those handlers to be called upon event.
* @init_spinlock: Indicate if this method should init the admin spinlock or
* the spinlock was init before (for example, in a case of FLR).
*
* Initialize the admin submission and completion queues.
* Initialize the asynchronous events notification queues.
......@@ -406,8 +443,7 @@ void ena_com_mmio_reg_read_request_destroy(struct ena_com_dev *ena_dev);
* @return - 0 on success, negative value on failure.
*/
int ena_com_admin_init(struct ena_com_dev *ena_dev,
struct ena_aenq_handlers *aenq_handlers,
bool init_spinlock);
struct ena_aenq_handlers *aenq_handlers);
/* ena_com_admin_destroy - Destroy the admin and the async events queues.
* @ena_dev: ENA communication layer struct
......@@ -935,6 +971,16 @@ void ena_com_get_intr_moderation_entry(struct ena_com_dev *ena_dev,
enum ena_intr_moder_level level,
struct ena_intr_moder_entry *entry);
/* ena_com_config_dev_mode - Configure the placement policy of the device.
* @ena_dev: ENA communication layer struct
* @llq_features: LLQ feature descriptor, retrieve via
* ena_com_get_dev_attr_feat.
* @ena_llq_config: The default driver LLQ parameters configurations
*/
int ena_com_config_dev_mode(struct ena_com_dev *ena_dev,
struct ena_admin_feature_llq_desc *llq_features,
struct ena_llq_configurations *llq_default_config);
static inline bool ena_com_get_adaptive_moderation_enabled(struct ena_com_dev *ena_dev)
{
return ena_dev->adaptive_coalescing;
......@@ -1044,4 +1090,21 @@ static inline void ena_com_update_intr_reg(struct ena_eth_io_intr_reg *intr_reg,
intr_reg->intr_control |= ENA_ETH_IO_INTR_REG_INTR_UNMASK_MASK;
}
static inline u8 *ena_com_get_next_bounce_buffer(struct ena_com_io_bounce_buffer_control *bounce_buf_ctrl)
{
u16 size, buffers_num;
u8 *buf;
size = bounce_buf_ctrl->buffer_size;
buffers_num = bounce_buf_ctrl->buffers_num;
buf = bounce_buf_ctrl->base_buffer +
(bounce_buf_ctrl->next_to_use++ & (buffers_num - 1)) * size;
prefetchw(bounce_buf_ctrl->base_buffer +
(bounce_buf_ctrl->next_to_use & (buffers_num - 1)) * size);
return buf;
}
#endif /* !(ENA_COM) */
......@@ -32,8 +32,8 @@
#ifndef _ENA_COMMON_H_
#define _ENA_COMMON_H_
#define ENA_COMMON_SPEC_VERSION_MAJOR 0 /* */
#define ENA_COMMON_SPEC_VERSION_MINOR 10 /* */
#define ENA_COMMON_SPEC_VERSION_MAJOR 2
#define ENA_COMMON_SPEC_VERSION_MINOR 0
/* ENA operates with 48-bit memory addresses. ena_mem_addr_t */
struct ena_common_mem_addr {
......
......@@ -67,6 +67,7 @@ struct ena_com_rx_ctx {
enum ena_eth_io_l4_proto_index l4_proto;
bool l3_csum_err;
bool l4_csum_err;
u8 l4_csum_checked;
/* fragmented packet */
bool frag;
u32 hash;
......@@ -86,8 +87,6 @@ int ena_com_add_single_rx_desc(struct ena_com_io_sq *io_sq,
struct ena_com_buf *ena_buf,
u16 req_id);
int ena_com_tx_comp_req_id_get(struct ena_com_io_cq *io_cq, u16 *req_id);
bool ena_com_cq_empty(struct ena_com_io_cq *io_cq);
static inline void ena_com_unmask_intr(struct ena_com_io_cq *io_cq,
......@@ -96,7 +95,7 @@ static inline void ena_com_unmask_intr(struct ena_com_io_cq *io_cq,
writel(intr_reg->intr_control, io_cq->unmask_reg);
}
static inline int ena_com_sq_empty_space(struct ena_com_io_sq *io_sq)
static inline int ena_com_free_desc(struct ena_com_io_sq *io_sq)
{
u16 tail, next_to_comp, cnt;
......@@ -107,11 +106,28 @@ static inline int ena_com_sq_empty_space(struct ena_com_io_sq *io_sq)
return io_sq->q_depth - 1 - cnt;
}
static inline int ena_com_write_sq_doorbell(struct ena_com_io_sq *io_sq)
/* Check if the submission queue has enough space to hold required_buffers */
static inline bool ena_com_sq_have_enough_space(struct ena_com_io_sq *io_sq,
u16 required_buffers)
{
u16 tail;
int temp;
tail = io_sq->tail;
if (io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_HOST)
return ena_com_free_desc(io_sq) >= required_buffers;
/* This calculation doesn't need to be 100% accurate. So to reduce
* the calculation overhead just Subtract 2 lines from the free descs
* (one for the header line and one to compensate the devision
* down calculation.
*/
temp = required_buffers / io_sq->llq_info.descs_per_entry + 2;
return ena_com_free_desc(io_sq) > temp;
}
static inline int ena_com_write_sq_doorbell(struct ena_com_io_sq *io_sq)
{
u16 tail = io_sq->tail;
pr_debug("write submission queue doorbell for queue: %d tail: %d\n",
io_sq->qid, tail);
......@@ -159,4 +175,48 @@ static inline void ena_com_comp_ack(struct ena_com_io_sq *io_sq, u16 elem)
io_sq->next_to_comp += elem;
}
static inline void ena_com_cq_inc_head(struct ena_com_io_cq *io_cq)
{
io_cq->head++;
/* Switch phase bit in case of wrap around */
if (unlikely((io_cq->head & (io_cq->q_depth - 1)) == 0))
io_cq->phase ^= 1;
}
static inline int ena_com_tx_comp_req_id_get(struct ena_com_io_cq *io_cq,
u16 *req_id)
{
u8 expected_phase, cdesc_phase;
struct ena_eth_io_tx_cdesc *cdesc;
u16 masked_head;
masked_head = io_cq->head & (io_cq->q_depth - 1);
expected_phase = io_cq->phase;
cdesc = (struct ena_eth_io_tx_cdesc *)
((uintptr_t)io_cq->cdesc_addr.virt_addr +
(masked_head * io_cq->cdesc_entry_size_in_bytes));
/* When the current completion descriptor phase isn't the same as the
* expected, it mean that the device still didn't update
* this completion.
*/
cdesc_phase = READ_ONCE(cdesc->flags) & ENA_ETH_IO_TX_CDESC_PHASE_MASK;
if (cdesc_phase != expected_phase)
return -EAGAIN;
dma_rmb();
*req_id = READ_ONCE(cdesc->req_id);
if (unlikely(*req_id >= io_cq->q_depth)) {
pr_err("Invalid req id %d\n", cdesc->req_id);
return -EINVAL;
}
ena_com_cq_inc_head(io_cq);
return 0;
}
#endif /* ENA_ETH_COM_H_ */
......@@ -34,23 +34,16 @@
enum ena_eth_io_l3_proto_index {
ENA_ETH_IO_L3_PROTO_UNKNOWN = 0,
ENA_ETH_IO_L3_PROTO_IPV4 = 8,
ENA_ETH_IO_L3_PROTO_IPV6 = 11,
ENA_ETH_IO_L3_PROTO_FCOE = 21,
ENA_ETH_IO_L3_PROTO_ROCE = 22,
};
enum ena_eth_io_l4_proto_index {
ENA_ETH_IO_L4_PROTO_UNKNOWN = 0,
ENA_ETH_IO_L4_PROTO_TCP = 12,
ENA_ETH_IO_L4_PROTO_UDP = 13,
ENA_ETH_IO_L4_PROTO_ROUTEABLE_ROCE = 23,
};
......@@ -242,9 +235,13 @@ struct ena_eth_io_rx_cdesc_base {
* checksum error detected, or, the controller didn't
* validate the checksum. This bit is valid only when
* l4_proto_idx indicates TCP/UDP packet, and,
* ipv4_frag is not set
* ipv4_frag is not set. This bit is valid only when
* l4_csum_checked below is set.
* 15 : ipv4_frag - Indicates IPv4 fragmented packet
* 23:16 : reserved16
* 16 : l4_csum_checked - L4 checksum was verified
* (could be OK or error), when cleared the status of
* checksum is unknown
* 23:17 : reserved17 - MBZ
* 24 : phase
* 25 : l3_csum2 - second checksum engine result
* 26 : first - Indicates first descriptor in
......@@ -390,6 +387,8 @@ struct ena_eth_io_numa_node_cfg_reg {
#define ENA_ETH_IO_RX_CDESC_BASE_L4_CSUM_ERR_MASK BIT(14)
#define ENA_ETH_IO_RX_CDESC_BASE_IPV4_FRAG_SHIFT 15
#define ENA_ETH_IO_RX_CDESC_BASE_IPV4_FRAG_MASK BIT(15)
#define ENA_ETH_IO_RX_CDESC_BASE_L4_CSUM_CHECKED_SHIFT 16
#define ENA_ETH_IO_RX_CDESC_BASE_L4_CSUM_CHECKED_MASK BIT(16)
#define ENA_ETH_IO_RX_CDESC_BASE_PHASE_SHIFT 24
#define ENA_ETH_IO_RX_CDESC_BASE_PHASE_MASK BIT(24)
#define ENA_ETH_IO_RX_CDESC_BASE_L3_CSUM2_SHIFT 25
......
......@@ -81,6 +81,7 @@ static const struct ena_stats ena_stats_tx_strings[] = {
ENA_STAT_TX_ENTRY(doorbells),
ENA_STAT_TX_ENTRY(prepare_ctx_err),
ENA_STAT_TX_ENTRY(bad_req_id),
ENA_STAT_TX_ENTRY(llq_buffer_copy),
ENA_STAT_TX_ENTRY(missed_tx),
};
......@@ -96,6 +97,7 @@ static const struct ena_stats ena_stats_rx_strings[] = {
ENA_STAT_RX_ENTRY(rx_copybreak_pkt),
ENA_STAT_RX_ENTRY(bad_req_id),
ENA_STAT_RX_ENTRY(empty_rx_ring),
ENA_STAT_RX_ENTRY(csum_unchecked),
};
static const struct ena_stats ena_stats_ena_com_strings[] = {
......
......@@ -43,9 +43,9 @@
#include "ena_com.h"
#include "ena_eth_com.h"
#define DRV_MODULE_VER_MAJOR 1
#define DRV_MODULE_VER_MINOR 5
#define DRV_MODULE_VER_SUBMINOR 0
#define DRV_MODULE_VER_MAJOR 2
#define DRV_MODULE_VER_MINOR 0
#define DRV_MODULE_VER_SUBMINOR 1
#define DRV_MODULE_NAME "ena"
#ifndef DRV_MODULE_VERSION
......@@ -61,6 +61,17 @@
#define ENA_ADMIN_MSIX_VEC 1
#define ENA_MAX_MSIX_VEC(io_queues) (ENA_ADMIN_MSIX_VEC + (io_queues))
/* The ENA buffer length fields is 16 bit long. So when PAGE_SIZE == 64kB the
* driver passes 0.
* Since the max packet size the ENA handles is ~9kB limit the buffer length to
* 16kB.
*/
#if PAGE_SIZE > SZ_16K
#define ENA_PAGE_SIZE SZ_16K
#else
#define ENA_PAGE_SIZE PAGE_SIZE
#endif
#define ENA_MIN_MSIX_VEC 2
#define ENA_REG_BAR 0
......@@ -70,7 +81,7 @@
#define ENA_DEFAULT_RING_SIZE (1024)
#define ENA_TX_WAKEUP_THRESH (MAX_SKB_FRAGS + 2)
#define ENA_DEFAULT_RX_COPYBREAK (128 - NET_IP_ALIGN)
#define ENA_DEFAULT_RX_COPYBREAK (256 - NET_IP_ALIGN)
/* limit the buffer size to 600 bytes to handle MTU changes from very
* small to very large, in which case the number of buffers per packet
......@@ -95,10 +106,11 @@
*/
#define ENA_TX_POLL_BUDGET_DIVIDER 4
/* Refill Rx queue when number of available descriptors is below
* QUEUE_SIZE / ENA_RX_REFILL_THRESH_DIVIDER
/* Refill Rx queue when number of required descriptors is above
* QUEUE_SIZE / ENA_RX_REFILL_THRESH_DIVIDER or ENA_RX_REFILL_THRESH_PACKET
*/
#define ENA_RX_REFILL_THRESH_DIVIDER 8
#define ENA_RX_REFILL_THRESH_PACKET 256
/* Number of queues to check for missing queues per timer service */
#define ENA_MONITORED_TX_QUEUES 4
......@@ -151,6 +163,9 @@ struct ena_tx_buffer {
/* num of buffers used by this skb */
u32 num_of_bufs;
/* Indicate if bufs[0] map the linear data of the skb. */
u8 map_linear_data;
/* Used for detect missing tx packets to limit the number of prints */
u32 print_once;
/* Save the last jiffies to detect missing tx packets
......@@ -186,6 +201,7 @@ struct ena_stats_tx {
u64 tx_poll;
u64 doorbells;
u64 bad_req_id;
u64 llq_buffer_copy;
u64 missed_tx;
};
......@@ -201,6 +217,7 @@ struct ena_stats_rx {
u64 rx_copybreak_pkt;
u64 bad_req_id;
u64 empty_rx_ring;
u64 csum_unchecked;
};
struct ena_ring {
......@@ -257,6 +274,8 @@ struct ena_ring {
struct ena_stats_tx tx_stats;
struct ena_stats_rx rx_stats;
};
u8 *push_buf_intermediate_buf;
int empty_rx_queue;
} ____cacheline_aligned;
......@@ -355,15 +374,4 @@ void ena_dump_stats_to_buf(struct ena_adapter *adapter, u8 *buf);
int ena_get_sset_count(struct net_device *netdev, int sset);
/* The ENA buffer length fields is 16 bit long. So when PAGE_SIZE == 64kB the
* driver passas 0.
* Since the max packet size the ENA handles is ~9kB limit the buffer length to
* 16kB.
*/
#if PAGE_SIZE > SZ_16K
#define ENA_PAGE_SIZE SZ_16K
#else
#define ENA_PAGE_SIZE PAGE_SIZE
#endif
#endif /* !(ENA_H) */
......@@ -34,37 +34,25 @@
enum ena_regs_reset_reason_types {
ENA_REGS_RESET_NORMAL = 0,
ENA_REGS_RESET_KEEP_ALIVE_TO = 1,
ENA_REGS_RESET_ADMIN_TO = 2,
ENA_REGS_RESET_MISS_TX_CMPL = 3,
ENA_REGS_RESET_INV_RX_REQ_ID = 4,
ENA_REGS_RESET_INV_TX_REQ_ID = 5,
ENA_REGS_RESET_TOO_MANY_RX_DESCS = 6,
ENA_REGS_RESET_INIT_ERR = 7,
ENA_REGS_RESET_DRIVER_INVALID_STATE = 8,
ENA_REGS_RESET_OS_TRIGGER = 9,
ENA_REGS_RESET_OS_NETDEV_WD = 10,
ENA_REGS_RESET_SHUTDOWN = 11,
ENA_REGS_RESET_USER_TRIGGER = 12,
ENA_REGS_RESET_GENERIC = 13,
ENA_REGS_RESET_MISS_INTERRUPT = 14,
};
/* ena_registers offsets */
/* 0 base */
#define ENA_REGS_VERSION_OFF 0x0
#define ENA_REGS_CONTROLLER_VERSION_OFF 0x4
#define ENA_REGS_CAPS_OFF 0x8
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment