Commit 224e92e0 authored by Brenden Blanco's avatar Brenden Blanco Committed by David S. Miller

net/mlx4_en: break out tx_desc write into separate function

In preparation for writing the tx descriptor from multiple functions,
create a helper for both normal and blueflame access.
Signed-off-by: default avatarBrenden Blanco <bblanco@plumgrid.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent 6ce96ca3
...@@ -232,7 +232,7 @@ static void stamp_send_wqe(struct mlx4_ib_qp *qp, int n, int size) ...@@ -232,7 +232,7 @@ static void stamp_send_wqe(struct mlx4_ib_qp *qp, int n, int size)
} }
} else { } else {
ctrl = buf = get_send_wqe(qp, n & (qp->sq.wqe_cnt - 1)); ctrl = buf = get_send_wqe(qp, n & (qp->sq.wqe_cnt - 1));
s = (ctrl->fence_size & 0x3f) << 4; s = (ctrl->qpn_vlan.fence_size & 0x3f) << 4;
for (i = 64; i < s; i += 64) { for (i = 64; i < s; i += 64) {
wqe = buf + i; wqe = buf + i;
*wqe = cpu_to_be32(0xffffffff); *wqe = cpu_to_be32(0xffffffff);
...@@ -264,7 +264,7 @@ static void post_nop_wqe(struct mlx4_ib_qp *qp, int n, int size) ...@@ -264,7 +264,7 @@ static void post_nop_wqe(struct mlx4_ib_qp *qp, int n, int size)
inl->byte_count = cpu_to_be32(1 << 31 | (size - s - sizeof *inl)); inl->byte_count = cpu_to_be32(1 << 31 | (size - s - sizeof *inl));
} }
ctrl->srcrb_flags = 0; ctrl->srcrb_flags = 0;
ctrl->fence_size = size / 16; ctrl->qpn_vlan.fence_size = size / 16;
/* /*
* Make sure descriptor is fully written before setting ownership bit * Make sure descriptor is fully written before setting ownership bit
* (because HW can start executing as soon as we do). * (because HW can start executing as soon as we do).
...@@ -1992,7 +1992,8 @@ static int __mlx4_ib_modify_qp(struct ib_qp *ibqp, ...@@ -1992,7 +1992,8 @@ static int __mlx4_ib_modify_qp(struct ib_qp *ibqp,
ctrl = get_send_wqe(qp, i); ctrl = get_send_wqe(qp, i);
ctrl->owner_opcode = cpu_to_be32(1 << 31); ctrl->owner_opcode = cpu_to_be32(1 << 31);
if (qp->sq_max_wqes_per_wr == 1) if (qp->sq_max_wqes_per_wr == 1)
ctrl->fence_size = 1 << (qp->sq.wqe_shift - 4); ctrl->qpn_vlan.fence_size =
1 << (qp->sq.wqe_shift - 4);
stamp_send_wqe(qp, i, 1 << qp->sq.wqe_shift); stamp_send_wqe(qp, i, 1 << qp->sq.wqe_shift);
} }
...@@ -3169,7 +3170,7 @@ int mlx4_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, ...@@ -3169,7 +3170,7 @@ int mlx4_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
wmb(); wmb();
*lso_wqe = lso_hdr_sz; *lso_wqe = lso_hdr_sz;
ctrl->fence_size = (wr->send_flags & IB_SEND_FENCE ? ctrl->qpn_vlan.fence_size = (wr->send_flags & IB_SEND_FENCE ?
MLX4_WQE_CTRL_FENCE : 0) | size; MLX4_WQE_CTRL_FENCE : 0) | size;
/* /*
......
...@@ -631,8 +631,7 @@ static int get_real_size(const struct sk_buff *skb, ...@@ -631,8 +631,7 @@ static int get_real_size(const struct sk_buff *skb,
static void build_inline_wqe(struct mlx4_en_tx_desc *tx_desc, static void build_inline_wqe(struct mlx4_en_tx_desc *tx_desc,
const struct sk_buff *skb, const struct sk_buff *skb,
const struct skb_shared_info *shinfo, const struct skb_shared_info *shinfo,
int real_size, u16 *vlan_tag, void *fragptr)
int tx_ind, void *fragptr)
{ {
struct mlx4_wqe_inline_seg *inl = &tx_desc->inl; struct mlx4_wqe_inline_seg *inl = &tx_desc->inl;
int spc = MLX4_INLINE_ALIGN - CTRL_SIZE - sizeof *inl; int spc = MLX4_INLINE_ALIGN - CTRL_SIZE - sizeof *inl;
...@@ -700,10 +699,66 @@ static void mlx4_bf_copy(void __iomem *dst, const void *src, ...@@ -700,10 +699,66 @@ static void mlx4_bf_copy(void __iomem *dst, const void *src,
__iowrite64_copy(dst, src, bytecnt / 8); __iowrite64_copy(dst, src, bytecnt / 8);
} }
void mlx4_en_xmit_doorbell(struct mlx4_en_tx_ring *ring)
{
wmb();
/* Since there is no iowrite*_native() that writes the
* value as is, without byteswapping - using the one
* the doesn't do byteswapping in the relevant arch
* endianness.
*/
#if defined(__LITTLE_ENDIAN)
iowrite32(
#else
iowrite32be(
#endif
ring->doorbell_qpn,
ring->bf.uar->map + MLX4_SEND_DOORBELL);
}
static void mlx4_en_tx_write_desc(struct mlx4_en_tx_ring *ring,
struct mlx4_en_tx_desc *tx_desc,
union mlx4_wqe_qpn_vlan qpn_vlan,
int desc_size, int bf_index,
__be32 op_own, bool bf_ok,
bool send_doorbell)
{
tx_desc->ctrl.qpn_vlan = qpn_vlan;
if (bf_ok) {
op_own |= htonl((bf_index & 0xffff) << 8);
/* Ensure new descriptor hits memory
* before setting ownership of this descriptor to HW
*/
dma_wmb();
tx_desc->ctrl.owner_opcode = op_own;
wmb();
mlx4_bf_copy(ring->bf.reg + ring->bf.offset, &tx_desc->ctrl,
desc_size);
wmb();
ring->bf.offset ^= ring->bf.buf_size;
} else {
/* Ensure new descriptor hits memory
* before setting ownership of this descriptor to HW
*/
dma_wmb();
tx_desc->ctrl.owner_opcode = op_own;
if (send_doorbell)
mlx4_en_xmit_doorbell(ring);
else
ring->xmit_more++;
}
}
netdev_tx_t mlx4_en_xmit(struct sk_buff *skb, struct net_device *dev) netdev_tx_t mlx4_en_xmit(struct sk_buff *skb, struct net_device *dev)
{ {
struct skb_shared_info *shinfo = skb_shinfo(skb); struct skb_shared_info *shinfo = skb_shinfo(skb);
struct mlx4_en_priv *priv = netdev_priv(dev); struct mlx4_en_priv *priv = netdev_priv(dev);
union mlx4_wqe_qpn_vlan qpn_vlan = {};
struct device *ddev = priv->ddev; struct device *ddev = priv->ddev;
struct mlx4_en_tx_ring *ring; struct mlx4_en_tx_ring *ring;
struct mlx4_en_tx_desc *tx_desc; struct mlx4_en_tx_desc *tx_desc;
...@@ -715,7 +770,6 @@ netdev_tx_t mlx4_en_xmit(struct sk_buff *skb, struct net_device *dev) ...@@ -715,7 +770,6 @@ netdev_tx_t mlx4_en_xmit(struct sk_buff *skb, struct net_device *dev)
int real_size; int real_size;
u32 index, bf_index; u32 index, bf_index;
__be32 op_own; __be32 op_own;
u16 vlan_tag = 0;
u16 vlan_proto = 0; u16 vlan_proto = 0;
int i_frag; int i_frag;
int lso_header_size; int lso_header_size;
...@@ -725,6 +779,7 @@ netdev_tx_t mlx4_en_xmit(struct sk_buff *skb, struct net_device *dev) ...@@ -725,6 +779,7 @@ netdev_tx_t mlx4_en_xmit(struct sk_buff *skb, struct net_device *dev)
bool stop_queue; bool stop_queue;
bool inline_ok; bool inline_ok;
u32 ring_cons; u32 ring_cons;
bool bf_ok;
tx_ind = skb_get_queue_mapping(skb); tx_ind = skb_get_queue_mapping(skb);
ring = priv->tx_ring[tx_ind]; ring = priv->tx_ring[tx_ind];
...@@ -749,9 +804,17 @@ netdev_tx_t mlx4_en_xmit(struct sk_buff *skb, struct net_device *dev) ...@@ -749,9 +804,17 @@ netdev_tx_t mlx4_en_xmit(struct sk_buff *skb, struct net_device *dev)
goto tx_drop; goto tx_drop;
} }
bf_ok = ring->bf_enabled;
if (skb_vlan_tag_present(skb)) { if (skb_vlan_tag_present(skb)) {
vlan_tag = skb_vlan_tag_get(skb); qpn_vlan.vlan_tag = cpu_to_be16(skb_vlan_tag_get(skb));
vlan_proto = be16_to_cpu(skb->vlan_proto); vlan_proto = be16_to_cpu(skb->vlan_proto);
if (vlan_proto == ETH_P_8021AD)
qpn_vlan.ins_vlan = MLX4_WQE_CTRL_INS_SVLAN;
else if (vlan_proto == ETH_P_8021Q)
qpn_vlan.ins_vlan = MLX4_WQE_CTRL_INS_CVLAN;
else
qpn_vlan.ins_vlan = 0;
bf_ok = false;
} }
netdev_txq_bql_enqueue_prefetchw(ring->tx_queue); netdev_txq_bql_enqueue_prefetchw(ring->tx_queue);
...@@ -771,6 +834,7 @@ netdev_tx_t mlx4_en_xmit(struct sk_buff *skb, struct net_device *dev) ...@@ -771,6 +834,7 @@ netdev_tx_t mlx4_en_xmit(struct sk_buff *skb, struct net_device *dev)
else { else {
tx_desc = (struct mlx4_en_tx_desc *) ring->bounce_buf; tx_desc = (struct mlx4_en_tx_desc *) ring->bounce_buf;
bounce = true; bounce = true;
bf_ok = false;
} }
/* Save skb in tx_info ring */ /* Save skb in tx_info ring */
...@@ -907,8 +971,7 @@ netdev_tx_t mlx4_en_xmit(struct sk_buff *skb, struct net_device *dev) ...@@ -907,8 +971,7 @@ netdev_tx_t mlx4_en_xmit(struct sk_buff *skb, struct net_device *dev)
AVG_PERF_COUNTER(priv->pstats.tx_pktsz_avg, skb->len); AVG_PERF_COUNTER(priv->pstats.tx_pktsz_avg, skb->len);
if (tx_info->inl) if (tx_info->inl)
build_inline_wqe(tx_desc, skb, shinfo, real_size, &vlan_tag, build_inline_wqe(tx_desc, skb, shinfo, fragptr);
tx_ind, fragptr);
if (skb->encapsulation) { if (skb->encapsulation) {
union { union {
...@@ -946,60 +1009,15 @@ netdev_tx_t mlx4_en_xmit(struct sk_buff *skb, struct net_device *dev) ...@@ -946,60 +1009,15 @@ netdev_tx_t mlx4_en_xmit(struct sk_buff *skb, struct net_device *dev)
real_size = (real_size / 16) & 0x3f; real_size = (real_size / 16) & 0x3f;
if (ring->bf_enabled && desc_size <= MAX_BF && !bounce && bf_ok &= desc_size <= MAX_BF && send_doorbell;
!skb_vlan_tag_present(skb) && send_doorbell) {
tx_desc->ctrl.bf_qpn = ring->doorbell_qpn |
cpu_to_be32(real_size);
op_own |= htonl((bf_index & 0xffff) << 8);
/* Ensure new descriptor hits memory
* before setting ownership of this descriptor to HW
*/
dma_wmb();
tx_desc->ctrl.owner_opcode = op_own;
wmb();
mlx4_bf_copy(ring->bf.reg + ring->bf.offset, &tx_desc->ctrl,
desc_size);
wmb();
ring->bf.offset ^= ring->bf.buf_size; if (bf_ok)
} else { qpn_vlan.bf_qpn = ring->doorbell_qpn | cpu_to_be32(real_size);
tx_desc->ctrl.vlan_tag = cpu_to_be16(vlan_tag);
if (vlan_proto == ETH_P_8021AD)
tx_desc->ctrl.ins_vlan = MLX4_WQE_CTRL_INS_SVLAN;
else if (vlan_proto == ETH_P_8021Q)
tx_desc->ctrl.ins_vlan = MLX4_WQE_CTRL_INS_CVLAN;
else else
tx_desc->ctrl.ins_vlan = 0; qpn_vlan.fence_size = real_size;
tx_desc->ctrl.fence_size = real_size;
/* Ensure new descriptor hits memory mlx4_en_tx_write_desc(ring, tx_desc, qpn_vlan, desc_size, bf_index,
* before setting ownership of this descriptor to HW op_own, bf_ok, send_doorbell);
*/
dma_wmb();
tx_desc->ctrl.owner_opcode = op_own;
if (send_doorbell) {
wmb();
/* Since there is no iowrite*_native() that writes the
* value as is, without byteswapping - using the one
* the doesn't do byteswapping in the relevant arch
* endianness.
*/
#if defined(__LITTLE_ENDIAN)
iowrite32(
#else
iowrite32be(
#endif
ring->doorbell_qpn,
ring->bf.uar->map + MLX4_SEND_DOORBELL);
} else {
ring->xmit_more++;
}
}
if (unlikely(stop_queue)) { if (unlikely(stop_queue)) {
/* If queue was emptied after the if (stop_queue) , and before /* If queue was emptied after the if (stop_queue) , and before
......
...@@ -291,16 +291,18 @@ enum { ...@@ -291,16 +291,18 @@ enum {
MLX4_WQE_CTRL_FORCE_LOOPBACK = 1 << 0, MLX4_WQE_CTRL_FORCE_LOOPBACK = 1 << 0,
}; };
struct mlx4_wqe_ctrl_seg { union mlx4_wqe_qpn_vlan {
__be32 owner_opcode;
union {
struct { struct {
__be16 vlan_tag; __be16 vlan_tag;
u8 ins_vlan; u8 ins_vlan;
u8 fence_size; u8 fence_size;
}; };
__be32 bf_qpn; __be32 bf_qpn;
}; };
struct mlx4_wqe_ctrl_seg {
__be32 owner_opcode;
union mlx4_wqe_qpn_vlan qpn_vlan;
/* /*
* High 24 bits are SRC remote buffer; low 8 bits are flags: * High 24 bits are SRC remote buffer; low 8 bits are flags:
* [7] SO (strong ordering) * [7] SO (strong ordering)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment