Skip to content

L
linux
Commit 7b1e46c5 authored by David S. Miller's avatar David S. Miller
Browse files
Options

Merge branch 'ixgbevf' 

Aaron Brown says:

====================
Intel Wired LAN Driver Updates

This series contains updates from Emil to ixgbevf.

He cleans up the code by removing the adapter structure as a
parameter from multiple functions in favor of using the ixgbevf_ring
structure and moves hot-path specific statistic int the ring
structure for anticipated performance gains.

He also removes the Tx/Rx counters for checksum offload and adds
counters for tx_restart_queue and tx_timeout_count.

Next he makes it so that the first tx_buffer structure acts as a
central storage location for most the skb info we are about to
transmit, then takes advantage of the dma buffer always being
present in the first descriptor and mapped as single allowing a
call to dma_unmap_single which alleviates the need to check for
DMA mapping in ixgbevf_clean_tx_irq().

Finally he merges the ixgbevf_tx_map call and the ixgbevf_tx_queue
call into a single function.
====================
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parents 144651d1 29d37fa1
Show whitespace changes
Inline Side-by-side
Showing with 403 additions and 370 deletions
drivers/net/ethernet/intel/ixgbevf/defines.h
View file @ 7b1e46c5
......@@ -183,6 +183,7 @@ typedef u32 ixgbe_link_speed;
#define IXGBE_TXD_CMD_DEXT 0x20000000 /* Descriptor extension (0 = legacy) */
#define IXGBE_TXD_CMD_VLE 0x40000000 /* Add VLAN tag */
#define IXGBE_TXD_STAT_DD 0x00000001 /* Descriptor Done */
#define IXGBE_TXD_CMD (IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS)
/* Transmit Descriptor - Advanced */
union ixgbe_adv_tx_desc {
......
drivers/net/ethernet/intel/ixgbevf/ethtool.c
View file @ 7b1e46c5
......@@ -77,11 +77,11 @@ static const struct ixgbe_stats ixgbe_gstrings_stats[] = {
{"tx_bytes", IXGBEVF_STAT(stats.vfgotc, stats.base_vfgotc,
stats.saved_reset_vfgotc)},
{"tx_busy", IXGBEVF_ZSTAT(tx_busy)},
{"tx_restart_queue", IXGBEVF_ZSTAT(restart_queue)},
{"tx_timeout_count", IXGBEVF_ZSTAT(tx_timeout_count)},
{"multicast", IXGBEVF_STAT(stats.vfmprc, stats.base_vfmprc,
stats.saved_reset_vfmprc)},
{"rx_csum_offload_good", IXGBEVF_ZSTAT(hw_csum_rx_good)},
{"rx_csum_offload_errors", IXGBEVF_ZSTAT(hw_csum_rx_error)},
{"tx_csum_offload_ctxt", IXGBEVF_ZSTAT(hw_csum_tx_good)},
#ifdef BP_EXTENDED_STATS
{"rx_bp_poll_yield", IXGBEVF_ZSTAT(bp_rx_yields)},
{"rx_bp_cleaned", IXGBEVF_ZSTAT(bp_rx_cleaned)},
......@@ -305,12 +305,11 @@ static int ixgbevf_set_ringparam(struct net_device *netdev,
/* clone ring and setup updated count */
tx_ring[i] = *adapter->tx_ring[i];
tx_ring[i].count = new_tx_count;
err = ixgbevf_setup_tx_resources(adapter, &tx_ring[i]);
if (!err)
continue;
err = ixgbevf_setup_tx_resources(&tx_ring[i]);
if (err) {
while (i) {
i--;
ixgbevf_free_tx_resources(adapter, &tx_ring[i]);
ixgbevf_free_tx_resources(&tx_ring[i]);
}
vfree(tx_ring);
......@@ -319,6 +318,7 @@ static int ixgbevf_set_ringparam(struct net_device *netdev,
goto clear_reset;
}
}
}
if (new_rx_count != adapter->rx_ring_count) {
rx_ring = vmalloc(adapter->num_rx_queues * sizeof(*rx_ring));
......@@ -331,12 +331,11 @@ static int ixgbevf_set_ringparam(struct net_device *netdev,
/* clone ring and setup updated count */
rx_ring[i] = *adapter->rx_ring[i];
rx_ring[i].count = new_rx_count;
err = ixgbevf_setup_rx_resources(adapter, &rx_ring[i]);
if (!err)
continue;
err = ixgbevf_setup_rx_resources(&rx_ring[i]);
if (err) {
while (i) {
i--;
ixgbevf_free_rx_resources(adapter, &rx_ring[i]);
ixgbevf_free_rx_resources(&rx_ring[i]);
}
vfree(rx_ring);
......@@ -345,6 +344,7 @@ static int ixgbevf_set_ringparam(struct net_device *netdev,
goto clear_reset;
}
}
}
/* bring interface down to prepare for update */
ixgbevf_down(adapter);
......@@ -352,7 +352,7 @@ static int ixgbevf_set_ringparam(struct net_device *netdev,
/* Tx */
if (tx_ring) {
for (i = 0; i < adapter->num_tx_queues; i++) {
ixgbevf_free_tx_resources(adapter, adapter->tx_ring[i]);
ixgbevf_free_tx_resources(adapter->tx_ring[i]);
*adapter->tx_ring[i] = tx_ring[i];
}
adapter->tx_ring_count = new_tx_count;
......@@ -364,7 +364,7 @@ static int ixgbevf_set_ringparam(struct net_device *netdev,
/* Rx */
if (rx_ring) {
for (i = 0; i < adapter->num_rx_queues; i++) {
ixgbevf_free_rx_resources(adapter, adapter->rx_ring[i]);
ixgbevf_free_rx_resources(adapter->rx_ring[i]);
*adapter->rx_ring[i] = rx_ring[i];
}
adapter->rx_ring_count = new_rx_count;
......@@ -380,7 +380,7 @@ static int ixgbevf_set_ringparam(struct net_device *netdev,
/* free Tx resources if Rx error is encountered */
if (tx_ring) {
for (i = 0; i < adapter->num_tx_queues; i++)
ixgbevf_free_tx_resources(adapter, &tx_ring[i]);
ixgbevf_free_tx_resources(&tx_ring[i]);
vfree(tx_ring);
}
......@@ -411,15 +411,15 @@ static void ixgbevf_get_ethtool_stats(struct net_device *netdev,
tx_yields = 0, tx_cleaned = 0, tx_missed = 0;
for (i = 0; i < adapter->num_rx_queues; i++) {
rx_yields += adapter->rx_ring[i]->bp_yields;
rx_cleaned += adapter->rx_ring[i]->bp_cleaned;
rx_yields += adapter->rx_ring[i]->bp_yields;
rx_yields += adapter->rx_ring[i]->stats.yields;
rx_cleaned += adapter->rx_ring[i]->stats.cleaned;
rx_yields += adapter->rx_ring[i]->stats.yields;
}
for (i = 0; i < adapter->num_tx_queues; i++) {
tx_yields += adapter->tx_ring[i]->bp_yields;
tx_cleaned += adapter->tx_ring[i]->bp_cleaned;
tx_yields += adapter->tx_ring[i]->bp_yields;
tx_yields += adapter->tx_ring[i]->stats.yields;
tx_cleaned += adapter->tx_ring[i]->stats.cleaned;
tx_yields += adapter->tx_ring[i]->stats.yields;
}
adapter->bp_rx_yields = rx_yields;
......
drivers/net/ethernet/intel/ixgbevf/ixgbevf.h
View file @ 7b1e46c5
......@@ -46,12 +46,15 @@
/* wrapper around a pointer to a socket buffer,
* so a DMA handle can be stored along with the buffer */
struct ixgbevf_tx_buffer {
struct sk_buff *skb;
dma_addr_t dma;
unsigned long time_stamp;
union ixgbe_adv_tx_desc *next_to_watch;
u16 length;
u16 mapped_as_page;
unsigned long time_stamp;
struct sk_buff *skb;
unsigned int bytecount;
unsigned short gso_segs;
__be16 protocol;
DEFINE_DMA_UNMAP_ADDR(dma);
DEFINE_DMA_UNMAP_LEN(len);
u32 tx_flags;
};
struct ixgbevf_rx_buffer {
......@@ -59,6 +62,29 @@ struct ixgbevf_rx_buffer {
dma_addr_t dma;
};
struct ixgbevf_stats {
u64 packets;
u64 bytes;
#ifdef BP_EXTENDED_STATS
u64 yields;
u64 misses;
u64 cleaned;
#endif
};
struct ixgbevf_tx_queue_stats {
u64 restart_queue;
u64 tx_busy;
u64 tx_done_old;
};
struct ixgbevf_rx_queue_stats {
u64 non_eop_descs;
u64 alloc_rx_page_failed;
u64 alloc_rx_buff_failed;
u64 csum_err;
};
struct ixgbevf_ring {
struct ixgbevf_ring *next;
struct net_device *netdev;
......@@ -70,22 +96,19 @@ struct ixgbevf_ring {
unsigned int next_to_use;
unsigned int next_to_clean;
int queue_index; /* needed for multiqueue queue management */
union {
struct ixgbevf_tx_buffer *tx_buffer_info;
struct ixgbevf_rx_buffer *rx_buffer_info;
};
u64 total_bytes;
u64 total_packets;
struct ixgbevf_stats stats;
struct u64_stats_sync syncp;
union {
struct ixgbevf_tx_queue_stats tx_stats;
struct ixgbevf_rx_queue_stats rx_stats;
};
u64 hw_csum_rx_error;
u64 hw_csum_rx_good;
#ifdef BP_EXTENDED_STATS
u64 bp_yields;
u64 bp_misses;
u64 bp_cleaned;
#endif
u8 __iomem *tail;
u16 reg_idx; /* holds the special value that gets the hardware register
......@@ -93,6 +116,7 @@ struct ixgbevf_ring {
* for DCB and RSS modes */
u16 rx_buf_len;
int queue_index; /* needed for multiqueue queue management */
};
/* How many Rx Buffers do we bundle into one write to the hardware ? */
......@@ -123,8 +147,6 @@ struct ixgbevf_ring {
#define IXGBE_TX_FLAGS_VLAN (u32)(1 << 1)
#define IXGBE_TX_FLAGS_TSO (u32)(1 << 2)
#define IXGBE_TX_FLAGS_IPV4 (u32)(1 << 3)
#define IXGBE_TX_FLAGS_FCOE (u32)(1 << 4)
#define IXGBE_TX_FLAGS_FSO (u32)(1 << 5)
#define IXGBE_TX_FLAGS_VLAN_MASK 0xffff0000
#define IXGBE_TX_FLAGS_VLAN_PRIO_MASK 0x0000e000
#define IXGBE_TX_FLAGS_VLAN_SHIFT 16
......@@ -186,7 +208,7 @@ static inline bool ixgbevf_qv_lock_napi(struct ixgbevf_q_vector *q_vector)
q_vector->state |= IXGBEVF_QV_STATE_NAPI_YIELD;
rc = false;
#ifdef BP_EXTENDED_STATS
q_vector->tx.ring->bp_yields++;
q_vector->tx.ring->stats.yields++;
#endif
} else {
/* we don't care if someone yielded */
......@@ -221,7 +243,7 @@ static inline bool ixgbevf_qv_lock_poll(struct ixgbevf_q_vector *q_vector)
q_vector->state |= IXGBEVF_QV_STATE_POLL_YIELD;
rc = false;
#ifdef BP_EXTENDED_STATS
q_vector->rx.ring->bp_yields++;
q_vector->rx.ring->stats.yields++;
#endif
} else {
/* preserve yield marks */
......@@ -314,7 +336,6 @@ static inline u16 ixgbevf_desc_unused(struct ixgbevf_ring *ring)
struct ixgbevf_adapter {
struct timer_list watchdog_timer;
unsigned long active_vlans[BITS_TO_LONGS(VLAN_N_VID)];
u16 bd_number;
struct work_struct reset_task;
struct ixgbevf_q_vector *q_vector[MAX_MSIX_Q_VECTORS];
......@@ -327,25 +348,18 @@ struct ixgbevf_adapter {
u32 eims_other;
/* TX */
struct ixgbevf_ring *tx_ring[MAX_TX_QUEUES]; /* One per active queue */
int num_tx_queues;
struct ixgbevf_ring *tx_ring[MAX_TX_QUEUES]; /* One per active queue */
u64 restart_queue;
u64 hw_csum_tx_good;
u64 lsc_int;
u64 hw_tso_ctxt;
u64 hw_tso6_ctxt;
u32 tx_timeout_count;
/* RX */
struct ixgbevf_ring *rx_ring[MAX_TX_QUEUES]; /* One per active queue */
int num_rx_queues;
struct ixgbevf_ring *rx_ring[MAX_TX_QUEUES]; /* One per active queue */
u64 hw_csum_rx_error;
u64 hw_rx_no_dma_resources;
u64 hw_csum_rx_good;
u64 non_eop_descs;
int num_msix_vectors;
struct msix_entry *msix_entries;
u32 alloc_rx_page_failed;
u32 alloc_rx_buff_failed;
......@@ -357,6 +371,8 @@ struct ixgbevf_adapter {
#define IXGBE_FLAG_IN_NETPOLL (u32)(1 << 1)
#define IXGBEVF_FLAG_QUEUE_RESET_REQUESTED (u32)(1 << 2)
struct msix_entry *msix_entries;
/* OS defined structs */
struct net_device *netdev;
struct pci_dev *pdev;
......@@ -364,10 +380,12 @@ struct ixgbevf_adapter {
/* structs defined in ixgbe_vf.h */
struct ixgbe_hw hw;
u16 msg_enable;
struct ixgbevf_hw_stats stats;
u16 bd_number;
/* Interrupt Throttle Rate */
u32 eitr_param;
struct ixgbevf_hw_stats stats;
unsigned long state;
u64 tx_busy;
unsigned int tx_ring_count;
......@@ -386,9 +404,9 @@ struct ixgbevf_adapter {
u32 link_speed;
bool link_up;
struct work_struct watchdog_task;
spinlock_t mbx_lock;
struct work_struct watchdog_task;
};
enum ixbgevf_state_t {
......@@ -420,10 +438,10 @@ void ixgbevf_down(struct ixgbevf_adapter *adapter);
void ixgbevf_reinit_locked(struct ixgbevf_adapter *adapter);
void ixgbevf_reset(struct ixgbevf_adapter *adapter);
void ixgbevf_set_ethtool_ops(struct net_device *netdev);
int ixgbevf_setup_rx_resources(struct ixgbevf_adapter *, struct ixgbevf_ring *);
int ixgbevf_setup_tx_resources(struct ixgbevf_adapter *, struct ixgbevf_ring *);
void ixgbevf_free_rx_resources(struct ixgbevf_adapter *, struct ixgbevf_ring *);
void ixgbevf_free_tx_resources(struct ixgbevf_adapter *, struct ixgbevf_ring *);
int ixgbevf_setup_rx_resources(struct ixgbevf_ring *);
int ixgbevf_setup_tx_resources(struct ixgbevf_ring *);
void ixgbevf_free_rx_resources(struct ixgbevf_ring *);
void ixgbevf_free_tx_resources(struct ixgbevf_ring *);
void ixgbevf_update_stats(struct ixgbevf_adapter *adapter);
int ethtool_ioctl(struct ifreq *ifr);
......
drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c
View file @ 7b1e46c5
......@@ -145,28 +145,25 @@ static void ixgbevf_set_ivar(struct ixgbevf_adapter *adapter, s8 direction,
}
static void ixgbevf_unmap_and_free_tx_resource(struct ixgbevf_ring *tx_ring,
struct ixgbevf_tx_buffer
*tx_buffer_info)
struct ixgbevf_tx_buffer *tx_buffer)
{
if (tx_buffer_info->dma) {
if (tx_buffer_info->mapped_as_page)
dma_unmap_page(tx_ring->dev,
tx_buffer_info->dma,
tx_buffer_info->length,
DMA_TO_DEVICE);
else
if (tx_buffer->skb) {
dev_kfree_skb_any(tx_buffer->skb);
if (dma_unmap_len(tx_buffer, len))
dma_unmap_single(tx_ring->dev,
tx_buffer_info->dma,
tx_buffer_info->length,
dma_unmap_addr(tx_buffer, dma),
dma_unmap_len(tx_buffer, len),
DMA_TO_DEVICE);
} else if (dma_unmap_len(tx_buffer, len)) {
dma_unmap_page(tx_ring->dev,
dma_unmap_addr(tx_buffer, dma),
dma_unmap_len(tx_buffer, len),
DMA_TO_DEVICE);
tx_buffer_info->dma = 0;
}
if (tx_buffer_info->skb) {
dev_kfree_skb_any(tx_buffer_info->skb);
tx_buffer_info->skb = NULL;
}
tx_buffer_info->time_stamp = 0;
/* tx_buffer_info must be completely set up in the transmit path */
tx_buffer->next_to_watch = NULL;
tx_buffer->skb = NULL;
dma_unmap_len_set(tx_buffer, len, 0);
/* tx_buffer must be completely set up in the transmit path */
}
#define IXGBE_MAX_TXD_PWR 14
......@@ -187,20 +184,21 @@ static bool ixgbevf_clean_tx_irq(struct ixgbevf_q_vector *q_vector,
struct ixgbevf_ring *tx_ring)
{
struct ixgbevf_adapter *adapter = q_vector->adapter;
union ixgbe_adv_tx_desc *tx_desc, *eop_desc;
struct ixgbevf_tx_buffer *tx_buffer_info;
unsigned int i, count = 0;
struct ixgbevf_tx_buffer *tx_buffer;
union ixgbe_adv_tx_desc *tx_desc;
unsigned int total_bytes = 0, total_packets = 0;
unsigned int budget = tx_ring->count / 2;
unsigned int i = tx_ring->next_to_clean;
if (test_bit(__IXGBEVF_DOWN, &adapter->state))
return true;
i = tx_ring->next_to_clean;
tx_buffer_info = &tx_ring->tx_buffer_info[i];
eop_desc = tx_buffer_info->next_to_watch;
tx_buffer = &tx_ring->tx_buffer_info[i];
tx_desc = IXGBEVF_TX_DESC(tx_ring, i);
i -= tx_ring->count;
do {
bool cleaned = false;
union ixgbe_adv_tx_desc *eop_desc = tx_buffer->next_to_watch;
/* if next_to_watch is not set then there is no work pending */
if (!eop_desc)
......@@ -214,67 +212,90 @@ static bool ixgbevf_clean_tx_irq(struct ixgbevf_q_vector *q_vector,
break;
/* clear next_to_watch to prevent false hangs */
tx_buffer_info->next_to_watch = NULL;
for ( ; !cleaned; count++) {
struct sk_buff *skb;
tx_desc = IXGBEVF_TX_DESC(tx_ring, i);
cleaned = (tx_desc == eop_desc);
skb = tx_buffer_info->skb;
tx_buffer->next_to_watch = NULL;
if (cleaned && skb) {
unsigned int segs, bytecount;
/* update the statistics for this packet */
total_bytes += tx_buffer->bytecount;
total_packets += tx_buffer->gso_segs;
/* gso_segs is currently only valid for tcp */
segs = skb_shinfo(skb)->gso_segs ?: 1;
/* multiply data chunks by size of headers */
bytecount = ((segs - 1) * skb_headlen(skb)) +
skb->len;
total_packets += segs;
total_bytes += bytecount;
}
/* free the skb */
dev_kfree_skb_any(tx_buffer->skb);
ixgbevf_unmap_and_free_tx_resource(tx_ring,
tx_buffer_info);
/* unmap skb header data */
dma_unmap_single(tx_ring->dev,
dma_unmap_addr(tx_buffer, dma),
dma_unmap_len(tx_buffer, len),
DMA_TO_DEVICE);
tx_desc->wb.status = 0;
/* clear tx_buffer data */
tx_buffer->skb = NULL;
dma_unmap_len_set(tx_buffer, len, 0);
/* unmap remaining buffers */
while (tx_desc != eop_desc) {
tx_buffer++;
tx_desc++;
i++;
if (i == tx_ring->count)
i = 0;
if (unlikely(!i)) {
i -= tx_ring->count;
tx_buffer = tx_ring->tx_buffer_info;
tx_desc = IXGBEVF_TX_DESC(tx_ring, 0);
}
tx_buffer_info = &tx_ring->tx_buffer_info[i];
/* unmap any remaining paged data */
if (dma_unmap_len(tx_buffer, len)) {
dma_unmap_page(tx_ring->dev,
dma_unmap_addr(tx_buffer, dma),
dma_unmap_len(tx_buffer, len),
DMA_TO_DEVICE);
dma_unmap_len_set(tx_buffer, len, 0);
}
}
eop_desc = tx_buffer_info->next_to_watch;
} while (count < tx_ring->count);
/* move us one more past the eop_desc for start of next pkt */
tx_buffer++;
tx_desc++;
i++;
if (unlikely(!i)) {
i -= tx_ring->count;
tx_buffer = tx_ring->tx_buffer_info;
tx_desc = IXGBEVF_TX_DESC(tx_ring, 0);
}
/* issue prefetch for next Tx descriptor */
prefetch(tx_desc);
/* update budget accounting */
budget--;
} while (likely(budget));
i += tx_ring->count;
tx_ring->next_to_clean = i;
u64_stats_update_begin(&tx_ring->syncp);
tx_ring->stats.bytes += total_bytes;
tx_ring->stats.packets += total_packets;
u64_stats_update_end(&tx_ring->syncp);
q_vector->tx.total_bytes += total_bytes;
q_vector->tx.total_packets += total_packets;
#define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
if (unlikely(count && netif_carrier_ok(tx_ring->netdev) &&
if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) &&
(ixgbevf_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD))) {
/* Make sure that anybody stopping the queue after this
* sees the new next_to_clean.
*/
smp_mb();
if (__netif_subqueue_stopped(tx_ring->netdev,
tx_ring->queue_index) &&
!test_bit(__IXGBEVF_DOWN, &adapter->state)) {
netif_wake_subqueue(tx_ring->netdev,
tx_ring->queue_index);
++adapter->restart_queue;
++tx_ring->tx_stats.restart_queue;
}
}
u64_stats_update_begin(&tx_ring->syncp);
tx_ring->total_bytes += total_bytes;
tx_ring->total_packets += total_packets;
u64_stats_update_end(&tx_ring->syncp);
q_vector->tx.total_bytes += total_bytes;
q_vector->tx.total_packets += total_packets;
return count < tx_ring->count;
return !!budget;
}
/**
......@@ -343,7 +364,7 @@ static inline void ixgbevf_rx_checksum(struct ixgbevf_ring *ring,
/* if IP and error */
if ((status_err & IXGBE_RXD_STAT_IPCS) &&
(status_err & IXGBE_RXDADV_ERR_IPE)) {
ring->hw_csum_rx_error++;
ring->rx_stats.csum_err++;
return;
}
......@@ -351,51 +372,46 @@ static inline void ixgbevf_rx_checksum(struct ixgbevf_ring *ring,
return;
if (status_err & IXGBE_RXDADV_ERR_TCPE) {
ring->hw_csum_rx_error++;
ring->rx_stats.csum_err++;
return;
}
/* It must be a TCP or UDP packet with a valid checksum */
skb->ip_summed = CHECKSUM_UNNECESSARY;
ring->hw_csum_rx_good++;
}
/**
* ixgbevf_alloc_rx_buffers - Replace used receive buffers; packet split
* @adapter: address of board private structure
* @rx_ring: rx descriptor ring (for a specific queue) to setup buffers on
**/
static void ixgbevf_alloc_rx_buffers(struct ixgbevf_adapter *adapter,
struct ixgbevf_ring *rx_ring,
static void ixgbevf_alloc_rx_buffers(struct ixgbevf_ring *rx_ring,
int cleaned_count)
{
struct pci_dev *pdev = adapter->pdev;
union ixgbe_adv_rx_desc *rx_desc;
struct ixgbevf_rx_buffer *bi;
unsigned int i = rx_ring->next_to_use;
bi = &rx_ring->rx_buffer_info[i];
while (cleaned_count--) {
rx_desc = IXGBEVF_RX_DESC(rx_ring, i);
bi = &rx_ring->rx_buffer_info[i];
if (!bi->skb) {
struct sk_buff *skb;
skb = netdev_alloc_skb_ip_align(rx_ring->netdev,
rx_ring->rx_buf_len);
if (!skb) {
adapter->alloc_rx_buff_failed++;
if (!skb)
goto no_buffers;
}
bi->skb = skb;
bi->dma = dma_map_single(&pdev->dev, skb->data,
bi->dma = dma_map_single(rx_ring->dev, skb->data,
rx_ring->rx_buf_len,
DMA_FROM_DEVICE);
if (dma_mapping_error(&pdev->dev, bi->dma)) {
if (dma_mapping_error(rx_ring->dev, bi->dma)) {
dev_kfree_skb(skb);
bi->skb = NULL;
dev_err(&pdev->dev, "RX DMA map failed\n");
dev_err(rx_ring->dev, "Rx DMA map failed\n");
break;
}
}
......@@ -404,10 +420,10 @@ static void ixgbevf_alloc_rx_buffers(struct ixgbevf_adapter *adapter,
i++;
if (i == rx_ring->count)
i = 0;
bi = &rx_ring->rx_buffer_info[i];
}
no_buffers:
rx_ring->rx_stats.alloc_rx_buff_failed++;
if (rx_ring->next_to_use != i)
ixgbevf_release_rx_desc(rx_ring, i);
}
......@@ -424,8 +440,6 @@ static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector,
struct ixgbevf_ring *rx_ring,
int budget)
{
struct ixgbevf_adapter *adapter = q_vector->adapter;
struct pci_dev *pdev = adapter->pdev;
union ixgbe_adv_rx_desc *rx_desc, *next_rxd;
struct ixgbevf_rx_buffer *rx_buffer_info, *next_buffer;
struct sk_buff *skb;
......@@ -451,7 +465,7 @@ static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector,
rx_buffer_info->skb = NULL;
if (rx_buffer_info->dma) {
dma_unmap_single(&pdev->dev, rx_buffer_info->dma,
dma_unmap_single(rx_ring->dev, rx_buffer_info->dma,
rx_ring->rx_buf_len,
DMA_FROM_DEVICE);
rx_buffer_info->dma = 0;
......@@ -471,7 +485,7 @@ static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector,
if (!(staterr & IXGBE_RXD_STAT_EOP)) {
skb->next = next_buffer->skb;
IXGBE_CB(skb->next)->prev = skb;
adapter->non_eop_descs++;
rx_ring->rx_stats.non_eop_descs++;
goto next_desc;
}
......@@ -503,7 +517,7 @@ static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector,
* source pruning.
*/
if ((skb->pkt_type & (PACKET_BROADCAST | PACKET_MULTICAST)) &&
ether_addr_equal(adapter->netdev->dev_addr,
ether_addr_equal(rx_ring->netdev->dev_addr,
eth_hdr(skb)->h_source)) {
dev_kfree_skb_irq(skb);
goto next_desc;
......@@ -516,8 +530,7 @@ static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector,
/* return some buffers to hardware, one at a time is too slow */
if (cleaned_count >= IXGBEVF_RX_BUFFER_WRITE) {
ixgbevf_alloc_rx_buffers(adapter, rx_ring,
cleaned_count);
ixgbevf_alloc_rx_buffers(rx_ring, cleaned_count);
cleaned_count = 0;
}
......@@ -532,11 +545,11 @@ static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector,
cleaned_count = ixgbevf_desc_unused(rx_ring);
if (cleaned_count)
ixgbevf_alloc_rx_buffers(adapter, rx_ring, cleaned_count);
ixgbevf_alloc_rx_buffers(rx_ring, cleaned_count);
u64_stats_update_begin(&rx_ring->syncp);
rx_ring->total_packets += total_rx_packets;
rx_ring->total_bytes += total_rx_bytes;
rx_ring->stats.packets += total_rx_packets;
rx_ring->stats.bytes += total_rx_bytes;
u64_stats_update_end(&rx_ring->syncp);
q_vector->rx.total_packets += total_rx_packets;
q_vector->rx.total_bytes += total_rx_bytes;
......@@ -641,9 +654,9 @@ static int ixgbevf_busy_poll_recv(struct napi_struct *napi)
found = ixgbevf_clean_rx_irq(q_vector, ring, 4);
#ifdef BP_EXTENDED_STATS
if (found)
ring->bp_cleaned += found;
ring->stats.cleaned += found;
else
ring->bp_misses++;
ring->stats.misses++;
#endif
if (found)
break;
......@@ -1317,7 +1330,7 @@ static void ixgbevf_configure_rx_ring(struct ixgbevf_adapter *adapter,
IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(reg_idx), rxdctl);
ixgbevf_rx_desc_queue_enable(adapter, ring);
ixgbevf_alloc_rx_buffers(adapter, ring, ixgbevf_desc_unused(ring));
ixgbevf_alloc_rx_buffers(ring, ixgbevf_desc_unused(ring));
}
/**
......@@ -1633,13 +1646,10 @@ void ixgbevf_up(struct ixgbevf_adapter *adapter)
/**
* ixgbevf_clean_rx_ring - Free Rx Buffers per Queue
* @adapter: board private structure
* @rx_ring: ring to free buffers from
**/
static void ixgbevf_clean_rx_ring(struct ixgbevf_adapter *adapter,
struct ixgbevf_ring *rx_ring)
static void ixgbevf_clean_rx_ring(struct ixgbevf_ring *rx_ring)
{
struct pci_dev *pdev = adapter->pdev;
unsigned long size;
unsigned int i;
......@@ -1652,7 +1662,7 @@ static void ixgbevf_clean_rx_ring(struct ixgbevf_adapter *adapter,
rx_buffer_info = &rx_ring->rx_buffer_info[i];
if (rx_buffer_info->dma) {
dma_unmap_single(&pdev->dev, rx_buffer_info->dma,
dma_unmap_single(rx_ring->dev, rx_buffer_info->dma,
rx_ring->rx_buf_len,
DMA_FROM_DEVICE);
rx_buffer_info->dma = 0;
......@@ -1677,11 +1687,9 @@ static void ixgbevf_clean_rx_ring(struct ixgbevf_adapter *adapter,
/**
* ixgbevf_clean_tx_ring - Free Tx Buffers
* @adapter: board private structure
* @tx_ring: ring to be cleaned
**/
static void ixgbevf_clean_tx_ring(struct ixgbevf_adapter *adapter,
struct ixgbevf_ring *tx_ring)
static void ixgbevf_clean_tx_ring(struct ixgbevf_ring *tx_ring)
{
struct ixgbevf_tx_buffer *tx_buffer_info;
unsigned long size;
......@@ -1711,7 +1719,7 @@ static void ixgbevf_clean_all_rx_rings(struct ixgbevf_adapter *adapter)
int i;
for (i = 0; i < adapter->num_rx_queues; i++)
ixgbevf_clean_rx_ring(adapter, adapter->rx_ring[i]);
ixgbevf_clean_rx_ring(adapter->rx_ring[i]);
}
/**
......@@ -1723,7 +1731,7 @@ static void ixgbevf_clean_all_tx_rings(struct ixgbevf_adapter *adapter)
int i;
for (i = 0; i < adapter->num_tx_queues; i++)
ixgbevf_clean_tx_ring(adapter, adapter->tx_ring[i]);
ixgbevf_clean_tx_ring(adapter->tx_ring[i]);
}
void ixgbevf_down(struct ixgbevf_adapter *adapter)
......@@ -2275,10 +2283,7 @@ void ixgbevf_update_stats(struct ixgbevf_adapter *adapter)
for (i = 0; i < adapter->num_rx_queues; i++) {
adapter->hw_csum_rx_error +=
adapter->rx_ring[i]->hw_csum_rx_error;
adapter->hw_csum_rx_good +=
adapter->rx_ring[i]->hw_csum_rx_good;
adapter->rx_ring[i]->hw_csum_rx_error = 0;
adapter->rx_ring[i]->hw_csum_rx_good = 0;
}
}
......@@ -2426,17 +2431,13 @@ static void ixgbevf_watchdog_task(struct work_struct *work)
/**
* ixgbevf_free_tx_resources - Free Tx Resources per Queue
* @adapter: board private structure
* @tx_ring: Tx descriptor ring for a specific queue
*
* Free all transmit software resources
**/
void ixgbevf_free_tx_resources(struct ixgbevf_adapter *adapter,
struct ixgbevf_ring *tx_ring)
void ixgbevf_free_tx_resources(struct ixgbevf_ring *tx_ring)
{
struct pci_dev *pdev = adapter->pdev;
ixgbevf_clean_tx_ring(adapter, tx_ring);
ixgbevf_clean_tx_ring(tx_ring);
vfree(tx_ring->tx_buffer_info);
tx_ring->tx_buffer_info = NULL;
......@@ -2445,7 +2446,7 @@ void ixgbevf_free_tx_resources(struct ixgbevf_adapter *adapter,
if (!tx_ring->desc)
return;
dma_free_coherent(&pdev->dev, tx_ring->size, tx_ring->desc,
dma_free_coherent(tx_ring->dev, tx_ring->size, tx_ring->desc,
tx_ring->dma);
tx_ring->desc = NULL;
......@@ -2463,20 +2464,17 @@ static void ixgbevf_free_all_tx_resources(struct ixgbevf_adapter *adapter)
for (i = 0; i < adapter->num_tx_queues; i++)
if (adapter->tx_ring[i]->desc)
ixgbevf_free_tx_resources(adapter, adapter->tx_ring[i]);
ixgbevf_free_tx_resources(adapter->tx_ring[i]);
}
/**
* ixgbevf_setup_tx_resources - allocate Tx resources (Descriptors)
* @adapter: board private structure
* @tx_ring: tx descriptor ring (for a specific queue) to setup
*
* Return 0 on success, negative on failure
**/
int ixgbevf_setup_tx_resources(struct ixgbevf_adapter *adapter,
struct ixgbevf_ring *tx_ring)
int ixgbevf_setup_tx_resources(struct ixgbevf_ring *tx_ring)
{
struct pci_dev *pdev = adapter->pdev;
int size;
size = sizeof(struct ixgbevf_tx_buffer) * tx_ring->count;
......@@ -2488,7 +2486,7 @@ int ixgbevf_setup_tx_resources(struct ixgbevf_adapter *adapter,
tx_ring->size = tx_ring->count * sizeof(union ixgbe_adv_tx_desc);
tx_ring->size = ALIGN(tx_ring->size, 4096);
tx_ring->desc = dma_alloc_coherent(&pdev->dev, tx_ring->size,
tx_ring->desc = dma_alloc_coherent(tx_ring->dev, tx_ring->size,
&tx_ring->dma, GFP_KERNEL);
if (!tx_ring->desc)
goto err;
......@@ -2518,7 +2516,7 @@ static int ixgbevf_setup_all_tx_resources(struct ixgbevf_adapter *adapter)
int i, err = 0;
for (i = 0; i < adapter->num_tx_queues; i++) {
err = ixgbevf_setup_tx_resources(adapter, adapter->tx_ring[i]);
err = ixgbevf_setup_tx_resources(adapter->tx_ring[i]);
if (!err)
continue;
hw_dbg(&adapter->hw,
......@@ -2531,37 +2529,34 @@ static int ixgbevf_setup_all_tx_resources(struct ixgbevf_adapter *adapter)
/**
* ixgbevf_setup_rx_resources - allocate Rx resources (Descriptors)
* @adapter: board private structure
* @rx_ring: rx descriptor ring (for a specific queue) to setup
*
* Returns 0 on success, negative on failure
**/
int ixgbevf_setup_rx_resources(struct ixgbevf_adapter *adapter,
struct ixgbevf_ring *rx_ring)
int ixgbevf_setup_rx_resources(struct ixgbevf_ring *rx_ring)
{
struct pci_dev *pdev = adapter->pdev;
int size;
size = sizeof(struct ixgbevf_rx_buffer) * rx_ring->count;
rx_ring->rx_buffer_info = vzalloc(size);
if (!rx_ring->rx_buffer_info)
goto alloc_failed;
goto err;
/* Round up to nearest 4K */
rx_ring->size = rx_ring->count * sizeof(union ixgbe_adv_rx_desc);
rx_ring->size = ALIGN(rx_ring->size, 4096);
rx_ring->desc = dma_alloc_coherent(&pdev->dev, rx_ring->size,
rx_ring->desc = dma_alloc_coherent(rx_ring->dev, rx_ring->size,
&rx_ring->dma, GFP_KERNEL);
if (!rx_ring->desc) {
vfree(rx_ring->rx_buffer_info);
rx_ring->rx_buffer_info = NULL;
goto alloc_failed;
}
if (!rx_ring->desc)
goto err;
return 0;
alloc_failed:
err:
vfree(rx_ring->rx_buffer_info);
rx_ring->rx_buffer_info = NULL;
dev_err(rx_ring->dev, "Unable to allocate memory for the Rx descriptor ring\n");
return -ENOMEM;
}
......@@ -2580,7 +2575,7 @@ static int ixgbevf_setup_all_rx_resources(struct ixgbevf_adapter *adapter)
int i, err = 0;
for (i = 0; i < adapter->num_rx_queues; i++) {
err = ixgbevf_setup_rx_resources(adapter, adapter->rx_ring[i]);
err = ixgbevf_setup_rx_resources(adapter->rx_ring[i]);
if (!err)
continue;
hw_dbg(&adapter->hw,
......@@ -2592,22 +2587,18 @@ static int ixgbevf_setup_all_rx_resources(struct ixgbevf_adapter *adapter)
/**
* ixgbevf_free_rx_resources - Free Rx Resources
* @adapter: board private structure
* @rx_ring: ring to clean the resources from
*
* Free all receive software resources
**/
void ixgbevf_free_rx_resources(struct ixgbevf_adapter *adapter,
struct ixgbevf_ring *rx_ring)
void ixgbevf_free_rx_resources(struct ixgbevf_ring *rx_ring)
{
struct pci_dev *pdev = adapter->pdev;
ixgbevf_clean_rx_ring(adapter, rx_ring);
ixgbevf_clean_rx_ring(rx_ring);
vfree(rx_ring->rx_buffer_info);
rx_ring->rx_buffer_info = NULL;
dma_free_coherent(&pdev->dev, rx_ring->size, rx_ring->desc,
dma_free_coherent(rx_ring->dev, rx_ring->size, rx_ring->desc,
rx_ring->dma);
rx_ring->desc = NULL;
......@@ -2625,7 +2616,7 @@ static void ixgbevf_free_all_rx_resources(struct ixgbevf_adapter *adapter)
for (i = 0; i < adapter->num_rx_queues; i++)
if (adapter->rx_ring[i]->desc)
ixgbevf_free_rx_resources(adapter, adapter->rx_ring[i]);
ixgbevf_free_rx_resources(adapter->rx_ring[i]);
}
/**
......@@ -2789,8 +2780,10 @@ static void ixgbevf_tx_ctxtdesc(struct ixgbevf_ring *tx_ring,
}
static int ixgbevf_tso(struct ixgbevf_ring *tx_ring,
struct sk_buff *skb, u32 tx_flags, u8 *hdr_len)
struct ixgbevf_tx_buffer *first,
u8 *hdr_len)
{
struct sk_buff *skb = first->skb;
u32 vlan_macip_lens, type_tucmd;
u32 mss_l4len_idx, l4len;
......@@ -2815,12 +2808,17 @@ static int ixgbevf_tso(struct ixgbevf_ring *tx_ring,
IPPROTO_TCP,
0);
type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4;
first->tx_flags |= IXGBE_TX_FLAGS_TSO |
IXGBE_TX_FLAGS_CSUM |
IXGBE_TX_FLAGS_IPV4;
} else if (skb_is_gso_v6(skb)) {
ipv6_hdr(skb)->payload_len = 0;
tcp_hdr(skb)->check =
~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
&ipv6_hdr(skb)->daddr,
0, IPPROTO_TCP, 0);
first->tx_flags |= IXGBE_TX_FLAGS_TSO |
IXGBE_TX_FLAGS_CSUM;
}
/* compute header lengths */
......@@ -2828,6 +2826,10 @@ static int ixgbevf_tso(struct ixgbevf_ring *tx_ring,
*hdr_len += l4len;
*hdr_len = skb_transport_offset(skb) + l4len;
/* update gso size and bytecount with header size */
first->gso_segs = skb_shinfo(skb)->gso_segs;
first->bytecount += (first->gso_segs - 1) * *hdr_len;
/* mss_l4len_id: use 1 as index for TSO */
mss_l4len_idx = l4len << IXGBE_ADVTXD_L4LEN_SHIFT;
mss_l4len_idx |= skb_shinfo(skb)->gso_size << IXGBE_ADVTXD_MSS_SHIFT;
......@@ -2836,7 +2838,7 @@ static int ixgbevf_tso(struct ixgbevf_ring *tx_ring,
/* vlan_macip_lens: HEADLEN, MACLEN, VLAN tag */
vlan_macip_lens = skb_network_header_len(skb);
vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT;
vlan_macip_lens |= tx_flags & IXGBE_TX_FLAGS_VLAN_MASK;
vlan_macip_lens |= first->tx_flags & IXGBE_TX_FLAGS_VLAN_MASK;
ixgbevf_tx_ctxtdesc(tx_ring, vlan_macip_lens,
type_tucmd, mss_l4len_idx);
......@@ -2844,9 +2846,10 @@ static int ixgbevf_tso(struct ixgbevf_ring *tx_ring,
return 1;
}
static bool ixgbevf_tx_csum(struct ixgbevf_ring *tx_ring,
struct sk_buff *skb, u32 tx_flags)
static void ixgbevf_tx_csum(struct ixgbevf_ring *tx_ring,
struct ixgbevf_tx_buffer *first)
{
struct sk_buff *skb = first->skb;
u32 vlan_macip_lens = 0;
u32 mss_l4len_idx = 0;
u32 type_tucmd = 0;
......@@ -2867,7 +2870,7 @@ static bool ixgbevf_tx_csum(struct ixgbevf_ring *tx_ring,
if (unlikely(net_ratelimit())) {
dev_warn(tx_ring->dev,
"partial checksum but proto=%x!\n",
skb->protocol);
first->protocol);
}
break;
}
......@@ -2895,184 +2898,190 @@ static bool ixgbevf_tx_csum(struct ixgbevf_ring *tx_ring,
}
break;
}
/* update TX checksum flag */
first->tx_flags |= IXGBE_TX_FLAGS_CSUM;
}
/* vlan_macip_lens: MACLEN, VLAN tag */
vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT;
vlan_macip_lens |= tx_flags & IXGBE_TX_FLAGS_VLAN_MASK;
vlan_macip_lens |= first->tx_flags & IXGBE_TX_FLAGS_VLAN_MASK;
ixgbevf_tx_ctxtdesc(tx_ring, vlan_macip_lens,
type_tucmd, mss_l4len_idx);
}
return (skb->ip_summed == CHECKSUM_PARTIAL);
static __le32 ixgbevf_tx_cmd_type(u32 tx_flags)
{
/* set type for advanced descriptor with frame checksum insertion */
__le32 cmd_type = cpu_to_le32(IXGBE_ADVTXD_DTYP_DATA |
IXGBE_ADVTXD_DCMD_IFCS |
IXGBE_ADVTXD_DCMD_DEXT);
/* set HW vlan bit if vlan is present */
if (tx_flags & IXGBE_TX_FLAGS_VLAN)
cmd_type |= cpu_to_le32(IXGBE_ADVTXD_DCMD_VLE);
/* set segmentation enable bits for TSO/FSO */
if (tx_flags & IXGBE_TX_FLAGS_TSO)
cmd_type |= cpu_to_le32(IXGBE_ADVTXD_DCMD_TSE);
return cmd_type;
}
static int ixgbevf_tx_map(struct ixgbevf_ring *tx_ring,
struct sk_buff *skb, u32 tx_flags)
static void ixgbevf_tx_olinfo_status(union ixgbe_adv_tx_desc *tx_desc,
u32 tx_flags, unsigned int paylen)
{
struct ixgbevf_tx_buffer *tx_buffer_info;
unsigned int len;
unsigned int total = skb->len;
unsigned int offset = 0, size;
int count = 0;
unsigned int nr_frags = skb_shinfo(skb)->nr_frags;
unsigned int f;
int i;
__le32 olinfo_status = cpu_to_le32(paylen << IXGBE_ADVTXD_PAYLEN_SHIFT);
i = tx_ring->next_to_use;
/* enable L4 checksum for TSO and TX checksum offload */
if (tx_flags & IXGBE_TX_FLAGS_CSUM)
olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_POPTS_TXSM);
len = min(skb_headlen(skb), total);
while (len) {
tx_buffer_info = &tx_ring->tx_buffer_info[i];
size = min(len, (unsigned int)IXGBE_MAX_DATA_PER_TXD);
tx_buffer_info->length = size;
tx_buffer_info->mapped_as_page = false;
tx_buffer_info->dma = dma_map_single(tx_ring->dev,
skb->data + offset,
size, DMA_TO_DEVICE);
if (dma_mapping_error(tx_ring->dev, tx_buffer_info->dma))
goto dma_error;
/* enble IPv4 checksum for TSO */
if (tx_flags & IXGBE_TX_FLAGS_IPV4)
olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_POPTS_IXSM);
len -= size;
total -= size;
offset += size;
count++;
i++;
if (i == tx_ring->count)
i = 0;
}
/* use index 1 context for TSO/FSO/FCOE */
if (tx_flags & IXGBE_TX_FLAGS_TSO)
olinfo_status |= cpu_to_le32(1 << IXGBE_ADVTXD_IDX_SHIFT);
for (f = 0; f < nr_frags; f++) {
const struct skb_frag_struct *frag;
/* Check Context must be set if Tx switch is enabled, which it
* always is for case where virtual functions are running
*/
olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_CC);
frag = &skb_shinfo(skb)->frags[f];
len = min((unsigned int)skb_frag_size(frag), total);
offset = 0;
tx_desc->read.olinfo_status = olinfo_status;
}
while (len) {
tx_buffer_info = &tx_ring->tx_buffer_info[i];
size = min(len, (unsigned int)IXGBE_MAX_DATA_PER_TXD);
tx_buffer_info->length = size;
tx_buffer_info->dma =
skb_frag_dma_map(tx_ring->dev, frag,
offset, size, DMA_TO_DEVICE);
if (dma_mapping_error(tx_ring->dev,
tx_buffer_info->dma))
static void ixgbevf_tx_map(struct ixgbevf_ring *tx_ring,
struct ixgbevf_tx_buffer *first,
const u8 hdr_len)
{
dma_addr_t dma;
struct sk_buff *skb = first->skb;
struct ixgbevf_tx_buffer *tx_buffer;
union ixgbe_adv_tx_desc *tx_desc;
struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0];
unsigned int data_len = skb->data_len;
unsigned int size = skb_headlen(skb);
unsigned int paylen = skb->len - hdr_len;
u32 tx_flags = first->tx_flags;
__le32 cmd_type;
u16 i = tx_ring->next_to_use;
tx_desc = IXGBEVF_TX_DESC(tx_ring, i);
ixgbevf_tx_olinfo_status(tx_desc, tx_flags, paylen);
cmd_type = ixgbevf_tx_cmd_type(tx_flags);
dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);
if (dma_mapping_error(tx_ring->dev, dma))
goto dma_error;
tx_buffer_info->mapped_as_page = true;
len -= size;
total -= size;
offset += size;
count++;
/* record length, and DMA address */
dma_unmap_len_set(first, len, size);
dma_unmap_addr_set(first, dma, dma);
tx_desc->read.buffer_addr = cpu_to_le64(dma);
for (;;) {
while (unlikely(size > IXGBE_MAX_DATA_PER_TXD)) {
tx_desc->read.cmd_type_len =
cmd_type | cpu_to_le32(IXGBE_MAX_DATA_PER_TXD);
i++;
if (i == tx_ring->count)
tx_desc++;
if (i == tx_ring->count) {
tx_desc = IXGBEVF_TX_DESC(tx_ring, 0);
i = 0;
}
if (total == 0)
break;
}
if (i == 0)
i = tx_ring->count - 1;
else
i = i - 1;
tx_ring->tx_buffer_info[i].skb = skb;
dma += IXGBE_MAX_DATA_PER_TXD;
size -= IXGBE_MAX_DATA_PER_TXD;
return count;
tx_desc->read.buffer_addr = cpu_to_le64(dma);
tx_desc->read.olinfo_status = 0;
}
dma_error:
dev_err(tx_ring->dev, "TX DMA map failed\n");
if (likely(!data_len))
break;
/* clear timestamp and dma mappings for failed tx_buffer_info map */
tx_buffer_info->dma = 0;
count--;
tx_desc->read.cmd_type_len = cmd_type | cpu_to_le32(size);
/* clear timestamp and dma mappings for remaining portion of packet */
while (count >= 0) {
count--;
i--;
if (i < 0)
i += tx_ring->count;
tx_buffer_info = &tx_ring->tx_buffer_info[i];
ixgbevf_unmap_and_free_tx_resource(tx_ring, tx_buffer_info);
i++;
tx_desc++;
if (i == tx_ring->count) {
tx_desc = IXGBEVF_TX_DESC(tx_ring, 0);
i = 0;
}
return count;
}
static void ixgbevf_tx_queue(struct ixgbevf_ring *tx_ring, int tx_flags,
int count, unsigned int first, u32 paylen,
u8 hdr_len)
{
union ixgbe_adv_tx_desc *tx_desc = NULL;
struct ixgbevf_tx_buffer *tx_buffer_info;
u32 olinfo_status = 0, cmd_type_len = 0;
unsigned int i;
u32 txd_cmd = IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS | IXGBE_TXD_CMD_IFCS;
size = skb_frag_size(frag);
data_len -= size;
cmd_type_len |= IXGBE_ADVTXD_DTYP_DATA;
dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size,
DMA_TO_DEVICE);
if (dma_mapping_error(tx_ring->dev, dma))
goto dma_error;
cmd_type_len |= IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT;
tx_buffer = &tx_ring->tx_buffer_info[i];
dma_unmap_len_set(tx_buffer, len, size);
dma_unmap_addr_set(tx_buffer, dma, dma);
if (tx_flags & IXGBE_TX_FLAGS_VLAN)
cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE;
tx_desc->read.buffer_addr = cpu_to_le64(dma);
tx_desc->read.olinfo_status = 0;
if (tx_flags & IXGBE_TX_FLAGS_CSUM)
olinfo_status |= IXGBE_ADVTXD_POPTS_TXSM;
frag++;
}
if (tx_flags & IXGBE_TX_FLAGS_TSO) {
cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE;
/* write last descriptor with RS and EOP bits */
cmd_type |= cpu_to_le32(size) | cpu_to_le32(IXGBE_TXD_CMD);
tx_desc->read.cmd_type_len = cmd_type;
/* use index 1 context for tso */
olinfo_status |= (1 << IXGBE_ADVTXD_IDX_SHIFT);
if (tx_flags & IXGBE_TX_FLAGS_IPV4)
olinfo_status |= IXGBE_ADVTXD_POPTS_IXSM;
}
/* set the timestamp */
first->time_stamp = jiffies;
/*
* Check Context must be set if Tx switch is enabled, which it
* always is for case where virtual functions are running
/* Force memory writes to complete before letting h/w know there
* are new descriptors to fetch. (Only applicable for weak-ordered
* memory model archs, such as IA-64).
*
* We also need this memory barrier (wmb) to make certain all of the
* status bits have been updated before next_to_watch is written.
*/
olinfo_status |= IXGBE_ADVTXD_CC;
wmb();
olinfo_status |= ((paylen - hdr_len) << IXGBE_ADVTXD_PAYLEN_SHIFT);
/* set next_to_watch value indicating a packet is present */
first->next_to_watch = tx_desc;
i = tx_ring->next_to_use;
while (count--) {
tx_buffer_info = &tx_ring->tx_buffer_info[i];
tx_desc = IXGBEVF_TX_DESC(tx_ring, i);
tx_desc->read.buffer_addr = cpu_to_le64(tx_buffer_info->dma);
tx_desc->read.cmd_type_len =
cpu_to_le32(cmd_type_len | tx_buffer_info->length);
tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
i++;
if (i == tx_ring->count)
i = 0;
}
tx_desc->read.cmd_type_len |= cpu_to_le32(txd_cmd);
tx_ring->next_to_use = i;
tx_ring->tx_buffer_info[first].time_stamp = jiffies;
/* notify HW of packet */
writel(i, tx_ring->tail);
/* Force memory writes to complete before letting h/w
* know there are new descriptors to fetch. (Only
* applicable for weak-ordered memory model archs,
* such as IA-64).
*/
wmb();
return;
dma_error:
dev_err(tx_ring->dev, "TX DMA map failed\n");
/* clear dma mappings for failed tx_buffer_info map */
for (;;) {
tx_buffer = &tx_ring->tx_buffer_info[i];
ixgbevf_unmap_and_free_tx_resource(tx_ring, tx_buffer);
if (tx_buffer == first)
break;
if (i == 0)
i = tx_ring->count;
i--;
}
tx_ring->tx_buffer_info[first].next_to_watch = tx_desc;
tx_ring->next_to_use = i;
}
static int __ixgbevf_maybe_stop_tx(struct ixgbevf_ring *tx_ring, int size)
{
struct ixgbevf_adapter *adapter = netdev_priv(tx_ring->netdev);
netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);
/* Herbert's original patch had:
* smp_mb__after_netif_stop_queue();
......@@ -3086,7 +3095,8 @@ static int __ixgbevf_maybe_stop_tx(struct ixgbevf_ring *tx_ring, int size)
/* A reprieve! - use start_queue because it doesn't call schedule */
netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index);
++adapter->restart_queue;
++tx_ring->tx_stats.restart_queue;
return 0;
}
......@@ -3100,22 +3110,23 @@ static int ixgbevf_maybe_stop_tx(struct ixgbevf_ring *tx_ring, int size)
static int ixgbevf_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
{
struct ixgbevf_adapter *adapter = netdev_priv(netdev);
struct ixgbevf_tx_buffer *first;
struct ixgbevf_ring *tx_ring;
unsigned int first;
unsigned int tx_flags = 0;
u8 hdr_len = 0;
int r_idx = 0, tso;
int tso;
u32 tx_flags = 0;
u16 count = TXD_USE_COUNT(skb_headlen(skb));
#if PAGE_SIZE > IXGBE_MAX_DATA_PER_TXD
unsigned short f;
#endif
u8 hdr_len = 0;
u8 *dst_mac = skb_header_pointer(skb, 0, 0, NULL);
if (!dst_mac || is_link_local_ether_addr(dst_mac)) {
dev_kfree_skb(skb);
return NETDEV_TX_OK;
}
tx_ring = adapter->tx_ring[r_idx];
tx_ring = adapter->tx_ring[skb->queue_mapping];
/*
* need: 1 descriptor per page * PAGE_SIZE/IXGBE_MAX_DATA_PER_TXD,
......@@ -3131,38 +3142,41 @@ static int ixgbevf_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
count += skb_shinfo(skb)->nr_frags;
#endif
if (ixgbevf_maybe_stop_tx(tx_ring, count + 3)) {
adapter->tx_busy++;
tx_ring->tx_stats.tx_busy++;
return NETDEV_TX_BUSY;
}
/* record the location of the first descriptor for this packet */
first = &tx_ring->tx_buffer_info[tx_ring->next_to_use];
first->skb = skb;
first->bytecount = skb->len;
first->gso_segs = 1;
if (vlan_tx_tag_present(skb)) {
tx_flags |= vlan_tx_tag_get(skb);
tx_flags <<= IXGBE_TX_FLAGS_VLAN_SHIFT;
tx_flags |= IXGBE_TX_FLAGS_VLAN;
}
first = tx_ring->next_to_use;
/* record initial flags and protocol */
first->tx_flags = tx_flags;
first->protocol = vlan_get_protocol(skb);
if (skb->protocol == htons(ETH_P_IP))
tx_flags |= IXGBE_TX_FLAGS_IPV4;
tso = ixgbevf_tso(tx_ring, skb, tx_flags, &hdr_len);
if (tso < 0) {
dev_kfree_skb_any(skb);
return NETDEV_TX_OK;
}
tso = ixgbevf_tso(tx_ring, first, &hdr_len);
if (tso < 0)
goto out_drop;
else
ixgbevf_tx_csum(tx_ring, first);
if (tso)
tx_flags |= IXGBE_TX_FLAGS_TSO | IXGBE_TX_FLAGS_CSUM;
else if (ixgbevf_tx_csum(tx_ring, skb, tx_flags))
tx_flags |= IXGBE_TX_FLAGS_CSUM;
ixgbevf_tx_map(tx_ring, first, hdr_len);
ixgbevf_tx_queue(tx_ring, tx_flags,
ixgbevf_tx_map(tx_ring, skb, tx_flags),
first, skb->len, hdr_len);
ixgbevf_maybe_stop_tx(tx_ring, DESC_NEEDED);
writel(tx_ring->next_to_use, tx_ring->tail);
return NETDEV_TX_OK;
ixgbevf_maybe_stop_tx(tx_ring, DESC_NEEDED);
out_drop:
dev_kfree_skb_any(first->skb);
first->skb = NULL;
return NETDEV_TX_OK;
}
......@@ -3331,8 +3345,8 @@ static struct rtnl_link_stats64 *ixgbevf_get_stats(struct net_device *netdev,
ring = adapter->rx_ring[i];
do {
start = u64_stats_fetch_begin_bh(&ring->syncp);
bytes = ring->total_bytes;
packets = ring->total_packets;
bytes = ring->stats.bytes;
packets = ring->stats.packets;
} while (u64_stats_fetch_retry_bh(&ring->syncp, start));
stats->rx_bytes += bytes;
stats->rx_packets += packets;
......@@ -3342,8 +3356,8 @@ static struct rtnl_link_stats64 *ixgbevf_get_stats(struct net_device *netdev,
ring = adapter->tx_ring[i];
do {
start = u64_stats_fetch_begin_bh(&ring->syncp);
bytes = ring->total_bytes;
packets = ring->total_packets;
bytes = ring->stats.bytes;
packets = ring->stats.packets;
} while (u64_stats_fetch_retry_bh(&ring->syncp, start));
stats->tx_bytes += bytes;
stats->tx_packets += packets;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7