Skip to content

L
linux
Commit 5a976888 authored by Dave Jiang's avatar Dave Jiang Committed by Vinod Koul
Browse files
Options

dmaengine: ioatdma: clean up local dma channel data structure 

Kill the common ioatdma channel structure and everything that is not
dma_chan to be ioat_dma_chan. Since we don't have to worry about v1
and v2 ioatdma anymore this makes it much cleaner and obvious for
maintenance.
Signed-off-by: default avatarDave Jiang <dave.jiang@intel.com>
Acked-by: default avatarDan Williams <dan.j.williams@intel.com>
Signed-off-by: default avatarVinod Koul <vinod.koul@intel.com>
parent 7f832645
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 470 additions and 475 deletions
drivers/dma/ioat/dma.c
View file @ 5a976888
...@@ -50,7 +50,7 @@ MODULE_PARM_DESC(ioat_pending_level, ...@@ -50,7 +50,7 @@ MODULE_PARM_DESC(ioat_pending_level,
static irqreturn_t ioat_dma_do_interrupt(int irq, void *data) static irqreturn_t ioat_dma_do_interrupt(int irq, void *data)
{ {
struct ioatdma_device *instance = data; struct ioatdma_device *instance = data;
struct ioat_chan_common *chan; struct ioatdma_chan *ioat_chan;
unsigned long attnstatus; unsigned long attnstatus;
int bit; int bit;
u8 intrctrl; u8 intrctrl;
...@@ -67,9 +67,9 @@ static irqreturn_t ioat_dma_do_interrupt(int irq, void *data) ...@@ -67,9 +67,9 @@ static irqreturn_t ioat_dma_do_interrupt(int irq, void *data)
attnstatus = readl(instance->reg_base + IOAT_ATTNSTATUS_OFFSET); attnstatus = readl(instance->reg_base + IOAT_ATTNSTATUS_OFFSET);
for_each_set_bit(bit, &attnstatus, BITS_PER_LONG) { for_each_set_bit(bit, &attnstatus, BITS_PER_LONG) {
chan = ioat_chan_by_index(instance, bit); ioat_chan = ioat_chan_by_index(instance, bit);
if (test_bit(IOAT_RUN, &chan->state)) if (test_bit(IOAT_RUN, &ioat_chan->state))
tasklet_schedule(&chan->cleanup_task); tasklet_schedule(&ioat_chan->cleanup_task);
} }
writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET); writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET);
...@@ -83,45 +83,47 @@ static irqreturn_t ioat_dma_do_interrupt(int irq, void *data) ...@@ -83,45 +83,47 @@ static irqreturn_t ioat_dma_do_interrupt(int irq, void *data)
*/ */
static irqreturn_t ioat_dma_do_interrupt_msix(int irq, void *data) static irqreturn_t ioat_dma_do_interrupt_msix(int irq, void *data)
{ {
struct ioat_chan_common *chan = data; struct ioatdma_chan *ioat_chan = data;
if (test_bit(IOAT_RUN, &chan->state)) if (test_bit(IOAT_RUN, &ioat_chan->state))
tasklet_schedule(&chan->cleanup_task); tasklet_schedule(&ioat_chan->cleanup_task);
return IRQ_HANDLED; return IRQ_HANDLED;
} }
/* common channel initialization */ /* common channel initialization */
void ioat_init_channel(struct ioatdma_device *device, struct ioat_chan_common *chan, int idx) void
ioat_init_channel(struct ioatdma_device *device, struct ioatdma_chan *ioat_chan,
int idx)
{ {
struct dma_device *dma = &device->common; struct dma_device *dma = &device->common;
struct dma_chan *c = &chan->common; struct dma_chan *c = &ioat_chan->dma_chan;
unsigned long data = (unsigned long) c; unsigned long data = (unsigned long) c;
chan->device = device; ioat_chan->device = device;
chan->reg_base = device->reg_base + (0x80 * (idx + 1)); ioat_chan->reg_base = device->reg_base + (0x80 * (idx + 1));
spin_lock_init(&chan->cleanup_lock); spin_lock_init(&ioat_chan->cleanup_lock);
chan->common.device = dma; ioat_chan->dma_chan.device = dma;
dma_cookie_init(&chan->common); dma_cookie_init(&ioat_chan->dma_chan);
list_add_tail(&chan->common.device_node, &dma->channels); list_add_tail(&ioat_chan->dma_chan.device_node, &dma->channels);
device->idx[idx] = chan; device->idx[idx] = ioat_chan;
init_timer(&chan->timer); init_timer(&ioat_chan->timer);
chan->timer.function = device->timer_fn; ioat_chan->timer.function = device->timer_fn;
chan->timer.data = data; ioat_chan->timer.data = data;
tasklet_init(&chan->cleanup_task, device->cleanup_fn, data); tasklet_init(&ioat_chan->cleanup_task, device->cleanup_fn, data);
} }
void ioat_stop(struct ioat_chan_common *chan) void ioat_stop(struct ioatdma_chan *ioat_chan)
{ {
struct ioatdma_device *device = chan->device; struct ioatdma_device *device = ioat_chan->device;
struct pci_dev *pdev = device->pdev; struct pci_dev *pdev = device->pdev;
int chan_id = chan_num(chan); int chan_id = chan_num(ioat_chan);
struct msix_entry *msix; struct msix_entry *msix;
/* 1/ stop irq from firing tasklets /* 1/ stop irq from firing tasklets
* 2/ stop the tasklet from re-arming irqs * 2/ stop the tasklet from re-arming irqs
*/ */
clear_bit(IOAT_RUN, &chan->state); clear_bit(IOAT_RUN, &ioat_chan->state);
/* flush inflight interrupts */ /* flush inflight interrupts */
switch (device->irq_mode) { switch (device->irq_mode) {
...@@ -138,29 +140,30 @@ void ioat_stop(struct ioat_chan_common *chan) ...@@ -138,29 +140,30 @@ void ioat_stop(struct ioat_chan_common *chan)
} }
/* flush inflight timers */ /* flush inflight timers */
del_timer_sync(&chan->timer); del_timer_sync(&ioat_chan->timer);
/* flush inflight tasklet runs */ /* flush inflight tasklet runs */
tasklet_kill(&chan->cleanup_task); tasklet_kill(&ioat_chan->cleanup_task);
/* final cleanup now that everything is quiesced and can't re-arm */ /* final cleanup now that everything is quiesced and can't re-arm */
device->cleanup_fn((unsigned long) &chan->common); device->cleanup_fn((unsigned long)&ioat_chan->dma_chan);
} }
dma_addr_t ioat_get_current_completion(struct ioat_chan_common *chan) dma_addr_t ioat_get_current_completion(struct ioatdma_chan *ioat_chan)
{ {
dma_addr_t phys_complete; dma_addr_t phys_complete;
u64 completion; u64 completion;
completion = *chan->completion; completion = *ioat_chan->completion;
phys_complete = ioat_chansts_to_addr(completion); phys_complete = ioat_chansts_to_addr(completion);
dev_dbg(to_dev(chan), "%s: phys_complete: %#llx\n", __func__, dev_dbg(to_dev(ioat_chan), "%s: phys_complete: %#llx\n", __func__,
(unsigned long long) phys_complete); (unsigned long long) phys_complete);
if (is_ioat_halted(completion)) { if (is_ioat_halted(completion)) {
u32 chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET); u32 chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
dev_err(to_dev(chan), "Channel halted, chanerr = %x\n",
dev_err(to_dev(ioat_chan), "Channel halted, chanerr = %x\n",
chanerr); chanerr);
/* TODO do something to salvage the situation */ /* TODO do something to salvage the situation */
...@@ -169,14 +172,14 @@ dma_addr_t ioat_get_current_completion(struct ioat_chan_common *chan) ...@@ -169,14 +172,14 @@ dma_addr_t ioat_get_current_completion(struct ioat_chan_common *chan)
return phys_complete; return phys_complete;
} }
bool ioat_cleanup_preamble(struct ioat_chan_common *chan, bool ioat_cleanup_preamble(struct ioatdma_chan *ioat_chan,
dma_addr_t *phys_complete) dma_addr_t *phys_complete)
{ {
*phys_complete = ioat_get_current_completion(chan); *phys_complete = ioat_get_current_completion(ioat_chan);
if (*phys_complete == chan->last_completion) if (*phys_complete == ioat_chan->last_completion)
return false; return false;
clear_bit(IOAT_COMPLETION_ACK, &chan->state); clear_bit(IOAT_COMPLETION_ACK, &ioat_chan->state);
mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT); mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT);
return true; return true;
} }
...@@ -185,8 +188,8 @@ enum dma_status ...@@ -185,8 +188,8 @@ enum dma_status
ioat_dma_tx_status(struct dma_chan *c, dma_cookie_t cookie, ioat_dma_tx_status(struct dma_chan *c, dma_cookie_t cookie,
struct dma_tx_state *txstate) struct dma_tx_state *txstate)
{ {
struct ioat_chan_common *chan = to_chan_common(c); struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
struct ioatdma_device *device = chan->device; struct ioatdma_device *device = ioat_chan->device;
enum dma_status ret; enum dma_status ret;
ret = dma_cookie_status(c, cookie, txstate); ret = dma_cookie_status(c, cookie, txstate);
...@@ -322,7 +325,7 @@ MODULE_PARM_DESC(ioat_interrupt_style, ...@@ -322,7 +325,7 @@ MODULE_PARM_DESC(ioat_interrupt_style,
*/ */
int ioat_dma_setup_interrupts(struct ioatdma_device *device) int ioat_dma_setup_interrupts(struct ioatdma_device *device)
{ {
struct ioat_chan_common *chan; struct ioatdma_chan *ioat_chan;
struct pci_dev *pdev = device->pdev; struct pci_dev *pdev = device->pdev;
struct device *dev = &pdev->dev; struct device *dev = &pdev->dev;
struct msix_entry *msix; struct msix_entry *msix;
...@@ -351,15 +354,15 @@ int ioat_dma_setup_interrupts(struct ioatdma_device *device) ...@@ -351,15 +354,15 @@ int ioat_dma_setup_interrupts(struct ioatdma_device *device)
for (i = 0; i < msixcnt; i++) { for (i = 0; i < msixcnt; i++) {
msix = &device->msix_entries[i]; msix = &device->msix_entries[i];
chan = ioat_chan_by_index(device, i); ioat_chan = ioat_chan_by_index(device, i);
err = devm_request_irq(dev, msix->vector, err = devm_request_irq(dev, msix->vector,
ioat_dma_do_interrupt_msix, 0, ioat_dma_do_interrupt_msix, 0,
"ioat-msix", chan); "ioat-msix", ioat_chan);
if (err) { if (err) {
for (j = 0; j < i; j++) { for (j = 0; j < i; j++) {
msix = &device->msix_entries[j]; msix = &device->msix_entries[j];
chan = ioat_chan_by_index(device, j); ioat_chan = ioat_chan_by_index(device, j);
devm_free_irq(dev, msix->vector, chan); devm_free_irq(dev, msix->vector, ioat_chan);
} }
goto msi; goto msi;
} }
...@@ -507,14 +510,14 @@ static ssize_t ...@@ -507,14 +510,14 @@ static ssize_t
ioat_attr_show(struct kobject *kobj, struct attribute *attr, char *page) ioat_attr_show(struct kobject *kobj, struct attribute *attr, char *page)
{ {
struct ioat_sysfs_entry *entry; struct ioat_sysfs_entry *entry;
struct ioat_chan_common *chan; struct ioatdma_chan *ioat_chan;
entry = container_of(attr, struct ioat_sysfs_entry, attr); entry = container_of(attr, struct ioat_sysfs_entry, attr);
chan = container_of(kobj, struct ioat_chan_common, kobj); ioat_chan = container_of(kobj, struct ioatdma_chan, kobj);
if (!entry->show) if (!entry->show)
return -EIO; return -EIO;
return entry->show(&chan->common, page); return entry->show(&ioat_chan->dma_chan, page);
} }
const struct sysfs_ops ioat_sysfs_ops = { const struct sysfs_ops ioat_sysfs_ops = {
...@@ -527,16 +530,17 @@ void ioat_kobject_add(struct ioatdma_device *device, struct kobj_type *type) ...@@ -527,16 +530,17 @@ void ioat_kobject_add(struct ioatdma_device *device, struct kobj_type *type)
struct dma_chan *c; struct dma_chan *c;
list_for_each_entry(c, &dma->channels, device_node) { list_for_each_entry(c, &dma->channels, device_node) {
struct ioat_chan_common *chan = to_chan_common(c); struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
struct kobject *parent = &c->dev->device.kobj; struct kobject *parent = &c->dev->device.kobj;
int err; int err;
err = kobject_init_and_add(&chan->kobj, type, parent, "quickdata"); err = kobject_init_and_add(&ioat_chan->kobj, type,
parent, "quickdata");
if (err) { if (err) {
dev_warn(to_dev(chan), dev_warn(to_dev(ioat_chan),
"sysfs init error (%d), continuing...\n", err); "sysfs init error (%d), continuing...\n", err);
kobject_put(&chan->kobj); kobject_put(&ioat_chan->kobj);
set_bit(IOAT_KOBJ_INIT_FAIL, &chan->state); set_bit(IOAT_KOBJ_INIT_FAIL, &ioat_chan->state);
} }
} }
} }
...@@ -547,11 +551,11 @@ void ioat_kobject_del(struct ioatdma_device *device) ...@@ -547,11 +551,11 @@ void ioat_kobject_del(struct ioatdma_device *device)
struct dma_chan *c; struct dma_chan *c;
list_for_each_entry(c, &dma->channels, device_node) { list_for_each_entry(c, &dma->channels, device_node) {
struct ioat_chan_common *chan = to_chan_common(c); struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
if (!test_bit(IOAT_KOBJ_INIT_FAIL, &chan->state)) { if (!test_bit(IOAT_KOBJ_INIT_FAIL, &ioat_chan->state)) {
kobject_del(&chan->kobj); kobject_del(&ioat_chan->kobj);
kobject_put(&chan->kobj); kobject_put(&ioat_chan->kobj);
} }
} }
} }
......
drivers/dma/ioat/dma.h
View file @ 5a976888
...@@ -78,20 +78,20 @@ struct ioatdma_device { ...@@ -78,20 +78,20 @@ struct ioatdma_device {
struct dma_device common; struct dma_device common;
u8 version; u8 version;
struct msix_entry msix_entries[4]; struct msix_entry msix_entries[4];
struct ioat_chan_common *idx[4]; struct ioatdma_chan *idx[4];
struct dca_provider *dca; struct dca_provider *dca;
enum ioat_irq_mode irq_mode; enum ioat_irq_mode irq_mode;
u32 cap; u32 cap;
void (*intr_quirk)(struct ioatdma_device *device); void (*intr_quirk)(struct ioatdma_device *device);
int (*enumerate_channels)(struct ioatdma_device *device); int (*enumerate_channels)(struct ioatdma_device *device);
int (*reset_hw)(struct ioat_chan_common *chan); int (*reset_hw)(struct ioatdma_chan *ioat_chan);
void (*cleanup_fn)(unsigned long data); void (*cleanup_fn)(unsigned long data);
void (*timer_fn)(unsigned long data); void (*timer_fn)(unsigned long data);
int (*self_test)(struct ioatdma_device *device); int (*self_test)(struct ioatdma_device *device);
}; };
struct ioat_chan_common { struct ioatdma_chan {
struct dma_chan common; struct dma_chan dma_chan;
void __iomem *reg_base; void __iomem *reg_base;
dma_addr_t last_completion; dma_addr_t last_completion;
spinlock_t cleanup_lock; spinlock_t cleanup_lock;
...@@ -112,6 +112,27 @@ struct ioat_chan_common { ...@@ -112,6 +112,27 @@ struct ioat_chan_common {
u64 *completion; u64 *completion;
struct tasklet_struct cleanup_task; struct tasklet_struct cleanup_task;
struct kobject kobj; struct kobject kobj;
/* ioat v2 / v3 channel attributes
* @xfercap_log; log2 of channel max transfer length (for fast division)
* @head: allocated index
* @issued: hardware notification point
* @tail: cleanup index
* @dmacount: identical to 'head' except for occasionally resetting to zero
* @alloc_order: log2 of the number of allocated descriptors
* @produce: number of descriptors to produce at submit time
* @ring: software ring buffer implementation of hardware ring
* @prep_lock: serializes descriptor preparation (producers)
*/
size_t xfercap_log;
u16 head;
u16 issued;
u16 tail;
u16 dmacount;
u16 alloc_order;
u16 produce;
struct ioat_ring_ent **ring;
spinlock_t prep_lock;
}; };
struct ioat_sysfs_entry { struct ioat_sysfs_entry {
...@@ -133,11 +154,13 @@ struct ioat_sed_ent { ...@@ -133,11 +154,13 @@ struct ioat_sed_ent {
unsigned int hw_pool; unsigned int hw_pool;
}; };
static inline struct ioat_chan_common *to_chan_common(struct dma_chan *c) static inline struct ioatdma_chan *to_ioat_chan(struct dma_chan *c)
{ {
return container_of(c, struct ioat_chan_common, common); return container_of(c, struct ioatdma_chan, dma_chan);
} }
/* wrapper around hardware descriptor format + additional software fields */ /* wrapper around hardware descriptor format + additional software fields */
#ifdef DEBUG #ifdef DEBUG
...@@ -149,10 +172,10 @@ static inline struct ioat_chan_common *to_chan_common(struct dma_chan *c) ...@@ -149,10 +172,10 @@ static inline struct ioat_chan_common *to_chan_common(struct dma_chan *c)
#endif #endif
static inline void static inline void
__dump_desc_dbg(struct ioat_chan_common *chan, struct ioat_dma_descriptor *hw, __dump_desc_dbg(struct ioatdma_chan *ioat_chan, struct ioat_dma_descriptor *hw,
struct dma_async_tx_descriptor *tx, int id) struct dma_async_tx_descriptor *tx, int id)
{ {
struct device *dev = to_dev(chan); struct device *dev = to_dev(ioat_chan);
dev_dbg(dev, "desc[%d]: (%#llx->%#llx) cookie: %d flags: %#x" dev_dbg(dev, "desc[%d]: (%#llx->%#llx) cookie: %d flags: %#x"
" ctl: %#10.8x (op: %#x int_en: %d compl: %d)\n", id, " ctl: %#10.8x (op: %#x int_en: %d compl: %d)\n", id,
...@@ -162,25 +185,25 @@ __dump_desc_dbg(struct ioat_chan_common *chan, struct ioat_dma_descriptor *hw, ...@@ -162,25 +185,25 @@ __dump_desc_dbg(struct ioat_chan_common *chan, struct ioat_dma_descriptor *hw,
} }
#define dump_desc_dbg(c, d) \ #define dump_desc_dbg(c, d) \
({ if (d) __dump_desc_dbg(&c->base, d->hw, &d->txd, desc_id(d)); 0; }) ({ if (d) __dump_desc_dbg(c, d->hw, &d->txd, desc_id(d)); 0; })
static inline struct ioat_chan_common * static inline struct ioatdma_chan *
ioat_chan_by_index(struct ioatdma_device *device, int index) ioat_chan_by_index(struct ioatdma_device *device, int index)
{ {
return device->idx[index]; return device->idx[index];
} }
static inline u64 ioat_chansts_32(struct ioat_chan_common *chan) static inline u64 ioat_chansts_32(struct ioatdma_chan *ioat_chan)
{ {
u8 ver = chan->device->version; u8 ver = ioat_chan->device->version;
u64 status; u64 status;
u32 status_lo; u32 status_lo;
/* We need to read the low address first as this causes the /* We need to read the low address first as this causes the
* chipset to latch the upper bits for the subsequent read * chipset to latch the upper bits for the subsequent read
*/ */
status_lo = readl(chan->reg_base + IOAT_CHANSTS_OFFSET_LOW(ver)); status_lo = readl(ioat_chan->reg_base + IOAT_CHANSTS_OFFSET_LOW(ver));
status = readl(chan->reg_base + IOAT_CHANSTS_OFFSET_HIGH(ver)); status = readl(ioat_chan->reg_base + IOAT_CHANSTS_OFFSET_HIGH(ver));
status <<= 32; status <<= 32;
status |= status_lo; status |= status_lo;
...@@ -189,16 +212,16 @@ static inline u64 ioat_chansts_32(struct ioat_chan_common *chan) ...@@ -189,16 +212,16 @@ static inline u64 ioat_chansts_32(struct ioat_chan_common *chan)
#if BITS_PER_LONG == 64 #if BITS_PER_LONG == 64
static inline u64 ioat_chansts(struct ioat_chan_common *chan) static inline u64 ioat_chansts(struct ioatdma_chan *ioat_chan)
{ {
u8 ver = chan->device->version; u8 ver = ioat_chan->device->version;
u64 status; u64 status;
/* With IOAT v3.3 the status register is 64bit. */ /* With IOAT v3.3 the status register is 64bit. */
if (ver >= IOAT_VER_3_3) if (ver >= IOAT_VER_3_3)
status = readq(chan->reg_base + IOAT_CHANSTS_OFFSET(ver)); status = readq(ioat_chan->reg_base + IOAT_CHANSTS_OFFSET(ver));
else else
status = ioat_chansts_32(chan); status = ioat_chansts_32(ioat_chan);
return status; return status;
} }
...@@ -212,31 +235,33 @@ static inline u64 ioat_chansts_to_addr(u64 status) ...@@ -212,31 +235,33 @@ static inline u64 ioat_chansts_to_addr(u64 status)
return status & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR; return status & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR;
} }
static inline u32 ioat_chanerr(struct ioat_chan_common *chan) static inline u32 ioat_chanerr(struct ioatdma_chan *ioat_chan)
{ {
return readl(chan->reg_base + IOAT_CHANERR_OFFSET); return readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
} }
static inline void ioat_suspend(struct ioat_chan_common *chan) static inline void ioat_suspend(struct ioatdma_chan *ioat_chan)
{ {
u8 ver = chan->device->version; u8 ver = ioat_chan->device->version;
writeb(IOAT_CHANCMD_SUSPEND, chan->reg_base + IOAT_CHANCMD_OFFSET(ver)); writeb(IOAT_CHANCMD_SUSPEND,
ioat_chan->reg_base + IOAT_CHANCMD_OFFSET(ver));
} }
static inline void ioat_reset(struct ioat_chan_common *chan) static inline void ioat_reset(struct ioatdma_chan *ioat_chan)
{ {
u8 ver = chan->device->version; u8 ver = ioat_chan->device->version;
writeb(IOAT_CHANCMD_RESET, chan->reg_base + IOAT_CHANCMD_OFFSET(ver)); writeb(IOAT_CHANCMD_RESET,
ioat_chan->reg_base + IOAT_CHANCMD_OFFSET(ver));
} }
static inline bool ioat_reset_pending(struct ioat_chan_common *chan) static inline bool ioat_reset_pending(struct ioatdma_chan *ioat_chan)
{ {
u8 ver = chan->device->version; u8 ver = ioat_chan->device->version;
u8 cmd; u8 cmd;
cmd = readb(chan->reg_base + IOAT_CHANCMD_OFFSET(ver)); cmd = readb(ioat_chan->reg_base + IOAT_CHANCMD_OFFSET(ver));
return (cmd & IOAT_CHANCMD_RESET) == IOAT_CHANCMD_RESET; return (cmd & IOAT_CHANCMD_RESET) == IOAT_CHANCMD_RESET;
} }
...@@ -272,15 +297,15 @@ int ioat_dma_self_test(struct ioatdma_device *device); ...@@ -272,15 +297,15 @@ int ioat_dma_self_test(struct ioatdma_device *device);
void ioat_dma_remove(struct ioatdma_device *device); void ioat_dma_remove(struct ioatdma_device *device);
struct dca_provider *ioat_dca_init(struct pci_dev *pdev, void __iomem *iobase); struct dca_provider *ioat_dca_init(struct pci_dev *pdev, void __iomem *iobase);
void ioat_init_channel(struct ioatdma_device *device, void ioat_init_channel(struct ioatdma_device *device,
struct ioat_chan_common *chan, int idx); struct ioatdma_chan *ioat_chan, int idx);
enum dma_status ioat_dma_tx_status(struct dma_chan *c, dma_cookie_t cookie, enum dma_status ioat_dma_tx_status(struct dma_chan *c, dma_cookie_t cookie,
struct dma_tx_state *txstate); struct dma_tx_state *txstate);
bool ioat_cleanup_preamble(struct ioat_chan_common *chan, bool ioat_cleanup_preamble(struct ioatdma_chan *ioat_chan,
dma_addr_t *phys_complete); dma_addr_t *phys_complete);
void ioat_kobject_add(struct ioatdma_device *device, struct kobj_type *type); void ioat_kobject_add(struct ioatdma_device *device, struct kobj_type *type);
void ioat_kobject_del(struct ioatdma_device *device); void ioat_kobject_del(struct ioatdma_device *device);
int ioat_dma_setup_interrupts(struct ioatdma_device *device); int ioat_dma_setup_interrupts(struct ioatdma_device *device);
void ioat_stop(struct ioat_chan_common *chan); void ioat_stop(struct ioatdma_chan *ioat_chan);
extern const struct sysfs_ops ioat_sysfs_ops; extern const struct sysfs_ops ioat_sysfs_ops;
extern struct ioat_sysfs_entry ioat_version_attr; extern struct ioat_sysfs_entry ioat_version_attr;
extern struct ioat_sysfs_entry ioat_cap_attr; extern struct ioat_sysfs_entry ioat_cap_attr;
......
drivers/dma/ioat/dma_v2.c
View file @ 5a976888
This diff is collapsed.
drivers/dma/ioat/dma_v2.h
View file @ 5a976888
...@@ -38,65 +38,36 @@ extern int ioat_ring_alloc_order; ...@@ -38,65 +38,36 @@ extern int ioat_ring_alloc_order;
#define ioat_get_max_alloc_order() \ #define ioat_get_max_alloc_order() \
(min(ioat_ring_max_alloc_order, IOAT_MAX_ORDER)) (min(ioat_ring_max_alloc_order, IOAT_MAX_ORDER))
/* struct ioat2_dma_chan - ioat v2 / v3 channel attributes static inline u32 ioat2_ring_size(struct ioatdma_chan *ioat_chan)
* @base: common ioat channel parameters
* @xfercap_log; log2 of channel max transfer length (for fast division)
* @head: allocated index
* @issued: hardware notification point
* @tail: cleanup index
* @dmacount: identical to 'head' except for occasionally resetting to zero
* @alloc_order: log2 of the number of allocated descriptors
* @produce: number of descriptors to produce at submit time
* @ring: software ring buffer implementation of hardware ring
* @prep_lock: serializes descriptor preparation (producers)
*/
struct ioat2_dma_chan {
struct ioat_chan_common base;
size_t xfercap_log;
u16 head;
u16 issued;
u16 tail;
u16 dmacount;
u16 alloc_order;
u16 produce;
struct ioat_ring_ent **ring;
spinlock_t prep_lock;
};
static inline struct ioat2_dma_chan *to_ioat2_chan(struct dma_chan *c)
{
struct ioat_chan_common *chan = to_chan_common(c);
return container_of(chan, struct ioat2_dma_chan, base);
}
static inline u32 ioat2_ring_size(struct ioat2_dma_chan *ioat)
{ {
return 1 << ioat->alloc_order; return 1 << ioat_chan->alloc_order;
} }
/* count of descriptors in flight with the engine */ /* count of descriptors in flight with the engine */
static inline u16 ioat2_ring_active(struct ioat2_dma_chan *ioat) static inline u16 ioat2_ring_active(struct ioatdma_chan *ioat_chan)
{ {
return CIRC_CNT(ioat->head, ioat->tail, ioat2_ring_size(ioat)); return CIRC_CNT(ioat_chan->head, ioat_chan->tail,
ioat2_ring_size(ioat_chan));
} }
/* count of descriptors pending submission to hardware */ /* count of descriptors pending submission to hardware */
static inline u16 ioat2_ring_pending(struct ioat2_dma_chan *ioat) static inline u16 ioat2_ring_pending(struct ioatdma_chan *ioat_chan)
{ {
return CIRC_CNT(ioat->head, ioat->issued, ioat2_ring_size(ioat)); return CIRC_CNT(ioat_chan->head, ioat_chan->issued,
ioat2_ring_size(ioat_chan));
} }
static inline u32 ioat2_ring_space(struct ioat2_dma_chan *ioat) static inline u32 ioat2_ring_space(struct ioatdma_chan *ioat_chan)
{ {
return ioat2_ring_size(ioat) - ioat2_ring_active(ioat); return ioat2_ring_size(ioat_chan) - ioat2_ring_active(ioat_chan);
} }
static inline u16 ioat2_xferlen_to_descs(struct ioat2_dma_chan *ioat, size_t len) static inline u16
ioat2_xferlen_to_descs(struct ioatdma_chan *ioat_chan, size_t len)
{ {
u16 num_descs = len >> ioat->xfercap_log; u16 num_descs = len >> ioat_chan->xfercap_log;
num_descs += !!(len & ((1 << ioat->xfercap_log) - 1)); num_descs += !!(len & ((1 << ioat_chan->xfercap_log) - 1));
return num_descs; return num_descs;
} }
...@@ -136,25 +107,24 @@ struct ioat_ring_ent { ...@@ -136,25 +107,24 @@ struct ioat_ring_ent {
}; };
static inline struct ioat_ring_ent * static inline struct ioat_ring_ent *
ioat2_get_ring_ent(struct ioat2_dma_chan *ioat, u16 idx) ioat2_get_ring_ent(struct ioatdma_chan *ioat_chan, u16 idx)
{ {
return ioat->ring[idx & (ioat2_ring_size(ioat) - 1)]; return ioat_chan->ring[idx & (ioat2_ring_size(ioat_chan) - 1)];
} }
static inline void ioat2_set_chainaddr(struct ioat2_dma_chan *ioat, u64 addr) static inline void
ioat2_set_chainaddr(struct ioatdma_chan *ioat_chan, u64 addr)
{ {
struct ioat_chan_common *chan = &ioat->base;
writel(addr & 0x00000000FFFFFFFF, writel(addr & 0x00000000FFFFFFFF,
chan->reg_base + IOAT2_CHAINADDR_OFFSET_LOW); ioat_chan->reg_base + IOAT2_CHAINADDR_OFFSET_LOW);
writel(addr >> 32, writel(addr >> 32,
chan->reg_base + IOAT2_CHAINADDR_OFFSET_HIGH); ioat_chan->reg_base + IOAT2_CHAINADDR_OFFSET_HIGH);
} }
int ioat2_dma_probe(struct ioatdma_device *dev, int dca); int ioat2_dma_probe(struct ioatdma_device *dev, int dca);
int ioat3_dma_probe(struct ioatdma_device *dev, int dca); int ioat3_dma_probe(struct ioatdma_device *dev, int dca);
struct dca_provider *ioat3_dca_init(struct pci_dev *pdev, void __iomem *iobase); struct dca_provider *ioat3_dca_init(struct pci_dev *pdev, void __iomem *iobase);
int ioat2_check_space_lock(struct ioat2_dma_chan *ioat, int num_descs); int ioat2_check_space_lock(struct ioatdma_chan *ioat_chan, int num_descs);
int ioat2_enumerate_channels(struct ioatdma_device *device); int ioat2_enumerate_channels(struct ioatdma_device *device);
struct dma_async_tx_descriptor * struct dma_async_tx_descriptor *
ioat2_dma_prep_memcpy_lock(struct dma_chan *c, dma_addr_t dma_dest, ioat2_dma_prep_memcpy_lock(struct dma_chan *c, dma_addr_t dma_dest,
...@@ -162,12 +132,12 @@ ioat2_dma_prep_memcpy_lock(struct dma_chan *c, dma_addr_t dma_dest, ...@@ -162,12 +132,12 @@ ioat2_dma_prep_memcpy_lock(struct dma_chan *c, dma_addr_t dma_dest,
void ioat2_issue_pending(struct dma_chan *chan); void ioat2_issue_pending(struct dma_chan *chan);
int ioat2_alloc_chan_resources(struct dma_chan *c); int ioat2_alloc_chan_resources(struct dma_chan *c);
void ioat2_free_chan_resources(struct dma_chan *c); void ioat2_free_chan_resources(struct dma_chan *c);
void __ioat2_restart_chan(struct ioat2_dma_chan *ioat); void __ioat2_restart_chan(struct ioatdma_chan *ioat_chan);
bool reshape_ring(struct ioat2_dma_chan *ioat, int order); bool reshape_ring(struct ioatdma_chan *ioat, int order);
void __ioat2_issue_pending(struct ioat2_dma_chan *ioat); void __ioat2_issue_pending(struct ioatdma_chan *ioat_chan);
void ioat2_timer_event(unsigned long data); void ioat2_timer_event(unsigned long data);
int ioat2_quiesce(struct ioat_chan_common *chan, unsigned long tmo); int ioat2_quiesce(struct ioatdma_chan *ioat_chan, unsigned long tmo);
int ioat2_reset_sync(struct ioat_chan_common *chan, unsigned long tmo); int ioat2_reset_sync(struct ioatdma_chan *ioat_chan, unsigned long tmo);
extern struct kobj_type ioat2_ktype; extern struct kobj_type ioat2_ktype;
extern struct kmem_cache *ioat2_cache; extern struct kmem_cache *ioat2_cache;
#endif /* IOATDMA_V2_H */ #endif /* IOATDMA_V2_H */
drivers/dma/ioat/dma_v3.c
View file @ 5a976888
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7