Skip to content

L
linux
Commit 6447f55d authored by Linus Torvalds's avatar Linus Torvalds
Browse files
Options

Merge branch 'next' of git://git.kernel.org/pub/scm/linux/kernel/git/djbw/async_tx 

* 'next' of git://git.kernel.org/pub/scm/linux/kernel/git/djbw/async_tx: (66 commits)
  avr32: at32ap700x: fix typo in DMA master configuration
  dmaengine/dmatest: Pass timeout via module params
  dma: let IMX_DMA depend on IMX_HAVE_DMA_V1 instead of an explicit list of SoCs
  fsldma: make halt behave nicely on all supported controllers
  fsldma: reduce locking during descriptor cleanup
  fsldma: support async_tx dependencies and automatic unmapping
  fsldma: fix controller lockups
  fsldma: minor codingstyle and consistency fixes
  fsldma: improve link descriptor debugging
  fsldma: use channel name in printk output
  fsldma: move related helper functions near each other
  dmatest: fix automatic buffer unmap type
  drivers, pch_dma: Fix warning when CONFIG_PM=n.
  dmaengine/dw_dmac fix: use readl & writel instead of __raw_readl & __raw_writel
  avr32: at32ap700x: Specify DMA Flow Controller, Src and Dst msize
  dw_dmac: Setting Default Burst length for transfers as 16.
  dw_dmac: Allow src/dst msize & flow controller to be configured at runtime
  dw_dmac: Changing type of src_master and dest_master to u8.
  dw_dmac: Pass Channel Priority from platform_data
  dw_dmac: Pass Channel Allocation Order from platform_data
  ...
parents c50e3f51 3ea205c4
Hide whitespace changes
Inline Side-by-side
Showing with 1989 additions and 1262 deletions
arch/arm/mach-mxs/include/mach/dma.h 0 → 100644
View file @ 6447f55d
/*
* Copyright 2011 Freescale Semiconductor, Inc. All Rights Reserved.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*/
#ifndef __MACH_MXS_DMA_H__
#define __MACH_MXS_DMA_H__
struct mxs_dma_data {
int chan_irq;
};
static inline int mxs_dma_is_apbh(struct dma_chan *chan)
{
return !strcmp(dev_name(chan->device->dev), "mxs-dma-apbh");
}
static inline int mxs_dma_is_apbx(struct dma_chan *chan)
{
return !strcmp(dev_name(chan->device->dev), "mxs-dma-apbx");
}
#endif /* __MACH_MXS_DMA_H__ */
arch/arm/plat-nomadik/include/plat/ste_dma40.h
View file @ 6447f55d
......@@ -104,6 +104,8 @@ struct stedma40_half_channel_info {
*
* @dir: MEM 2 MEM, PERIPH 2 MEM , MEM 2 PERIPH, PERIPH 2 PERIPH
* @high_priority: true if high-priority
* @realtime: true if realtime mode is to be enabled. Only available on DMA40
* version 3+, i.e DB8500v2+
* @mode: channel mode: physical, logical, or operation
* @mode_opt: options for the chosen channel mode
* @src_dev_type: Src device type
......@@ -119,6 +121,7 @@ struct stedma40_half_channel_info {
struct stedma40_chan_cfg {
enum stedma40_xfer_dir dir;
bool high_priority;
bool realtime;
enum stedma40_mode mode;
enum stedma40_mode_opt mode_opt;
int src_dev_type;
......@@ -168,25 +171,6 @@ struct stedma40_platform_data {
bool stedma40_filter(struct dma_chan *chan, void *data);
/**
* stedma40_memcpy_sg() - extension of the dma framework, memcpy to/from
* scattergatter lists.
*
* @chan: dmaengine handle
* @sgl_dst: Destination scatter list
* @sgl_src: Source scatter list
* @sgl_len: The length of each scatterlist. Both lists must be of equal length
* and each element must match the corresponding element in the other scatter
* list.
* @flags: is actually enum dma_ctrl_flags. See dmaengine.h
*/
struct dma_async_tx_descriptor *stedma40_memcpy_sg(struct dma_chan *chan,
struct scatterlist *sgl_dst,
struct scatterlist *sgl_src,
unsigned int sgl_len,
unsigned long flags);
/**
* stedma40_slave_mem() - Transfers a raw data buffer to or from a slave
* (=device)
......
arch/avr32/mach-at32ap/at32ap700x.c
View file @ 6447f55d
......@@ -2048,6 +2048,11 @@ at32_add_device_ac97c(unsigned int id, struct ac97c_platform_data *data,
rx_dws->reg_width = DW_DMA_SLAVE_WIDTH_16BIT;
rx_dws->cfg_hi = DWC_CFGH_SRC_PER(3);
rx_dws->cfg_lo &= ~(DWC_CFGL_HS_DST_POL | DWC_CFGL_HS_SRC_POL);
rx_dws->src_master = 0;
rx_dws->dst_master = 1;
rx_dws->src_msize = DW_DMA_MSIZE_1;
rx_dws->dst_msize = DW_DMA_MSIZE_1;
rx_dws->fc = DW_DMA_FC_D_P2M;
}
/* Check if DMA slave interface for playback should be configured. */
......@@ -2056,6 +2061,11 @@ at32_add_device_ac97c(unsigned int id, struct ac97c_platform_data *data,
tx_dws->reg_width = DW_DMA_SLAVE_WIDTH_16BIT;
tx_dws->cfg_hi = DWC_CFGH_DST_PER(4);
tx_dws->cfg_lo &= ~(DWC_CFGL_HS_DST_POL | DWC_CFGL_HS_SRC_POL);
tx_dws->src_master = 0;
tx_dws->dst_master = 1;
tx_dws->src_msize = DW_DMA_MSIZE_1;
tx_dws->dst_msize = DW_DMA_MSIZE_1;
tx_dws->fc = DW_DMA_FC_D_M2P;
}
if (platform_device_add_data(pdev, data,
......@@ -2128,6 +2138,11 @@ at32_add_device_abdac(unsigned int id, struct atmel_abdac_pdata *data)
dws->reg_width = DW_DMA_SLAVE_WIDTH_32BIT;
dws->cfg_hi = DWC_CFGH_DST_PER(2);
dws->cfg_lo &= ~(DWC_CFGL_HS_DST_POL | DWC_CFGL_HS_SRC_POL);
dws->src_master = 0;
dws->dst_master = 1;
dws->src_msize = DW_DMA_MSIZE_1;
dws->dst_msize = DW_DMA_MSIZE_1;
dws->fc = DW_DMA_FC_D_M2P;
if (platform_device_add_data(pdev, data,
sizeof(struct atmel_abdac_pdata)))
......
drivers/dma/Kconfig
View file @ 6447f55d
......@@ -82,7 +82,7 @@ config INTEL_IOP_ADMA
config DW_DMAC
tristate "Synopsys DesignWare AHB DMA support"
depends on AVR32
depends on HAVE_CLK
select DMA_ENGINE
default y if CPU_AT32AP7000
help
......@@ -221,12 +221,20 @@ config IMX_SDMA
config IMX_DMA
tristate "i.MX DMA support"
depends on ARCH_MX1 || ARCH_MX21 || MACH_MX27
depends on IMX_HAVE_DMA_V1
select DMA_ENGINE
help
Support the i.MX DMA engine. This engine is integrated into
Freescale i.MX1/21/27 chips.
config MXS_DMA
bool "MXS DMA support"
depends on SOC_IMX23 || SOC_IMX28
select DMA_ENGINE
help
Support the MXS DMA engine. This engine including APBH-DMA
and APBX-DMA is integrated into Freescale i.MX23/28 chips.
config DMA_ENGINE
bool
......
drivers/dma/Makefile
View file @ 6447f55d
......@@ -19,6 +19,7 @@ obj-$(CONFIG_COH901318) += coh901318.o coh901318_lli.o
obj-$(CONFIG_AMCC_PPC440SPE_ADMA) += ppc4xx/
obj-$(CONFIG_IMX_SDMA) += imx-sdma.o
obj-$(CONFIG_IMX_DMA) += imx-dma.o
obj-$(CONFIG_MXS_DMA) += mxs-dma.o
obj-$(CONFIG_TIMB_DMA) += timb_dma.o
obj-$(CONFIG_STE_DMA40) += ste_dma40.o ste_dma40_ll.o
obj-$(CONFIG_PL330_DMA) += pl330.o
......
drivers/dma/dmatest.c
View file @ 6447f55d
......@@ -54,6 +54,11 @@ module_param(pq_sources, uint, S_IRUGO);
MODULE_PARM_DESC(pq_sources,
"Number of p+q source buffers (default: 3)");
static int timeout = 3000;
module_param(timeout, uint, S_IRUGO);
MODULE_PARM_DESC(timeout, "Transfer Timeout in msec (default: 3000), \
Pass -1 for infinite timeout");
/*
* Initialization patterns. All bytes in the source buffer has bit 7
* set, all bytes in the destination buffer has bit 7 cleared.
......@@ -285,7 +290,12 @@ static int dmatest_func(void *data)
set_user_nice(current, 10);
flags = DMA_CTRL_ACK | DMA_COMPL_SKIP_DEST_UNMAP | DMA_PREP_INTERRUPT;
/*
* src buffers are freed by the DMAEngine code with dma_unmap_single()
* dst buffers are freed by ourselves below
*/
flags = DMA_CTRL_ACK | DMA_PREP_INTERRUPT
| DMA_COMPL_SKIP_DEST_UNMAP | DMA_COMPL_SRC_UNMAP_SINGLE;
while (!kthread_should_stop()
&& !(iterations && total_tests >= iterations)) {
......@@ -294,7 +304,7 @@ static int dmatest_func(void *data)
dma_addr_t dma_srcs[src_cnt];
dma_addr_t dma_dsts[dst_cnt];
struct completion cmp;
unsigned long tmo = msecs_to_jiffies(3000);
unsigned long tmo = msecs_to_jiffies(timeout);
u8 align = 0;
total_tests++;
......
drivers/dma/dw_dmac.c
View file @ 6447f55d
......@@ -32,26 +32,30 @@
* which does not support descriptor writeback.
*/
/* NOTE: DMS+SMS is system-specific. We should get this information
* from the platform code somehow.
*/
#define DWC_DEFAULT_CTLLO (DWC_CTLL_DST_MSIZE(0) \
| DWC_CTLL_SRC_MSIZE(0) \
| DWC_CTLL_DMS(0) \
| DWC_CTLL_SMS(1) \
| DWC_CTLL_LLP_D_EN \
| DWC_CTLL_LLP_S_EN)
#define DWC_DEFAULT_CTLLO(private) ({ \
struct dw_dma_slave *__slave = (private); \
int dms = __slave ? __slave->dst_master : 0; \
int sms = __slave ? __slave->src_master : 1; \
u8 smsize = __slave ? __slave->src_msize : DW_DMA_MSIZE_16; \
u8 dmsize = __slave ? __slave->dst_msize : DW_DMA_MSIZE_16; \
\
(DWC_CTLL_DST_MSIZE(dmsize) \
| DWC_CTLL_SRC_MSIZE(smsize) \
| DWC_CTLL_LLP_D_EN \
| DWC_CTLL_LLP_S_EN \
| DWC_CTLL_DMS(dms) \
| DWC_CTLL_SMS(sms)); \
})
/*
* This is configuration-dependent and usually a funny size like 4095.
* Let's round it down to the nearest power of two.
*
* Note that this is a transfer count, i.e. if we transfer 32-bit
* words, we can do 8192 bytes per descriptor.
* words, we can do 16380 bytes per descriptor.
*
* This parameter is also system-specific.
*/
#define DWC_MAX_COUNT 2048U
#define DWC_MAX_COUNT 4095U
/*
* Number of descriptors to allocate for each channel. This should be
......@@ -84,11 +88,6 @@ static struct dw_desc *dwc_first_active(struct dw_dma_chan *dwc)
return list_entry(dwc->active_list.next, struct dw_desc, desc_node);
}
static struct dw_desc *dwc_first_queued(struct dw_dma_chan *dwc)
{
return list_entry(dwc->queue.next, struct dw_desc, desc_node);
}
static struct dw_desc *dwc_desc_get(struct dw_dma_chan *dwc)
{
struct dw_desc *desc, *_desc;
......@@ -201,6 +200,7 @@ dwc_descriptor_complete(struct dw_dma_chan *dwc, struct dw_desc *desc)
dma_async_tx_callback callback;
void *param;
struct dma_async_tx_descriptor *txd = &desc->txd;
struct dw_desc *child;
dev_vdbg(chan2dev(&dwc->chan), "descriptor %u complete\n", txd->cookie);
......@@ -209,6 +209,12 @@ dwc_descriptor_complete(struct dw_dma_chan *dwc, struct dw_desc *desc)
param = txd->callback_param;
dwc_sync_desc_for_cpu(dwc, desc);
/* async_tx_ack */
list_for_each_entry(child, &desc->tx_list, desc_node)
async_tx_ack(&child->txd);
async_tx_ack(&desc->txd);
list_splice_init(&desc->tx_list, &dwc->free_list);
list_move(&desc->desc_node, &dwc->free_list);
......@@ -259,10 +265,11 @@ static void dwc_complete_all(struct dw_dma *dw, struct dw_dma_chan *dwc)
* Submit queued descriptors ASAP, i.e. before we go through
* the completed ones.
*/
if (!list_empty(&dwc->queue))
dwc_dostart(dwc, dwc_first_queued(dwc));
list_splice_init(&dwc->active_list, &list);
list_splice_init(&dwc->queue, &dwc->active_list);
if (!list_empty(&dwc->queue)) {
list_move(dwc->queue.next, &dwc->active_list);
dwc_dostart(dwc, dwc_first_active(dwc));
}
list_for_each_entry_safe(desc, _desc, &list, desc_node)
dwc_descriptor_complete(dwc, desc);
......@@ -291,6 +298,9 @@ static void dwc_scan_descriptors(struct dw_dma *dw, struct dw_dma_chan *dwc)
return;
}
if (list_empty(&dwc->active_list))
return;
dev_vdbg(chan2dev(&dwc->chan), "scan_descriptors: llp=0x%x\n", llp);
list_for_each_entry_safe(desc, _desc, &dwc->active_list, desc_node) {
......@@ -319,8 +329,8 @@ static void dwc_scan_descriptors(struct dw_dma *dw, struct dw_dma_chan *dwc)
cpu_relax();
if (!list_empty(&dwc->queue)) {
dwc_dostart(dwc, dwc_first_queued(dwc));
list_splice_init(&dwc->queue, &dwc->active_list);
list_move(dwc->queue.next, &dwc->active_list);
dwc_dostart(dwc, dwc_first_active(dwc));
}
}
......@@ -346,7 +356,7 @@ static void dwc_handle_error(struct dw_dma *dw, struct dw_dma_chan *dwc)
*/
bad_desc = dwc_first_active(dwc);
list_del_init(&bad_desc->desc_node);
list_splice_init(&dwc->queue, dwc->active_list.prev);
list_move(dwc->queue.next, dwc->active_list.prev);
/* Clear the error flag and try to restart the controller */
dma_writel(dw, CLEAR.ERROR, dwc->mask);
......@@ -541,8 +551,8 @@ static dma_cookie_t dwc_tx_submit(struct dma_async_tx_descriptor *tx)
if (list_empty(&dwc->active_list)) {
dev_vdbg(chan2dev(tx->chan), "tx_submit: started %u\n",
desc->txd.cookie);
dwc_dostart(dwc, desc);
list_add_tail(&desc->desc_node, &dwc->active_list);
dwc_dostart(dwc, dwc_first_active(dwc));
} else {
dev_vdbg(chan2dev(tx->chan), "tx_submit: queued %u\n",
desc->txd.cookie);
......@@ -581,14 +591,16 @@ dwc_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
* We can be a lot more clever here, but this should take care
* of the most common optimization.
*/
if (!((src | dest | len) & 3))
if (!((src | dest | len) & 7))
src_width = dst_width = 3;
else if (!((src | dest | len) & 3))
src_width = dst_width = 2;
else if (!((src | dest | len) & 1))
src_width = dst_width = 1;
else
src_width = dst_width = 0;
ctllo = DWC_DEFAULT_CTLLO
ctllo = DWC_DEFAULT_CTLLO(chan->private)
| DWC_CTLL_DST_WIDTH(dst_width)
| DWC_CTLL_SRC_WIDTH(src_width)
| DWC_CTLL_DST_INC
......@@ -669,11 +681,11 @@ dwc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
switch (direction) {
case DMA_TO_DEVICE:
ctllo = (DWC_DEFAULT_CTLLO
ctllo = (DWC_DEFAULT_CTLLO(chan->private)
| DWC_CTLL_DST_WIDTH(reg_width)
| DWC_CTLL_DST_FIX
| DWC_CTLL_SRC_INC
| DWC_CTLL_FC_M2P);
| DWC_CTLL_FC(dws->fc));
reg = dws->tx_reg;
for_each_sg(sgl, sg, sg_len, i) {
struct dw_desc *desc;
......@@ -714,11 +726,11 @@ dwc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
}
break;
case DMA_FROM_DEVICE:
ctllo = (DWC_DEFAULT_CTLLO
ctllo = (DWC_DEFAULT_CTLLO(chan->private)
| DWC_CTLL_SRC_WIDTH(reg_width)
| DWC_CTLL_DST_INC
| DWC_CTLL_SRC_FIX
| DWC_CTLL_FC_P2M);
| DWC_CTLL_FC(dws->fc));
reg = dws->rx_reg;
for_each_sg(sgl, sg, sg_len, i) {
......@@ -834,7 +846,9 @@ dwc_tx_status(struct dma_chan *chan,
ret = dma_async_is_complete(cookie, last_complete, last_used);
if (ret != DMA_SUCCESS) {
spin_lock_bh(&dwc->lock);
dwc_scan_descriptors(to_dw_dma(chan->device), dwc);
spin_unlock_bh(&dwc->lock);
last_complete = dwc->completed;
last_used = chan->cookie;
......@@ -889,8 +903,11 @@ static int dwc_alloc_chan_resources(struct dma_chan *chan)
BUG_ON(!dws->dma_dev || dws->dma_dev != dw->dma.dev);
cfghi = dws->cfg_hi;
cfglo = dws->cfg_lo;
cfglo = dws->cfg_lo & ~DWC_CFGL_CH_PRIOR_MASK;
}
cfglo |= DWC_CFGL_CH_PRIOR(dwc->priority);
channel_writel(dwc, CFG_LO, cfglo);
channel_writel(dwc, CFG_HI, cfghi);
......@@ -1126,23 +1143,23 @@ struct dw_cyclic_desc *dw_dma_cyclic_prep(struct dma_chan *chan,
case DMA_TO_DEVICE:
desc->lli.dar = dws->tx_reg;
desc->lli.sar = buf_addr + (period_len * i);
desc->lli.ctllo = (DWC_DEFAULT_CTLLO
desc->lli.ctllo = (DWC_DEFAULT_CTLLO(chan->private)
| DWC_CTLL_DST_WIDTH(reg_width)
| DWC_CTLL_SRC_WIDTH(reg_width)
| DWC_CTLL_DST_FIX
| DWC_CTLL_SRC_INC
| DWC_CTLL_FC_M2P
| DWC_CTLL_FC(dws->fc)
| DWC_CTLL_INT_EN);
break;
case DMA_FROM_DEVICE:
desc->lli.dar = buf_addr + (period_len * i);
desc->lli.sar = dws->rx_reg;
desc->lli.ctllo = (DWC_DEFAULT_CTLLO
desc->lli.ctllo = (DWC_DEFAULT_CTLLO(chan->private)
| DWC_CTLL_SRC_WIDTH(reg_width)
| DWC_CTLL_DST_WIDTH(reg_width)
| DWC_CTLL_DST_INC
| DWC_CTLL_SRC_FIX
| DWC_CTLL_FC_P2M
| DWC_CTLL_FC(dws->fc)
| DWC_CTLL_INT_EN);
break;
default:
......@@ -1307,7 +1324,17 @@ static int __init dw_probe(struct platform_device *pdev)
dwc->chan.device = &dw->dma;
dwc->chan.cookie = dwc->completed = 1;
dwc->chan.chan_id = i;
list_add_tail(&dwc->chan.device_node, &dw->dma.channels);
if (pdata->chan_allocation_order == CHAN_ALLOCATION_ASCENDING)
list_add_tail(&dwc->chan.device_node,
&dw->dma.channels);
else
list_add(&dwc->chan.device_node, &dw->dma.channels);
/* 7 is highest priority & 0 is lowest. */
if (pdata->chan_priority == CHAN_PRIORITY_ASCENDING)
dwc->priority = 7 - i;
else
dwc->priority = i;
dwc->ch_regs = &__dw_regs(dw)->CHAN[i];
spin_lock_init(&dwc->lock);
......@@ -1335,6 +1362,8 @@ static int __init dw_probe(struct platform_device *pdev)
dma_cap_set(DMA_MEMCPY, dw->dma.cap_mask);
dma_cap_set(DMA_SLAVE, dw->dma.cap_mask);
if (pdata->is_private)
dma_cap_set(DMA_PRIVATE, dw->dma.cap_mask);
dw->dma.dev = &pdev->dev;
dw->dma.device_alloc_chan_resources = dwc_alloc_chan_resources;
dw->dma.device_free_chan_resources = dwc_free_chan_resources;
......@@ -1447,7 +1476,7 @@ static int __init dw_init(void)
{
return platform_driver_probe(&dw_driver, dw_probe);
}
module_init(dw_init);
subsys_initcall(dw_init);
static void __exit dw_exit(void)
{
......
drivers/dma/dw_dmac_regs.h
View file @ 6447f55d
......@@ -86,6 +86,7 @@ struct dw_dma_regs {
#define DWC_CTLL_SRC_MSIZE(n) ((n)<<14)
#define DWC_CTLL_S_GATH_EN (1 << 17) /* src gather, !FIX */
#define DWC_CTLL_D_SCAT_EN (1 << 18) /* dst scatter, !FIX */
#define DWC_CTLL_FC(n) ((n) << 20)
#define DWC_CTLL_FC_M2M (0 << 20) /* mem-to-mem */
#define DWC_CTLL_FC_M2P (1 << 20) /* mem-to-periph */
#define DWC_CTLL_FC_P2M (2 << 20) /* periph-to-mem */
......@@ -101,6 +102,8 @@ struct dw_dma_regs {
#define DWC_CTLH_BLOCK_TS_MASK 0x00000fff
/* Bitfields in CFG_LO. Platform-configurable bits are in <linux/dw_dmac.h> */
#define DWC_CFGL_CH_PRIOR_MASK (0x7 << 5) /* priority mask */
#define DWC_CFGL_CH_PRIOR(x) ((x) << 5) /* priority */
#define DWC_CFGL_CH_SUSP (1 << 8) /* pause xfer */
#define DWC_CFGL_FIFO_EMPTY (1 << 9) /* pause xfer */
#define DWC_CFGL_HS_DST (1 << 10) /* handshake w/dst */
......@@ -134,6 +137,7 @@ struct dw_dma_chan {
struct dma_chan chan;
void __iomem *ch_regs;
u8 mask;
u8 priority;
spinlock_t lock;
......@@ -155,9 +159,9 @@ __dwc_regs(struct dw_dma_chan *dwc)
}
#define channel_readl(dwc, name) \
__raw_readl(&(__dwc_regs(dwc)->name))
readl(&(__dwc_regs(dwc)->name))
#define channel_writel(dwc, name, val) \
__raw_writel((val), &(__dwc_regs(dwc)->name))
writel((val), &(__dwc_regs(dwc)->name))
static inline struct dw_dma_chan *to_dw_dma_chan(struct dma_chan *chan)
{
......@@ -181,9 +185,9 @@ static inline struct dw_dma_regs __iomem *__dw_regs(struct dw_dma *dw)
}
#define dma_readl(dw, name) \
__raw_readl(&(__dw_regs(dw)->name))
readl(&(__dw_regs(dw)->name))
#define dma_writel(dw, name, val) \
__raw_writel((val), &(__dw_regs(dw)->name))
writel((val), &(__dw_regs(dw)->name))
#define channel_set_bit(dw, reg, mask) \
dma_writel(dw, reg, ((mask) << 8) | (mask))
......
drivers/dma/fsldma.c
View file @ 6447f55d
......@@ -37,35 +37,16 @@
#include "fsldma.h"
static const char msg_ld_oom[] = "No free memory for link descriptor\n";
#define chan_dbg(chan, fmt, arg...) \
dev_dbg(chan->dev, "%s: " fmt, chan->name, ##arg)
#define chan_err(chan, fmt, arg...) \
dev_err(chan->dev, "%s: " fmt, chan->name, ##arg)
static void dma_init(struct fsldma_chan *chan)
{
/* Reset the channel */
DMA_OUT(chan, &chan->regs->mr, 0, 32);
static const char msg_ld_oom[] = "No free memory for link descriptor";
switch (chan->feature & FSL_DMA_IP_MASK) {
case FSL_DMA_IP_85XX:
/* Set the channel to below modes:
* EIE - Error interrupt enable
* EOSIE - End of segments interrupt enable (basic mode)
* EOLNIE - End of links interrupt enable
* BWC - Bandwidth sharing among channels
*/
DMA_OUT(chan, &chan->regs->mr, FSL_DMA_MR_BWC
| FSL_DMA_MR_EIE | FSL_DMA_MR_EOLNIE
| FSL_DMA_MR_EOSIE, 32);
break;
case FSL_DMA_IP_83XX:
/* Set the channel to below modes:
* EOTIE - End-of-transfer interrupt enable
* PRC_RM - PCI read multiple
*/
DMA_OUT(chan, &chan->regs->mr, FSL_DMA_MR_EOTIE
| FSL_DMA_MR_PRC_RM, 32);
break;
}
}
/*
* Register Helpers
*/
static void set_sr(struct fsldma_chan *chan, u32 val)
{
......@@ -77,14 +58,38 @@ static u32 get_sr(struct fsldma_chan *chan)
return DMA_IN(chan, &chan->regs->sr, 32);
}
static void set_cdar(struct fsldma_chan *chan, dma_addr_t addr)
{
DMA_OUT(chan, &chan->regs->cdar, addr | FSL_DMA_SNEN, 64);
}
static dma_addr_t get_cdar(struct fsldma_chan *chan)
{
return DMA_IN(chan, &chan->regs->cdar, 64) & ~FSL_DMA_SNEN;
}
static u32 get_bcr(struct fsldma_chan *chan)
{
return DMA_IN(chan, &chan->regs->bcr, 32);
}
/*
* Descriptor Helpers
*/
static void set_desc_cnt(struct fsldma_chan *chan,
struct fsl_dma_ld_hw *hw, u32 count)
{
hw->count = CPU_TO_DMA(chan, count, 32);
}
static u32 get_desc_cnt(struct fsldma_chan *chan, struct fsl_desc_sw *desc)
{
return DMA_TO_CPU(chan, desc->hw.count, 32);
}
static void set_desc_src(struct fsldma_chan *chan,
struct fsl_dma_ld_hw *hw, dma_addr_t src)
struct fsl_dma_ld_hw *hw, dma_addr_t src)
{
u64 snoop_bits;
......@@ -93,8 +98,18 @@ static void set_desc_src(struct fsldma_chan *chan,
hw->src_addr = CPU_TO_DMA(chan, snoop_bits | src, 64);
}
static dma_addr_t get_desc_src(struct fsldma_chan *chan,
struct fsl_desc_sw *desc)
{
u64 snoop_bits;
snoop_bits = ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_85XX)
? ((u64)FSL_DMA_SATR_SREADTYPE_SNOOP_READ << 32) : 0;
return DMA_TO_CPU(chan, desc->hw.src_addr, 64) & ~snoop_bits;
}
static void set_desc_dst(struct fsldma_chan *chan,
struct fsl_dma_ld_hw *hw, dma_addr_t dst)
struct fsl_dma_ld_hw *hw, dma_addr_t dst)
{
u64 snoop_bits;
......@@ -103,8 +118,18 @@ static void set_desc_dst(struct fsldma_chan *chan,
hw->dst_addr = CPU_TO_DMA(chan, snoop_bits | dst, 64);
}
static dma_addr_t get_desc_dst(struct fsldma_chan *chan,
struct fsl_desc_sw *desc)
{
u64 snoop_bits;
snoop_bits = ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_85XX)
? ((u64)FSL_DMA_DATR_DWRITETYPE_SNOOP_WRITE << 32) : 0;
return DMA_TO_CPU(chan, desc->hw.dst_addr, 64) & ~snoop_bits;
}
static void set_desc_next(struct fsldma_chan *chan,
struct fsl_dma_ld_hw *hw, dma_addr_t next)
struct fsl_dma_ld_hw *hw, dma_addr_t next)
{
u64 snoop_bits;
......@@ -113,24 +138,46 @@ static void set_desc_next(struct fsldma_chan *chan,
hw->next_ln_addr = CPU_TO_DMA(chan, snoop_bits | next, 64);
}
static void set_cdar(struct fsldma_chan *chan, dma_addr_t addr)
static void set_ld_eol(struct fsldma_chan *chan, struct fsl_desc_sw *desc)
{
DMA_OUT(chan, &chan->regs->cdar, addr | FSL_DMA_SNEN, 64);
}
u64 snoop_bits;
static dma_addr_t get_cdar(struct fsldma_chan *chan)
{
return DMA_IN(chan, &chan->regs->cdar, 64) & ~FSL_DMA_SNEN;
}
snoop_bits = ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_83XX)
? FSL_DMA_SNEN : 0;
static dma_addr_t get_ndar(struct fsldma_chan *chan)
{
return DMA_IN(chan, &chan->regs->ndar, 64);
desc->hw.next_ln_addr = CPU_TO_DMA(chan,
DMA_TO_CPU(chan, desc->hw.next_ln_addr, 64) | FSL_DMA_EOL
| snoop_bits, 64);
}
static u32 get_bcr(struct fsldma_chan *chan)
/*
* DMA Engine Hardware Control Helpers
*/
static void dma_init(struct fsldma_chan *chan)
{
return DMA_IN(chan, &chan->regs->bcr, 32);
/* Reset the channel */
DMA_OUT(chan, &chan->regs->mr, 0, 32);
switch (chan->feature & FSL_DMA_IP_MASK) {
case FSL_DMA_IP_85XX:
/* Set the channel to below modes:
* EIE - Error interrupt enable
* EOLNIE - End of links interrupt enable
* BWC - Bandwidth sharing among channels
*/
DMA_OUT(chan, &chan->regs->mr, FSL_DMA_MR_BWC
| FSL_DMA_MR_EIE | FSL_DMA_MR_EOLNIE, 32);
break;
case FSL_DMA_IP_83XX:
/* Set the channel to below modes:
* EOTIE - End-of-transfer interrupt enable
* PRC_RM - PCI read multiple
*/
DMA_OUT(chan, &chan->regs->mr, FSL_DMA_MR_EOTIE
| FSL_DMA_MR_PRC_RM, 32);
break;
}
}
static int dma_is_idle(struct fsldma_chan *chan)
......@@ -139,25 +186,32 @@ static int dma_is_idle(struct fsldma_chan *chan)
return (!(sr & FSL_DMA_SR_CB)) || (sr & FSL_DMA_SR_CH);
}
/*
* Start the DMA controller
*
* Preconditions:
* - the CDAR register must point to the start descriptor
* - the MRn[CS] bit must be cleared
*/
static void dma_start(struct fsldma_chan *chan)
{
u32 mode;
mode = DMA_IN(chan, &chan->regs->mr, 32);
if ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_85XX) {
if (chan->feature & FSL_DMA_CHAN_PAUSE_EXT) {
DMA_OUT(chan, &chan->regs->bcr, 0, 32);
mode |= FSL_DMA_MR_EMP_EN;
} else {
mode &= ~FSL_DMA_MR_EMP_EN;
}
if (chan->feature & FSL_DMA_CHAN_PAUSE_EXT) {
DMA_OUT(chan, &chan->regs->bcr, 0, 32);
mode |= FSL_DMA_MR_EMP_EN;
} else {
mode &= ~FSL_DMA_MR_EMP_EN;
}
if (chan->feature & FSL_DMA_CHAN_START_EXT)
if (chan->feature & FSL_DMA_CHAN_START_EXT) {
mode |= FSL_DMA_MR_EMS_EN;
else
} else {
mode &= ~FSL_DMA_MR_EMS_EN;
mode |= FSL_DMA_MR_CS;
}
DMA_OUT(chan, &chan->regs->mr, mode, 32);
}
......@@ -167,13 +221,26 @@ static void dma_halt(struct fsldma_chan *chan)
u32 mode;
int i;
/* read the mode register */
mode = DMA_IN(chan, &chan->regs->mr, 32);
mode |= FSL_DMA_MR_CA;
DMA_OUT(chan, &chan->regs->mr, mode, 32);
mode &= ~(FSL_DMA_MR_CS | FSL_DMA_MR_EMS_EN | FSL_DMA_MR_CA);
/*
* The 85xx controller supports channel abort, which will stop
* the current transfer. On 83xx, this bit is the transfer error
* mask bit, which should not be changed.
*/
if ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_85XX) {
mode |= FSL_DMA_MR_CA;
DMA_OUT(chan, &chan->regs->mr, mode, 32);
mode &= ~FSL_DMA_MR_CA;
}
/* stop the DMA controller */
mode &= ~(FSL_DMA_MR_CS | FSL_DMA_MR_EMS_EN);
DMA_OUT(chan, &chan->regs->mr, mode, 32);
/* wait for the DMA controller to become idle */
for (i = 0; i < 100; i++) {
if (dma_is_idle(chan))
return;
......@@ -182,20 +249,7 @@ static void dma_halt(struct fsldma_chan *chan)
}
if (!dma_is_idle(chan))
dev_err(chan->dev, "DMA halt timeout!\n");
}
static void set_ld_eol(struct fsldma_chan *chan,
struct fsl_desc_sw *desc)
{
u64 snoop_bits;
snoop_bits = ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_83XX)
? FSL_DMA_SNEN : 0;
desc->hw.next_ln_addr = CPU_TO_DMA(chan,
DMA_TO_CPU(chan, desc->hw.next_ln_addr, 64) | FSL_DMA_EOL
| snoop_bits, 64);
chan_err(chan, "DMA halt timeout!\n");
}
/**
......@@ -321,8 +375,7 @@ static void fsl_chan_toggle_ext_start(struct fsldma_chan *chan, int enable)
chan->feature &= ~FSL_DMA_CHAN_START_EXT;
}
static void append_ld_queue(struct fsldma_chan *chan,
struct fsl_desc_sw *desc)
static void append_ld_queue(struct fsldma_chan *chan, struct fsl_desc_sw *desc)
{
struct fsl_desc_sw *tail = to_fsl_desc(chan->ld_pending.prev);
......@@ -363,8 +416,8 @@ static dma_cookie_t fsl_dma_tx_submit(struct dma_async_tx_descriptor *tx)
cookie = chan->common.cookie;
list_for_each_entry(child, &desc->tx_list, node) {
cookie++;
if (cookie < 0)
cookie = 1;
if (cookie < DMA_MIN_COOKIE)
cookie = DMA_MIN_COOKIE;
child->async_tx.cookie = cookie;
}
......@@ -385,15 +438,14 @@ static dma_cookie_t fsl_dma_tx_submit(struct dma_async_tx_descriptor *tx)
*
* Return - The descriptor allocated. NULL for failed.
*/
static struct fsl_desc_sw *fsl_dma_alloc_descriptor(
struct fsldma_chan *chan)
static struct fsl_desc_sw *fsl_dma_alloc_descriptor(struct fsldma_chan *chan)
{
struct fsl_desc_sw *desc;
dma_addr_t pdesc;
desc = dma_pool_alloc(chan->desc_pool, GFP_ATOMIC, &pdesc);
if (!desc) {
dev_dbg(chan->dev, "out of memory for link desc\n");
chan_dbg(chan, "out of memory for link descriptor\n");
return NULL;
}
......@@ -403,10 +455,13 @@ static struct fsl_desc_sw *fsl_dma_alloc_descriptor(
desc->async_tx.tx_submit = fsl_dma_tx_submit;
desc->async_tx.phys = pdesc;
#ifdef FSL_DMA_LD_DEBUG
chan_dbg(chan, "LD %p allocated\n", desc);
#endif
return desc;
}
/**
* fsl_dma_alloc_chan_resources - Allocate resources for DMA channel.
* @chan : Freescale DMA channel
......@@ -427,13 +482,11 @@ static int fsl_dma_alloc_chan_resources(struct dma_chan *dchan)
* We need the descriptor to be aligned to 32bytes
* for meeting FSL DMA specification requirement.
*/
chan->desc_pool = dma_pool_create("fsl_dma_engine_desc_pool",
chan->dev,
chan->desc_pool = dma_pool_create(chan->name, chan->dev,
sizeof(struct fsl_desc_sw),
__alignof__(struct fsl_desc_sw), 0);
if (!chan->desc_pool) {
dev_err(chan->dev, "unable to allocate channel %d "
"descriptor pool\n", chan->id);
chan_err(chan, "unable to allocate descriptor pool\n");
return -ENOMEM;
}
......@@ -455,6 +508,9 @@ static void fsldma_free_desc_list(struct fsldma_chan *chan,
list_for_each_entry_safe(desc, _desc, list, node) {
list_del(&desc->node);
#ifdef FSL_DMA_LD_DEBUG
chan_dbg(chan, "LD %p free\n", desc);
#endif
dma_pool_free(chan->desc_pool, desc, desc->async_tx.phys);
}
}
......@@ -466,6 +522,9 @@ static void fsldma_free_desc_list_reverse(struct fsldma_chan *chan,
list_for_each_entry_safe_reverse(desc, _desc, list, node) {
list_del(&desc->node);
#ifdef FSL_DMA_LD_DEBUG
chan_dbg(chan, "LD %p free\n", desc);
#endif
dma_pool_free(chan->desc_pool, desc, desc->async_tx.phys);
}
}
......@@ -479,7 +538,7 @@ static void fsl_dma_free_chan_resources(struct dma_chan *dchan)
struct fsldma_chan *chan = to_fsl_chan(dchan);
unsigned long flags;
dev_dbg(chan->dev, "Free all channel resources.\n");
chan_dbg(chan, "free all channel resources\n");
spin_lock_irqsave(&chan->desc_lock, flags);
fsldma_free_desc_list(chan, &chan->ld_pending);
fsldma_free_desc_list(chan, &chan->ld_running);
......@@ -502,7 +561,7 @@ fsl_dma_prep_interrupt(struct dma_chan *dchan, unsigned long flags)
new = fsl_dma_alloc_descriptor(chan);
if (!new) {
dev_err(chan->dev, msg_ld_oom);
chan_err(chan, "%s\n", msg_ld_oom);
return NULL;
}
......@@ -512,14 +571,15 @@ fsl_dma_prep_interrupt(struct dma_chan *dchan, unsigned long flags)
/* Insert the link descriptor to the LD ring */
list_add_tail(&new->node, &new->tx_list);
/* Set End-of-link to the last link descriptor of new list*/
/* Set End-of-link to the last link descriptor of new list */
set_ld_eol(chan, new);
return &new->async_tx;
}
static struct dma_async_tx_descriptor *fsl_dma_prep_memcpy(
struct dma_chan *dchan, dma_addr_t dma_dst, dma_addr_t dma_src,
static struct dma_async_tx_descriptor *
fsl_dma_prep_memcpy(struct dma_chan *dchan,
dma_addr_t dma_dst, dma_addr_t dma_src,
size_t len, unsigned long flags)
{
struct fsldma_chan *chan;
......@@ -539,12 +599,9 @@ static struct dma_async_tx_descriptor *fsl_dma_prep_memcpy(
/* Allocate the link descriptor from DMA pool */
new = fsl_dma_alloc_descriptor(chan);
if (!new) {
dev_err(chan->dev, msg_ld_oom);
chan_err(chan, "%s\n", msg_ld_oom);
goto fail;
}
#ifdef FSL_DMA_LD_DEBUG
dev_dbg(chan->dev, "new link desc alloc %p\n", new);
#endif
copy = min(len, (size_t)FSL_DMA_BCR_MAX_CNT);
......@@ -572,7 +629,7 @@ static struct dma_async_tx_descriptor *fsl_dma_prep_memcpy(
new->async_tx.flags = flags; /* client is in control of this ack */
new->async_tx.cookie = -EBUSY;
/* Set End-of-link to the last link descriptor of new list*/
/* Set End-of-link to the last link descriptor of new list */
set_ld_eol(chan, new);
return &first->async_tx;
......@@ -627,12 +684,9 @@ static struct dma_async_tx_descriptor *fsl_dma_prep_sg(struct dma_chan *dchan,
/* allocate and populate the descriptor */
new = fsl_dma_alloc_descriptor(chan);
if (!new) {
dev_err(chan->dev, msg_ld_oom);
chan_err(chan, "%s\n", msg_ld_oom);
goto fail;
}
#ifdef FSL_DMA_LD_DEBUG
dev_dbg(chan->dev, "new link desc alloc %p\n", new);
#endif
set_desc_cnt(chan, &new->hw, len);
set_desc_src(chan, &new->hw, src);
......@@ -744,14 +798,15 @@ static int fsl_dma_device_control(struct dma_chan *dchan,
switch (cmd) {
case DMA_TERMINATE_ALL:
spin_lock_irqsave(&chan->desc_lock, flags);
/* Halt the DMA engine */
dma_halt(chan);
spin_lock_irqsave(&chan->desc_lock, flags);
/* Remove and free all of the descriptors in the LD queue */
fsldma_free_desc_list(chan, &chan->ld_pending);
fsldma_free_desc_list(chan, &chan->ld_running);
chan->idle = true;
spin_unlock_irqrestore(&chan->desc_lock, flags);
return 0;
......@@ -789,139 +844,86 @@ static int fsl_dma_device_control(struct dma_chan *dchan,
}
/**
* fsl_dma_update_completed_cookie - Update the completed cookie.
* @chan : Freescale DMA channel
*
* CONTEXT: hardirq
*/
static void fsl_dma_update_completed_cookie(struct fsldma_chan *chan)
{
struct fsl_desc_sw *desc;
unsigned long flags;
dma_cookie_t cookie;
spin_lock_irqsave(&chan->desc_lock, flags);
if (list_empty(&chan->ld_running)) {
dev_dbg(chan->dev, "no running descriptors\n");
goto out_unlock;
}
/* Get the last descriptor, update the cookie to that */
desc = to_fsl_desc(chan->ld_running.prev);
if (dma_is_idle(chan))
cookie = desc->async_tx.cookie;
else {
cookie = desc->async_tx.cookie - 1;
if (unlikely(cookie < DMA_MIN_COOKIE))
cookie = DMA_MAX_COOKIE;
}
chan->completed_cookie = cookie;
out_unlock:
spin_unlock_irqrestore(&chan->desc_lock, flags);
}
/**
* fsldma_desc_status - Check the status of a descriptor
* fsldma_cleanup_descriptor - cleanup and free a single link descriptor
* @chan: Freescale DMA channel
* @desc: DMA SW descriptor
*
* This function will return the status of the given descriptor
*/
static enum dma_status fsldma_desc_status(struct fsldma_chan *chan,
struct fsl_desc_sw *desc)
{
return dma_async_is_complete(desc->async_tx.cookie,
chan->completed_cookie,
chan->common.cookie);
}
/**
* fsl_chan_ld_cleanup - Clean up link descriptors
* @chan : Freescale DMA channel
* @desc: descriptor to cleanup and free
*
* This function clean up the ld_queue of DMA channel.
* This function is used on a descriptor which has been executed by the DMA
* controller. It will run any callbacks, submit any dependencies, and then
* free the descriptor.
*/
static void fsl_chan_ld_cleanup(struct fsldma_chan *chan)
static void fsldma_cleanup_descriptor(struct fsldma_chan *chan,
struct fsl_desc_sw *desc)
{
struct fsl_desc_sw *desc, *_desc;
unsigned long flags;
spin_lock_irqsave(&chan->desc_lock, flags);
dev_dbg(chan->dev, "chan completed_cookie = %d\n", chan->completed_cookie);
list_for_each_entry_safe(desc, _desc, &chan->ld_running, node) {
dma_async_tx_callback callback;
void *callback_param;
if (fsldma_desc_status(chan, desc) == DMA_IN_PROGRESS)
break;
struct dma_async_tx_descriptor *txd = &desc->async_tx;
struct device *dev = chan->common.device->dev;
dma_addr_t src = get_desc_src(chan, desc);
dma_addr_t dst = get_desc_dst(chan, desc);
u32 len = get_desc_cnt(chan, desc);
/* Run the link descriptor callback function */
if (txd->callback) {
#ifdef FSL_DMA_LD_DEBUG
chan_dbg(chan, "LD %p callback\n", desc);
#endif
txd->callback(txd->callback_param);
}
/* Remove from the list of running transactions */
list_del(&desc->node);
/* Run any dependencies */
dma_run_dependencies(txd);
/* Run the link descriptor callback function */
callback = desc->async_tx.callback;
callback_param = desc->async_tx.callback_param;
if (callback) {
spin_unlock_irqrestore(&chan->desc_lock, flags);
dev_dbg(chan->dev, "LD %p callback\n", desc);
callback(callback_param);
spin_lock_irqsave(&chan->desc_lock, flags);
}
/* Unmap the dst buffer, if requested */
if (!(txd->flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
if (txd->flags & DMA_COMPL_DEST_UNMAP_SINGLE)
dma_unmap_single(dev, dst, len, DMA_FROM_DEVICE);
else
dma_unmap_page(dev, dst, len, DMA_FROM_DEVICE);
}
/* Run any dependencies, then free the descriptor */
dma_run_dependencies(&desc->async_tx);
dma_pool_free(chan->desc_pool, desc, desc->async_tx.phys);
/* Unmap the src buffer, if requested */
if (!(txd->flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
if (txd->flags & DMA_COMPL_SRC_UNMAP_SINGLE)
dma_unmap_single(dev, src, len, DMA_TO_DEVICE);
else
dma_unmap_page(dev, src, len, DMA_TO_DEVICE);
}
spin_unlock_irqrestore(&chan->desc_lock, flags);
#ifdef FSL_DMA_LD_DEBUG
chan_dbg(chan, "LD %p free\n", desc);
#endif
dma_pool_free(chan->desc_pool, desc, txd->phys);
}
/**
* fsl_chan_xfer_ld_queue - transfer any pending transactions
* @chan : Freescale DMA channel
*
* This will make sure that any pending transactions will be run.
* If the DMA controller is idle, it will be started. Otherwise,
* the DMA controller's interrupt handler will start any pending
* transactions when it becomes idle.
* HARDWARE STATE: idle
* LOCKING: must hold chan->desc_lock
*/
static void fsl_chan_xfer_ld_queue(struct fsldma_chan *chan)
{
struct fsl_desc_sw *desc;
unsigned long flags;
spin_lock_irqsave(&chan->desc_lock, flags);
/*
* If the list of pending descriptors is empty, then we
* don't need to do any work at all
*/
if (list_empty(&chan->ld_pending)) {
dev_dbg(chan->dev, "no pending LDs\n");
goto out_unlock;
chan_dbg(chan, "no pending LDs\n");
return;
}
/*
* The DMA controller is not idle, which means the interrupt
* handler will start any queued transactions when it runs
* at the end of the current transaction
* The DMA controller is not idle, which means that the interrupt
* handler will start any queued transactions when it runs after
* this transaction finishes
*/
if (!dma_is_idle(chan)) {
dev_dbg(chan->dev, "DMA controller still busy\n");
goto out_unlock;
if (!chan->idle) {
chan_dbg(chan, "DMA controller still busy\n");
return;
}
/*
* TODO:
* make sure the dma_halt() function really un-wedges the
* controller as much as possible
*/
dma_halt(chan);
/*
* If there are some link descriptors which have not been
* transferred, we need to start the controller
......@@ -931,18 +933,32 @@ static void fsl_chan_xfer_ld_queue(struct fsldma_chan *chan)
* Move all elements from the queue of pending transactions
* onto the list of running transactions
*/
chan_dbg(chan, "idle, starting controller\n");
desc = list_first_entry(&chan->ld_pending, struct fsl_desc_sw, node);
list_splice_tail_init(&chan->ld_pending, &chan->ld_running);
/*
* The 85xx DMA controller doesn't clear the channel start bit
* automatically at the end of a transfer. Therefore we must clear
* it in software before starting the transfer.
*/
if ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_85XX) {
u32 mode;
mode = DMA_IN(chan, &chan->regs->mr, 32);
mode &= ~FSL_DMA_MR_CS;
DMA_OUT(chan, &chan->regs->mr, mode, 32);
}
/*
* Program the descriptor's address into the DMA controller,
* then start the DMA transaction
*/
set_cdar(chan, desc->async_tx.phys);
dma_start(chan);
get_cdar(chan);
out_unlock:
spin_unlock_irqrestore(&chan->desc_lock, flags);
dma_start(chan);
chan->idle = false;
}
/**
......@@ -952,7 +968,11 @@ static void fsl_chan_xfer_ld_queue(struct fsldma_chan *chan)
static void fsl_dma_memcpy_issue_pending(struct dma_chan *dchan)
{
struct fsldma_chan *chan = to_fsl_chan(dchan);
unsigned long flags;
spin_lock_irqsave(&chan->desc_lock, flags);
fsl_chan_xfer_ld_queue(chan);
spin_unlock_irqrestore(&chan->desc_lock, flags);
}
/**
......@@ -964,16 +984,18 @@ static enum dma_status fsl_tx_status(struct dma_chan *dchan,
struct dma_tx_state *txstate)
{
struct fsldma_chan *chan = to_fsl_chan(dchan);
dma_cookie_t last_used;
dma_cookie_t last_complete;
dma_cookie_t last_used;
unsigned long flags;
fsl_chan_ld_cleanup(chan);
spin_lock_irqsave(&chan->desc_lock, flags);
last_used = dchan->cookie;
last_complete = chan->completed_cookie;
last_used = dchan->cookie;
dma_set_tx_state(txstate, last_complete, last_used, 0);
spin_unlock_irqrestore(&chan->desc_lock, flags);
dma_set_tx_state(txstate, last_complete, last_used, 0);
return dma_async_is_complete(cookie, last_complete, last_used);
}
......@@ -984,21 +1006,20 @@ static enum dma_status fsl_tx_status(struct dma_chan *dchan,
static irqreturn_t fsldma_chan_irq(int irq, void *data)
{
struct fsldma_chan *chan = data;
int update_cookie = 0;
int xfer_ld_q = 0;
u32 stat;
/* save and clear the status register */
stat = get_sr(chan);
set_sr(chan, stat);
dev_dbg(chan->dev, "irq: channel %d, stat = 0x%x\n", chan->id, stat);
chan_dbg(chan, "irq: stat = 0x%x\n", stat);
/* check that this was really our device */
stat &= ~(FSL_DMA_SR_CB | FSL_DMA_SR_CH);
if (!stat)
return IRQ_NONE;
if (stat & FSL_DMA_SR_TE)
dev_err(chan->dev, "Transfer Error!\n");
chan_err(chan, "Transfer Error!\n");
/*
* Programming Error
......@@ -1006,29 +1027,10 @@ static irqreturn_t fsldma_chan_irq(int irq, void *data)
* triger a PE interrupt.
*/
if (stat & FSL_DMA_SR_PE) {
dev_dbg(chan->dev, "irq: Programming Error INT\n");
if (get_bcr(chan) == 0) {
/* BCR register is 0, this is a DMA_INTERRUPT async_tx.
* Now, update the completed cookie, and continue the
* next uncompleted transfer.
*/
update_cookie = 1;
xfer_ld_q = 1;
}
chan_dbg(chan, "irq: Programming Error INT\n");
stat &= ~FSL_DMA_SR_PE;
}
/*
* If the link descriptor segment transfer finishes,
* we will recycle the used descriptor.
*/
if (stat & FSL_DMA_SR_EOSI) {
dev_dbg(chan->dev, "irq: End-of-segments INT\n");
dev_dbg(chan->dev, "irq: clndar 0x%llx, nlndar 0x%llx\n",
(unsigned long long)get_cdar(chan),
(unsigned long long)get_ndar(chan));
stat &= ~FSL_DMA_SR_EOSI;
update_cookie = 1;
if (get_bcr(chan) != 0)
chan_err(chan, "Programming Error!\n");
}
/*
......@@ -1036,10 +1038,8 @@ static irqreturn_t fsldma_chan_irq(int irq, void *data)
* and start the next transfer if it exist.
*/
if (stat & FSL_DMA_SR_EOCDI) {
dev_dbg(chan->dev, "irq: End-of-Chain link INT\n");
chan_dbg(chan, "irq: End-of-Chain link INT\n");
stat &= ~FSL_DMA_SR_EOCDI;
update_cookie = 1;
xfer_ld_q = 1;
}
/*
......@@ -1048,27 +1048,79 @@ static irqreturn_t fsldma_chan_irq(int irq, void *data)
* prepare next transfer.
*/
if (stat & FSL_DMA_SR_EOLNI) {
dev_dbg(chan->dev, "irq: End-of-link INT\n");
chan_dbg(chan, "irq: End-of-link INT\n");
stat &= ~FSL_DMA_SR_EOLNI;
xfer_ld_q = 1;
}
if (update_cookie)
fsl_dma_update_completed_cookie(chan);
if (xfer_ld_q)
fsl_chan_xfer_ld_queue(chan);
/* check that the DMA controller is really idle */
if (!dma_is_idle(chan))
chan_err(chan, "irq: controller not idle!\n");
/* check that we handled all of the bits */
if (stat)
dev_dbg(chan->dev, "irq: unhandled sr 0x%02x\n", stat);
chan_err(chan, "irq: unhandled sr 0x%08x\n", stat);
dev_dbg(chan->dev, "irq: Exit\n");
/*
* Schedule the tasklet to handle all cleanup of the current
* transaction. It will start a new transaction if there is
* one pending.
*/
tasklet_schedule(&chan->tasklet);
chan_dbg(chan, "irq: Exit\n");
return IRQ_HANDLED;
}
static void dma_do_tasklet(unsigned long data)
{
struct fsldma_chan *chan = (struct fsldma_chan *)data;
fsl_chan_ld_cleanup(chan);
struct fsl_desc_sw *desc, *_desc;
LIST_HEAD(ld_cleanup);
unsigned long flags;
chan_dbg(chan, "tasklet entry\n");
spin_lock_irqsave(&chan->desc_lock, flags);
/* update the cookie if we have some descriptors to cleanup */
if (!list_empty(&chan->ld_running)) {
dma_cookie_t cookie;
desc = to_fsl_desc(chan->ld_running.prev);
cookie = desc->async_tx.cookie;
chan->completed_cookie = cookie;
chan_dbg(chan, "completed_cookie=%d\n", cookie);
}
/*
* move the descriptors to a temporary list so we can drop the lock
* during the entire cleanup operation
*/
list_splice_tail_init(&chan->ld_running, &ld_cleanup);
/* the hardware is now idle and ready for more */
chan->idle = true;
/*
* Start any pending transactions automatically
*
* In the ideal case, we keep the DMA controller busy while we go
* ahead and free the descriptors below.
*/
fsl_chan_xfer_ld_queue(chan);
spin_unlock_irqrestore(&chan->desc_lock, flags);
/* Run the callback for each descriptor, in order */
list_for_each_entry_safe(desc, _desc, &ld_cleanup, node) {
/* Remove from the list of transactions */
list_del(&desc->node);
/* Run all cleanup for this descriptor */
fsldma_cleanup_descriptor(chan, desc);
}
chan_dbg(chan, "tasklet exit\n");
}
static irqreturn_t fsldma_ctrl_irq(int irq, void *data)
......@@ -1116,7 +1168,7 @@ static void fsldma_free_irqs(struct fsldma_device *fdev)
for (i = 0; i < FSL_DMA_MAX_CHANS_PER_DEVICE; i++) {
chan = fdev->chan[i];
if (chan && chan->irq != NO_IRQ) {
dev_dbg(fdev->dev, "free channel %d IRQ\n", chan->id);
chan_dbg(chan, "free per-channel IRQ\n");
free_irq(chan->irq, chan);
}
}
......@@ -1143,19 +1195,16 @@ static int fsldma_request_irqs(struct fsldma_device *fdev)
continue;
if (chan->irq == NO_IRQ) {
dev_err(fdev->dev, "no interrupts property defined for "
"DMA channel %d. Please fix your "
"device tree\n", chan->id);
chan_err(chan, "interrupts property missing in device tree\n");
ret = -ENODEV;
goto out_unwind;
}
dev_dbg(fdev->dev, "request channel %d IRQ\n", chan->id);
chan_dbg(chan, "request per-channel IRQ\n");
ret = request_irq(chan->irq, fsldma_chan_irq, IRQF_SHARED,
"fsldma-chan", chan);
if (ret) {
dev_err(fdev->dev, "unable to request IRQ for DMA "
"channel %d\n", chan->id);
chan_err(chan, "unable to request per-channel IRQ\n");
goto out_unwind;
}
}
......@@ -1230,6 +1279,7 @@ static int __devinit fsl_dma_chan_probe(struct fsldma_device *fdev,
fdev->chan[chan->id] = chan;
tasklet_init(&chan->tasklet, dma_do_tasklet, (unsigned long)chan);
snprintf(chan->name, sizeof(chan->name), "chan%d", chan->id);
/* Initialize the channel */
dma_init(chan);
......@@ -1250,6 +1300,7 @@ static int __devinit fsl_dma_chan_probe(struct fsldma_device *fdev,
spin_lock_init(&chan->desc_lock);
INIT_LIST_HEAD(&chan->ld_pending);
INIT_LIST_HEAD(&chan->ld_running);
chan->idle = true;
chan->common.device = &fdev->common;
......
drivers/dma/fsldma.h
View file @ 6447f55d
......@@ -102,8 +102,8 @@ struct fsl_desc_sw {
} __attribute__((aligned(32)));
struct fsldma_chan_regs {
u32 mr; /* 0x00 - Mode Register */
u32 sr; /* 0x04 - Status Register */
u32 mr; /* 0x00 - Mode Register */
u32 sr; /* 0x04 - Status Register */
u64 cdar; /* 0x08 - Current descriptor address register */
u64 sar; /* 0x10 - Source Address Register */
u64 dar; /* 0x18 - Destination Address Register */
......@@ -135,6 +135,7 @@ struct fsldma_device {
#define FSL_DMA_CHAN_START_EXT 0x00002000
struct fsldma_chan {
char name[8]; /* Channel name */
struct fsldma_chan_regs __iomem *regs;
dma_cookie_t completed_cookie; /* The maximum cookie completed */
spinlock_t desc_lock; /* Descriptor operation lock */
......@@ -147,6 +148,7 @@ struct fsldma_chan {
int id; /* Raw id of this channel */
struct tasklet_struct tasklet;
u32 feature;
bool idle; /* DMA controller is idle */
void (*toggle_ext_pause)(struct fsldma_chan *fsl_chan, int enable);
void (*toggle_ext_start)(struct fsldma_chan *fsl_chan, int enable);
......
drivers/dma/mxs-dma.c 0 → 100644
View file @ 6447f55d
/*
* Copyright 2011 Freescale Semiconductor, Inc. All Rights Reserved.
*
* Refer to drivers/dma/imx-sdma.c
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*/
#include <linux/init.h>
#include <linux/types.h>
#include <linux/mm.h>
#include <linux/interrupt.h>
#include <linux/clk.h>
#include <linux/wait.h>
#include <linux/sched.h>
#include <linux/semaphore.h>
#include <linux/device.h>
#include <linux/dma-mapping.h>
#include <linux/slab.h>
#include <linux/platform_device.h>
#include <linux/dmaengine.h>
#include <linux/delay.h>
#include <asm/irq.h>
#include <mach/mxs.h>
#include <mach/dma.h>
#include <mach/common.h>
/*
* NOTE: The term "PIO" throughout the mxs-dma implementation means
* PIO mode of mxs apbh-dma and apbx-dma. With this working mode,
* dma can program the controller registers of peripheral devices.
*/
#define MXS_DMA_APBH 0
#define MXS_DMA_APBX 1
#define dma_is_apbh() (mxs_dma->dev_id == MXS_DMA_APBH)
#define APBH_VERSION_LATEST 3
#define apbh_is_old() (mxs_dma->version < APBH_VERSION_LATEST)
#define HW_APBHX_CTRL0 0x000
#define BM_APBH_CTRL0_APB_BURST8_EN (1 << 29)
#define BM_APBH_CTRL0_APB_BURST_EN (1 << 28)
#define BP_APBH_CTRL0_CLKGATE_CHANNEL 8
#define BP_APBH_CTRL0_RESET_CHANNEL 16
#define HW_APBHX_CTRL1 0x010
#define HW_APBHX_CTRL2 0x020
#define HW_APBHX_CHANNEL_CTRL 0x030
#define BP_APBHX_CHANNEL_CTRL_RESET_CHANNEL 16
#define HW_APBH_VERSION (cpu_is_mx23() ? 0x3f0 : 0x800)
#define HW_APBX_VERSION 0x800
#define BP_APBHX_VERSION_MAJOR 24
#define HW_APBHX_CHn_NXTCMDAR(n) \
(((dma_is_apbh() && apbh_is_old()) ? 0x050 : 0x110) + (n) * 0x70)
#define HW_APBHX_CHn_SEMA(n) \
(((dma_is_apbh() && apbh_is_old()) ? 0x080 : 0x140) + (n) * 0x70)
/*
* ccw bits definitions
*
* COMMAND: 0..1 (2)
* CHAIN: 2 (1)
* IRQ: 3 (1)
* NAND_LOCK: 4 (1) - not implemented
* NAND_WAIT4READY: 5 (1) - not implemented
* DEC_SEM: 6 (1)
* WAIT4END: 7 (1)
* HALT_ON_TERMINATE: 8 (1)
* TERMINATE_FLUSH: 9 (1)
* RESERVED: 10..11 (2)
* PIO_NUM: 12..15 (4)
*/
#define BP_CCW_COMMAND 0
#define BM_CCW_COMMAND (3 << 0)
#define CCW_CHAIN (1 << 2)
#define CCW_IRQ (1 << 3)
#define CCW_DEC_SEM (1 << 6)
#define CCW_WAIT4END (1 << 7)
#define CCW_HALT_ON_TERM (1 << 8)
#define CCW_TERM_FLUSH (1 << 9)
#define BP_CCW_PIO_NUM 12
#define BM_CCW_PIO_NUM (0xf << 12)
#define BF_CCW(value, field) (((value) << BP_CCW_##field) & BM_CCW_##field)
#define MXS_DMA_CMD_NO_XFER 0
#define MXS_DMA_CMD_WRITE 1
#define MXS_DMA_CMD_READ 2
#define MXS_DMA_CMD_DMA_SENSE 3 /* not implemented */
struct mxs_dma_ccw {
u32 next;
u16 bits;
u16 xfer_bytes;
#define MAX_XFER_BYTES 0xff00
u32 bufaddr;
#define MXS_PIO_WORDS 16
u32 pio_words[MXS_PIO_WORDS];
};
#define NUM_CCW (int)(PAGE_SIZE / sizeof(struct mxs_dma_ccw))
struct mxs_dma_chan {
struct mxs_dma_engine *mxs_dma;
struct dma_chan chan;
struct dma_async_tx_descriptor desc;
struct tasklet_struct tasklet;
int chan_irq;
struct mxs_dma_ccw *ccw;
dma_addr_t ccw_phys;
dma_cookie_t last_completed;
enum dma_status status;
unsigned int flags;
#define MXS_DMA_SG_LOOP (1 << 0)
};
#define MXS_DMA_CHANNELS 16
#define MXS_DMA_CHANNELS_MASK 0xffff
struct mxs_dma_engine {
int dev_id;
unsigned int version;
void __iomem *base;
struct clk *clk;
struct dma_device dma_device;
struct device_dma_parameters dma_parms;
struct mxs_dma_chan mxs_chans[MXS_DMA_CHANNELS];
};
static void mxs_dma_reset_chan(struct mxs_dma_chan *mxs_chan)
{
struct mxs_dma_engine *mxs_dma = mxs_chan->mxs_dma;
int chan_id = mxs_chan->chan.chan_id;
if (dma_is_apbh() && apbh_is_old())
writel(1 << (chan_id + BP_APBH_CTRL0_RESET_CHANNEL),
mxs_dma->base + HW_APBHX_CTRL0 + MXS_SET_ADDR);
else
writel(1 << (chan_id + BP_APBHX_CHANNEL_CTRL_RESET_CHANNEL),
mxs_dma->base + HW_APBHX_CHANNEL_CTRL + MXS_SET_ADDR);
}
static void mxs_dma_enable_chan(struct mxs_dma_chan *mxs_chan)
{
struct mxs_dma_engine *mxs_dma = mxs_chan->mxs_dma;
int chan_id = mxs_chan->chan.chan_id;
/* set cmd_addr up */
writel(mxs_chan->ccw_phys,
mxs_dma->base + HW_APBHX_CHn_NXTCMDAR(chan_id));
/* enable apbh channel clock */
if (dma_is_apbh()) {
if (apbh_is_old())
writel(1 << (chan_id + BP_APBH_CTRL0_CLKGATE_CHANNEL),
mxs_dma->base + HW_APBHX_CTRL0 + MXS_CLR_ADDR);
else
writel(1 << chan_id,
mxs_dma->base + HW_APBHX_CTRL0 + MXS_CLR_ADDR);
}
/* write 1 to SEMA to kick off the channel */
writel(1, mxs_dma->base + HW_APBHX_CHn_SEMA(chan_id));
}
static void mxs_dma_disable_chan(struct mxs_dma_chan *mxs_chan)
{
struct mxs_dma_engine *mxs_dma = mxs_chan->mxs_dma;
int chan_id = mxs_chan->chan.chan_id;
/* disable apbh channel clock */
if (dma_is_apbh()) {
if (apbh_is_old())
writel(1 << (chan_id + BP_APBH_CTRL0_CLKGATE_CHANNEL),
mxs_dma->base + HW_APBHX_CTRL0 + MXS_SET_ADDR);
else
writel(1 << chan_id,
mxs_dma->base + HW_APBHX_CTRL0 + MXS_SET_ADDR);
}
mxs_chan->status = DMA_SUCCESS;
}
static void mxs_dma_pause_chan(struct mxs_dma_chan *mxs_chan)
{
struct mxs_dma_engine *mxs_dma = mxs_chan->mxs_dma;
int chan_id = mxs_chan->chan.chan_id;
/* freeze the channel */
if (dma_is_apbh() && apbh_is_old())
writel(1 << chan_id,
mxs_dma->base + HW_APBHX_CTRL0 + MXS_SET_ADDR);
else
writel(1 << chan_id,
mxs_dma->base + HW_APBHX_CHANNEL_CTRL + MXS_SET_ADDR);
mxs_chan->status = DMA_PAUSED;
}
static void mxs_dma_resume_chan(struct mxs_dma_chan *mxs_chan)
{
struct mxs_dma_engine *mxs_dma = mxs_chan->mxs_dma;
int chan_id = mxs_chan->chan.chan_id;
/* unfreeze the channel */
if (dma_is_apbh() && apbh_is_old())
writel(1 << chan_id,
mxs_dma->base + HW_APBHX_CTRL0 + MXS_CLR_ADDR);
else
writel(1 << chan_id,
mxs_dma->base + HW_APBHX_CHANNEL_CTRL + MXS_CLR_ADDR);
mxs_chan->status = DMA_IN_PROGRESS;
}
static dma_cookie_t mxs_dma_assign_cookie(struct mxs_dma_chan *mxs_chan)
{
dma_cookie_t cookie = mxs_chan->chan.cookie;
if (++cookie < 0)
cookie = 1;
mxs_chan->chan.cookie = cookie;
mxs_chan->desc.cookie = cookie;
return cookie;
}
static struct mxs_dma_chan *to_mxs_dma_chan(struct dma_chan *chan)
{
return container_of(chan, struct mxs_dma_chan, chan);
}
static dma_cookie_t mxs_dma_tx_submit(struct dma_async_tx_descriptor *tx)
{
struct mxs_dma_chan *mxs_chan = to_mxs_dma_chan(tx->chan);
mxs_dma_enable_chan(mxs_chan);
return mxs_dma_assign_cookie(mxs_chan);
}
static void mxs_dma_tasklet(unsigned long data)
{
struct mxs_dma_chan *mxs_chan = (struct mxs_dma_chan *) data;
if (mxs_chan->desc.callback)
mxs_chan->desc.callback(mxs_chan->desc.callback_param);
}
static irqreturn_t mxs_dma_int_handler(int irq, void *dev_id)
{
struct mxs_dma_engine *mxs_dma = dev_id;
u32 stat1, stat2;
/* completion status */
stat1 = readl(mxs_dma->base + HW_APBHX_CTRL1);
stat1 &= MXS_DMA_CHANNELS_MASK;
writel(stat1, mxs_dma->base + HW_APBHX_CTRL1 + MXS_CLR_ADDR);
/* error status */
stat2 = readl(mxs_dma->base + HW_APBHX_CTRL2);
writel(stat2, mxs_dma->base + HW_APBHX_CTRL2 + MXS_CLR_ADDR);
/*
* When both completion and error of termination bits set at the
* same time, we do not take it as an error. IOW, it only becomes
* an error we need to handler here in case of ether it's (1) an bus
* error or (2) a termination error with no completion.
*/
stat2 = ((stat2 >> MXS_DMA_CHANNELS) & stat2) | /* (1) */
(~(stat2 >> MXS_DMA_CHANNELS) & stat2 & ~stat1); /* (2) */
/* combine error and completion status for checking */
stat1 = (stat2 << MXS_DMA_CHANNELS) | stat1;
while (stat1) {
int channel = fls(stat1) - 1;
struct mxs_dma_chan *mxs_chan =
&mxs_dma->mxs_chans[channel % MXS_DMA_CHANNELS];
if (channel >= MXS_DMA_CHANNELS) {
dev_dbg(mxs_dma->dma_device.dev,
"%s: error in channel %d\n", __func__,
channel - MXS_DMA_CHANNELS);
mxs_chan->status = DMA_ERROR;
mxs_dma_reset_chan(mxs_chan);
} else {
if (mxs_chan->flags & MXS_DMA_SG_LOOP)
mxs_chan->status = DMA_IN_PROGRESS;
else
mxs_chan->status = DMA_SUCCESS;
}
stat1 &= ~(1 << channel);
if (mxs_chan->status == DMA_SUCCESS)
mxs_chan->last_completed = mxs_chan->desc.cookie;
/* schedule tasklet on this channel */
tasklet_schedule(&mxs_chan->tasklet);
}
return IRQ_HANDLED;
}
static int mxs_dma_alloc_chan_resources(struct dma_chan *chan)
{
struct mxs_dma_chan *mxs_chan = to_mxs_dma_chan(chan);
struct mxs_dma_data *data = chan->private;
struct mxs_dma_engine *mxs_dma = mxs_chan->mxs_dma;
int ret;
if (!data)
return -EINVAL;
mxs_chan->chan_irq = data->chan_irq;
mxs_chan->ccw = dma_alloc_coherent(mxs_dma->dma_device.dev, PAGE_SIZE,
&mxs_chan->ccw_phys, GFP_KERNEL);
if (!mxs_chan->ccw) {
ret = -ENOMEM;
goto err_alloc;
}
memset(mxs_chan->ccw, 0, PAGE_SIZE);
ret = request_irq(mxs_chan->chan_irq, mxs_dma_int_handler,
0, "mxs-dma", mxs_dma);
if (ret)
goto err_irq;
ret = clk_enable(mxs_dma->clk);
if (ret)
goto err_clk;
mxs_dma_reset_chan(mxs_chan);
dma_async_tx_descriptor_init(&mxs_chan->desc, chan);
mxs_chan->desc.tx_submit = mxs_dma_tx_submit;
/* the descriptor is ready */
async_tx_ack(&mxs_chan->desc);
return 0;
err_clk:
free_irq(mxs_chan->chan_irq, mxs_dma);
err_irq:
dma_free_coherent(mxs_dma->dma_device.dev, PAGE_SIZE,
mxs_chan->ccw, mxs_chan->ccw_phys);
err_alloc:
return ret;
}
static void mxs_dma_free_chan_resources(struct dma_chan *chan)
{
struct mxs_dma_chan *mxs_chan = to_mxs_dma_chan(chan);
struct mxs_dma_engine *mxs_dma = mxs_chan->mxs_dma;
mxs_dma_disable_chan(mxs_chan);
free_irq(mxs_chan->chan_irq, mxs_dma);
dma_free_coherent(mxs_dma->dma_device.dev, PAGE_SIZE,
mxs_chan->ccw, mxs_chan->ccw_phys);
clk_disable(mxs_dma->clk);
}
static struct dma_async_tx_descriptor *mxs_dma_prep_slave_sg(
struct dma_chan *chan, struct scatterlist *sgl,
unsigned int sg_len, enum dma_data_direction direction,
unsigned long append)
{
struct mxs_dma_chan *mxs_chan = to_mxs_dma_chan(chan);
struct mxs_dma_engine *mxs_dma = mxs_chan->mxs_dma;
struct mxs_dma_ccw *ccw;
struct scatterlist *sg;
int i, j;
u32 *pio;
static int idx;
if (mxs_chan->status == DMA_IN_PROGRESS && !append)
return NULL;
if (sg_len + (append ? idx : 0) > NUM_CCW) {
dev_err(mxs_dma->dma_device.dev,
"maximum number of sg exceeded: %d > %d\n",
sg_len, NUM_CCW);
goto err_out;
}
mxs_chan->status = DMA_IN_PROGRESS;
mxs_chan->flags = 0;
/*
* If the sg is prepared with append flag set, the sg
* will be appended to the last prepared sg.
*/
if (append) {
BUG_ON(idx < 1);
ccw = &mxs_chan->ccw[idx - 1];
ccw->next = mxs_chan->ccw_phys + sizeof(*ccw) * idx;
ccw->bits |= CCW_CHAIN;
ccw->bits &= ~CCW_IRQ;
ccw->bits &= ~CCW_DEC_SEM;
ccw->bits &= ~CCW_WAIT4END;
} else {
idx = 0;
}
if (direction == DMA_NONE) {
ccw = &mxs_chan->ccw[idx++];
pio = (u32 *) sgl;
for (j = 0; j < sg_len;)
ccw->pio_words[j++] = *pio++;
ccw->bits = 0;
ccw->bits |= CCW_IRQ;
ccw->bits |= CCW_DEC_SEM;
ccw->bits |= CCW_WAIT4END;
ccw->bits |= CCW_HALT_ON_TERM;
ccw->bits |= CCW_TERM_FLUSH;
ccw->bits |= BF_CCW(sg_len, PIO_NUM);
ccw->bits |= BF_CCW(MXS_DMA_CMD_NO_XFER, COMMAND);
} else {
for_each_sg(sgl, sg, sg_len, i) {
if (sg->length > MAX_XFER_BYTES) {
dev_err(mxs_dma->dma_device.dev, "maximum bytes for sg entry exceeded: %d > %d\n",
sg->length, MAX_XFER_BYTES);
goto err_out;
}
ccw = &mxs_chan->ccw[idx++];
ccw->next = mxs_chan->ccw_phys + sizeof(*ccw) * idx;
ccw->bufaddr = sg->dma_address;
ccw->xfer_bytes = sg->length;
ccw->bits = 0;
ccw->bits |= CCW_CHAIN;
ccw->bits |= CCW_HALT_ON_TERM;
ccw->bits |= CCW_TERM_FLUSH;
ccw->bits |= BF_CCW(direction == DMA_FROM_DEVICE ?
MXS_DMA_CMD_WRITE : MXS_DMA_CMD_READ,
COMMAND);
if (i + 1 == sg_len) {
ccw->bits &= ~CCW_CHAIN;
ccw->bits |= CCW_IRQ;
ccw->bits |= CCW_DEC_SEM;
ccw->bits |= CCW_WAIT4END;
}
}
}
return &mxs_chan->desc;
err_out:
mxs_chan->status = DMA_ERROR;
return NULL;
}
static struct dma_async_tx_descriptor *mxs_dma_prep_dma_cyclic(
struct dma_chan *chan, dma_addr_t dma_addr, size_t buf_len,
size_t period_len, enum dma_data_direction direction)
{
struct mxs_dma_chan *mxs_chan = to_mxs_dma_chan(chan);
struct mxs_dma_engine *mxs_dma = mxs_chan->mxs_dma;
int num_periods = buf_len / period_len;
int i = 0, buf = 0;
if (mxs_chan->status == DMA_IN_PROGRESS)
return NULL;
mxs_chan->status = DMA_IN_PROGRESS;
mxs_chan->flags |= MXS_DMA_SG_LOOP;
if (num_periods > NUM_CCW) {
dev_err(mxs_dma->dma_device.dev,
"maximum number of sg exceeded: %d > %d\n",
num_periods, NUM_CCW);
goto err_out;
}
if (period_len > MAX_XFER_BYTES) {
dev_err(mxs_dma->dma_device.dev,
"maximum period size exceeded: %d > %d\n",
period_len, MAX_XFER_BYTES);
goto err_out;
}
while (buf < buf_len) {
struct mxs_dma_ccw *ccw = &mxs_chan->ccw[i];
if (i + 1 == num_periods)
ccw->next = mxs_chan->ccw_phys;
else
ccw->next = mxs_chan->ccw_phys + sizeof(*ccw) * (i + 1);
ccw->bufaddr = dma_addr;
ccw->xfer_bytes = period_len;
ccw->bits = 0;
ccw->bits |= CCW_CHAIN;
ccw->bits |= CCW_IRQ;
ccw->bits |= CCW_HALT_ON_TERM;
ccw->bits |= CCW_TERM_FLUSH;
ccw->bits |= BF_CCW(direction == DMA_FROM_DEVICE ?
MXS_DMA_CMD_WRITE : MXS_DMA_CMD_READ, COMMAND);
dma_addr += period_len;
buf += period_len;
i++;
}
return &mxs_chan->desc;
err_out:
mxs_chan->status = DMA_ERROR;
return NULL;
}
static int mxs_dma_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
unsigned long arg)
{
struct mxs_dma_chan *mxs_chan = to_mxs_dma_chan(chan);
int ret = 0;
switch (cmd) {
case DMA_TERMINATE_ALL:
mxs_dma_disable_chan(mxs_chan);
break;
case DMA_PAUSE:
mxs_dma_pause_chan(mxs_chan);
break;
case DMA_RESUME:
mxs_dma_resume_chan(mxs_chan);
break;
default:
ret = -ENOSYS;
}
return ret;
}
static enum dma_status mxs_dma_tx_status(struct dma_chan *chan,
dma_cookie_t cookie, struct dma_tx_state *txstate)
{
struct mxs_dma_chan *mxs_chan = to_mxs_dma_chan(chan);
dma_cookie_t last_used;
last_used = chan->cookie;
dma_set_tx_state(txstate, mxs_chan->last_completed, last_used, 0);
return mxs_chan->status;
}
static void mxs_dma_issue_pending(struct dma_chan *chan)
{
/*
* Nothing to do. We only have a single descriptor.
*/
}
static int __init mxs_dma_init(struct mxs_dma_engine *mxs_dma)
{
int ret;
ret = clk_enable(mxs_dma->clk);
if (ret)
goto err_out;
ret = mxs_reset_block(mxs_dma->base);
if (ret)
goto err_out;
/* only major version matters */
mxs_dma->version = readl(mxs_dma->base +
((mxs_dma->dev_id == MXS_DMA_APBX) ?
HW_APBX_VERSION : HW_APBH_VERSION)) >>
BP_APBHX_VERSION_MAJOR;
/* enable apbh burst */
if (dma_is_apbh()) {
writel(BM_APBH_CTRL0_APB_BURST_EN,
mxs_dma->base + HW_APBHX_CTRL0 + MXS_SET_ADDR);
writel(BM_APBH_CTRL0_APB_BURST8_EN,
mxs_dma->base + HW_APBHX_CTRL0 + MXS_SET_ADDR);
}
/* enable irq for all the channels */
writel(MXS_DMA_CHANNELS_MASK << MXS_DMA_CHANNELS,
mxs_dma->base + HW_APBHX_CTRL1 + MXS_SET_ADDR);
clk_disable(mxs_dma->clk);
return 0;
err_out:
return ret;
}
static int __init mxs_dma_probe(struct platform_device *pdev)
{
const struct platform_device_id *id_entry =
platform_get_device_id(pdev);
struct mxs_dma_engine *mxs_dma;
struct resource *iores;
int ret, i;
mxs_dma = kzalloc(sizeof(*mxs_dma), GFP_KERNEL);
if (!mxs_dma)
return -ENOMEM;
mxs_dma->dev_id = id_entry->driver_data;
iores = platform_get_resource(pdev, IORESOURCE_MEM, 0);
if (!request_mem_region(iores->start, resource_size(iores),
pdev->name)) {
ret = -EBUSY;
goto err_request_region;
}
mxs_dma->base = ioremap(iores->start, resource_size(iores));
if (!mxs_dma->base) {
ret = -ENOMEM;
goto err_ioremap;
}
mxs_dma->clk = clk_get(&pdev->dev, NULL);
if (IS_ERR(mxs_dma->clk)) {
ret = PTR_ERR(mxs_dma->clk);
goto err_clk;
}
dma_cap_set(DMA_SLAVE, mxs_dma->dma_device.cap_mask);
dma_cap_set(DMA_CYCLIC, mxs_dma->dma_device.cap_mask);
INIT_LIST_HEAD(&mxs_dma->dma_device.channels);
/* Initialize channel parameters */
for (i = 0; i < MXS_DMA_CHANNELS; i++) {
struct mxs_dma_chan *mxs_chan = &mxs_dma->mxs_chans[i];
mxs_chan->mxs_dma = mxs_dma;
mxs_chan->chan.device = &mxs_dma->dma_device;
tasklet_init(&mxs_chan->tasklet, mxs_dma_tasklet,
(unsigned long) mxs_chan);
/* Add the channel to mxs_chan list */
list_add_tail(&mxs_chan->chan.device_node,
&mxs_dma->dma_device.channels);
}
ret = mxs_dma_init(mxs_dma);
if (ret)
goto err_init;
mxs_dma->dma_device.dev = &pdev->dev;
/* mxs_dma gets 65535 bytes maximum sg size */
mxs_dma->dma_device.dev->dma_parms = &mxs_dma->dma_parms;
dma_set_max_seg_size(mxs_dma->dma_device.dev, MAX_XFER_BYTES);
mxs_dma->dma_device.device_alloc_chan_resources = mxs_dma_alloc_chan_resources;
mxs_dma->dma_device.device_free_chan_resources = mxs_dma_free_chan_resources;
mxs_dma->dma_device.device_tx_status = mxs_dma_tx_status;
mxs_dma->dma_device.device_prep_slave_sg = mxs_dma_prep_slave_sg;
mxs_dma->dma_device.device_prep_dma_cyclic = mxs_dma_prep_dma_cyclic;
mxs_dma->dma_device.device_control = mxs_dma_control;
mxs_dma->dma_device.device_issue_pending = mxs_dma_issue_pending;
ret = dma_async_device_register(&mxs_dma->dma_device);
if (ret) {
dev_err(mxs_dma->dma_device.dev, "unable to register\n");
goto err_init;
}
dev_info(mxs_dma->dma_device.dev, "initialized\n");
return 0;
err_init:
clk_put(mxs_dma->clk);
err_clk:
iounmap(mxs_dma->base);
err_ioremap:
release_mem_region(iores->start, resource_size(iores));
err_request_region:
kfree(mxs_dma);
return ret;
}
static struct platform_device_id mxs_dma_type[] = {
{
.name = "mxs-dma-apbh",
.driver_data = MXS_DMA_APBH,
}, {
.name = "mxs-dma-apbx",
.driver_data = MXS_DMA_APBX,
}
};
static struct platform_driver mxs_dma_driver = {
.driver = {
.name = "mxs-dma",
},
.id_table = mxs_dma_type,
};
static int __init mxs_dma_module_init(void)
{
return platform_driver_probe(&mxs_dma_driver, mxs_dma_probe);
}
subsys_initcall(mxs_dma_module_init);
drivers/dma/pch_dma.c
View file @ 6447f55d
......@@ -82,7 +82,7 @@ struct pch_dma_regs {
u32 dma_sts1;
u32 reserved2;
u32 reserved3;
struct pch_dma_desc_regs desc[0];
struct pch_dma_desc_regs desc[MAX_CHAN_NR];
};
struct pch_dma_desc {
......@@ -124,7 +124,7 @@ struct pch_dma {
struct pci_pool *pool;
struct pch_dma_regs regs;
struct pch_dma_desc_regs ch_regs[MAX_CHAN_NR];
struct pch_dma_chan channels[0];
struct pch_dma_chan channels[MAX_CHAN_NR];
};
#define PCH_DMA_CTL0 0x00
......@@ -366,7 +366,7 @@ static dma_cookie_t pd_tx_submit(struct dma_async_tx_descriptor *txd)
struct pch_dma_chan *pd_chan = to_pd_chan(txd->chan);
dma_cookie_t cookie;
spin_lock_bh(&pd_chan->lock);
spin_lock(&pd_chan->lock);
cookie = pdc_assign_cookie(pd_chan, desc);
if (list_empty(&pd_chan->active_list)) {
......@@ -376,7 +376,7 @@ static dma_cookie_t pd_tx_submit(struct dma_async_tx_descriptor *txd)
list_add_tail(&desc->desc_node, &pd_chan->queue);
}
spin_unlock_bh(&pd_chan->lock);
spin_unlock(&pd_chan->lock);
return 0;
}
......@@ -386,7 +386,7 @@ static struct pch_dma_desc *pdc_alloc_desc(struct dma_chan *chan, gfp_t flags)
struct pch_dma *pd = to_pd(chan->device);
dma_addr_t addr;
desc = pci_pool_alloc(pd->pool, GFP_KERNEL, &addr);
desc = pci_pool_alloc(pd->pool, flags, &addr);
if (desc) {
memset(desc, 0, sizeof(struct pch_dma_desc));
INIT_LIST_HEAD(&desc->tx_list);
......@@ -405,7 +405,7 @@ static struct pch_dma_desc *pdc_desc_get(struct pch_dma_chan *pd_chan)
struct pch_dma_desc *ret = NULL;
int i;
spin_lock_bh(&pd_chan->lock);
spin_lock(&pd_chan->lock);
list_for_each_entry_safe(desc, _d, &pd_chan->free_list, desc_node) {
i++;
if (async_tx_test_ack(&desc->txd)) {
......@@ -415,15 +415,15 @@ static struct pch_dma_desc *pdc_desc_get(struct pch_dma_chan *pd_chan)
}
dev_dbg(chan2dev(&pd_chan->chan), "desc %p not ACKed\n", desc);
}
spin_unlock_bh(&pd_chan->lock);
spin_unlock(&pd_chan->lock);
dev_dbg(chan2dev(&pd_chan->chan), "scanned %d descriptors\n", i);
if (!ret) {
ret = pdc_alloc_desc(&pd_chan->chan, GFP_NOIO);
if (ret) {
spin_lock_bh(&pd_chan->lock);
spin_lock(&pd_chan->lock);
pd_chan->descs_allocated++;
spin_unlock_bh(&pd_chan->lock);
spin_unlock(&pd_chan->lock);
} else {
dev_err(chan2dev(&pd_chan->chan),
"failed to alloc desc\n");
......@@ -437,10 +437,10 @@ static void pdc_desc_put(struct pch_dma_chan *pd_chan,
struct pch_dma_desc *desc)
{
if (desc) {
spin_lock_bh(&pd_chan->lock);
spin_lock(&pd_chan->lock);
list_splice_init(&desc->tx_list, &pd_chan->free_list);
list_add(&desc->desc_node, &pd_chan->free_list);
spin_unlock_bh(&pd_chan->lock);
spin_unlock(&pd_chan->lock);
}
}
......@@ -530,9 +530,9 @@ static void pd_issue_pending(struct dma_chan *chan)
struct pch_dma_chan *pd_chan = to_pd_chan(chan);
if (pdc_is_idle(pd_chan)) {
spin_lock_bh(&pd_chan->lock);
spin_lock(&pd_chan->lock);
pdc_advance_work(pd_chan);
spin_unlock_bh(&pd_chan->lock);
spin_unlock(&pd_chan->lock);
}
}
......@@ -592,7 +592,6 @@ static struct dma_async_tx_descriptor *pd_prep_slave_sg(struct dma_chan *chan,
goto err_desc_get;
}
if (!first) {
first = desc;
} else {
......@@ -641,13 +640,13 @@ static int pd_device_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
spin_unlock_bh(&pd_chan->lock);
return 0;
}
static void pdc_tasklet(unsigned long data)
{
struct pch_dma_chan *pd_chan = (struct pch_dma_chan *)data;
unsigned long flags;
if (!pdc_is_idle(pd_chan)) {
dev_err(chan2dev(&pd_chan->chan),
......@@ -655,12 +654,12 @@ static void pdc_tasklet(unsigned long data)
return;
}
spin_lock_bh(&pd_chan->lock);
spin_lock_irqsave(&pd_chan->lock, flags);
if (test_and_clear_bit(0, &pd_chan->err_status))
pdc_handle_error(pd_chan);
else
pdc_advance_work(pd_chan);
spin_unlock_bh(&pd_chan->lock);
spin_unlock_irqrestore(&pd_chan->lock, flags);
}
static irqreturn_t pd_irq(int irq, void *devid)
......@@ -694,6 +693,7 @@ static irqreturn_t pd_irq(int irq, void *devid)
return ret;
}
#ifdef CONFIG_PM
static void pch_dma_save_regs(struct pch_dma *pd)
{
struct pch_dma_chan *pd_chan;
......@@ -771,6 +771,7 @@ static int pch_dma_resume(struct pci_dev *pdev)
return 0;
}
#endif
static int __devinit pch_dma_probe(struct pci_dev *pdev,
const struct pci_device_id *id)
......
drivers/dma/ste_dma40.c
View file @ 6447f55d
......@@ -68,6 +68,7 @@ enum d40_command {
* @base: Pointer to memory area when the pre_alloc_lli's are not large
* enough, IE bigger than the most common case, 1 dst and 1 src. NULL if
* pre_alloc_lli is used.
* @dma_addr: DMA address, if mapped
* @size: The size in bytes of the memory at base or the size of pre_alloc_lli.
* @pre_alloc_lli: Pre allocated area for the most common case of transfers,
* one buffer to one buffer.
......@@ -75,6 +76,7 @@ enum d40_command {
struct d40_lli_pool {
void *base;
int size;
dma_addr_t dma_addr;
/* Space for dst and src, plus an extra for padding */
u8 pre_alloc_lli[3 * sizeof(struct d40_phy_lli)];
};
......@@ -94,7 +96,6 @@ struct d40_lli_pool {
* during a transfer.
* @node: List entry.
* @is_in_client_list: true if the client owns this descriptor.
* @is_hw_linked: true if this job will automatically be continued for
* the previous one.
*
* This descriptor is used for both logical and physical transfers.
......@@ -114,7 +115,7 @@ struct d40_desc {
struct list_head node;
bool is_in_client_list;
bool is_hw_linked;
bool cyclic;
};
/**
......@@ -130,6 +131,7 @@ struct d40_desc {
*/
struct d40_lcla_pool {
void *base;
dma_addr_t dma_addr;
void *base_unaligned;
int pages;
spinlock_t lock;
......@@ -303,9 +305,37 @@ struct d40_reg_val {
unsigned int val;
};
static int d40_pool_lli_alloc(struct d40_desc *d40d,
int lli_len, bool is_log)
static struct device *chan2dev(struct d40_chan *d40c)
{
return &d40c->chan.dev->device;
}
static bool chan_is_physical(struct d40_chan *chan)
{
return chan->log_num == D40_PHY_CHAN;
}
static bool chan_is_logical(struct d40_chan *chan)
{
return !chan_is_physical(chan);
}
static void __iomem *chan_base(struct d40_chan *chan)
{
return chan->base->virtbase + D40_DREG_PCBASE +
chan->phy_chan->num * D40_DREG_PCDELTA;
}
#define d40_err(dev, format, arg...) \
dev_err(dev, "[%s] " format, __func__, ## arg)
#define chan_err(d40c, format, arg...) \
d40_err(chan2dev(d40c), format, ## arg)
static int d40_pool_lli_alloc(struct d40_chan *d40c, struct d40_desc *d40d,
int lli_len)
{
bool is_log = chan_is_logical(d40c);
u32 align;
void *base;
......@@ -319,7 +349,7 @@ static int d40_pool_lli_alloc(struct d40_desc *d40d,
d40d->lli_pool.size = sizeof(d40d->lli_pool.pre_alloc_lli);
d40d->lli_pool.base = NULL;
} else {
d40d->lli_pool.size = ALIGN(lli_len * 2 * align, align);
d40d->lli_pool.size = lli_len * 2 * align;
base = kmalloc(d40d->lli_pool.size + align, GFP_NOWAIT);
d40d->lli_pool.base = base;
......@@ -329,22 +359,37 @@ static int d40_pool_lli_alloc(struct d40_desc *d40d,
}
if (is_log) {
d40d->lli_log.src = PTR_ALIGN((struct d40_log_lli *) base,
align);
d40d->lli_log.dst = PTR_ALIGN(d40d->lli_log.src + lli_len,
align);
d40d->lli_log.src = PTR_ALIGN(base, align);
d40d->lli_log.dst = d40d->lli_log.src + lli_len;
d40d->lli_pool.dma_addr = 0;
} else {
d40d->lli_phy.src = PTR_ALIGN((struct d40_phy_lli *)base,
align);
d40d->lli_phy.dst = PTR_ALIGN(d40d->lli_phy.src + lli_len,
align);
d40d->lli_phy.src = PTR_ALIGN(base, align);
d40d->lli_phy.dst = d40d->lli_phy.src + lli_len;
d40d->lli_pool.dma_addr = dma_map_single(d40c->base->dev,
d40d->lli_phy.src,
d40d->lli_pool.size,
DMA_TO_DEVICE);
if (dma_mapping_error(d40c->base->dev,
d40d->lli_pool.dma_addr)) {
kfree(d40d->lli_pool.base);
d40d->lli_pool.base = NULL;
d40d->lli_pool.dma_addr = 0;
return -ENOMEM;
}
}
return 0;
}
static void d40_pool_lli_free(struct d40_desc *d40d)
static void d40_pool_lli_free(struct d40_chan *d40c, struct d40_desc *d40d)
{
if (d40d->lli_pool.dma_addr)
dma_unmap_single(d40c->base->dev, d40d->lli_pool.dma_addr,
d40d->lli_pool.size, DMA_TO_DEVICE);
kfree(d40d->lli_pool.base);
d40d->lli_pool.base = NULL;
d40d->lli_pool.size = 0;
......@@ -391,7 +436,7 @@ static int d40_lcla_free_all(struct d40_chan *d40c,
int i;
int ret = -EINVAL;
if (d40c->log_num == D40_PHY_CHAN)
if (chan_is_physical(d40c))
return 0;
spin_lock_irqsave(&d40c->base->lcla_pool.lock, flags);
......@@ -430,7 +475,7 @@ static struct d40_desc *d40_desc_get(struct d40_chan *d40c)
list_for_each_entry_safe(d, _d, &d40c->client, node)
if (async_tx_test_ack(&d->txd)) {
d40_pool_lli_free(d);
d40_pool_lli_free(d40c, d);
d40_desc_remove(d);
desc = d;
memset(desc, 0, sizeof(*desc));
......@@ -450,6 +495,7 @@ static struct d40_desc *d40_desc_get(struct d40_chan *d40c)
static void d40_desc_free(struct d40_chan *d40c, struct d40_desc *d40d)
{
d40_pool_lli_free(d40c, d40d);
d40_lcla_free_all(d40c, d40d);
kmem_cache_free(d40c->base->desc_slab, d40d);
}
......@@ -459,57 +505,128 @@ static void d40_desc_submit(struct d40_chan *d40c, struct d40_desc *desc)
list_add_tail(&desc->node, &d40c->active);
}
static void d40_desc_load(struct d40_chan *d40c, struct d40_desc *d40d)
static void d40_phy_lli_load(struct d40_chan *chan, struct d40_desc *desc)
{
int curr_lcla = -EINVAL, next_lcla;
struct d40_phy_lli *lli_dst = desc->lli_phy.dst;
struct d40_phy_lli *lli_src = desc->lli_phy.src;
void __iomem *base = chan_base(chan);
writel(lli_src->reg_cfg, base + D40_CHAN_REG_SSCFG);
writel(lli_src->reg_elt, base + D40_CHAN_REG_SSELT);
writel(lli_src->reg_ptr, base + D40_CHAN_REG_SSPTR);
writel(lli_src->reg_lnk, base + D40_CHAN_REG_SSLNK);
writel(lli_dst->reg_cfg, base + D40_CHAN_REG_SDCFG);
writel(lli_dst->reg_elt, base + D40_CHAN_REG_SDELT);
writel(lli_dst->reg_ptr, base + D40_CHAN_REG_SDPTR);
writel(lli_dst->reg_lnk, base + D40_CHAN_REG_SDLNK);
}
if (d40c->log_num == D40_PHY_CHAN) {
d40_phy_lli_write(d40c->base->virtbase,
d40c->phy_chan->num,
d40d->lli_phy.dst,
d40d->lli_phy.src);
d40d->lli_current = d40d->lli_len;
} else {
static void d40_log_lli_to_lcxa(struct d40_chan *chan, struct d40_desc *desc)
{
struct d40_lcla_pool *pool = &chan->base->lcla_pool;
struct d40_log_lli_bidir *lli = &desc->lli_log;
int lli_current = desc->lli_current;
int lli_len = desc->lli_len;
bool cyclic = desc->cyclic;
int curr_lcla = -EINVAL;
int first_lcla = 0;
bool linkback;
if ((d40d->lli_len - d40d->lli_current) > 1)
curr_lcla = d40_lcla_alloc_one(d40c, d40d);
/*
* We may have partially running cyclic transfers, in case we did't get
* enough LCLA entries.
*/
linkback = cyclic && lli_current == 0;
d40_log_lli_lcpa_write(d40c->lcpa,
&d40d->lli_log.dst[d40d->lli_current],
&d40d->lli_log.src[d40d->lli_current],
curr_lcla);
/*
* For linkback, we need one LCLA even with only one link, because we
* can't link back to the one in LCPA space
*/
if (linkback || (lli_len - lli_current > 1)) {
curr_lcla = d40_lcla_alloc_one(chan, desc);
first_lcla = curr_lcla;
}
d40d->lli_current++;
for (; d40d->lli_current < d40d->lli_len; d40d->lli_current++) {
struct d40_log_lli *lcla;
/*
* For linkback, we normally load the LCPA in the loop since we need to
* link it to the second LCLA and not the first. However, if we
* couldn't even get a first LCLA, then we have to run in LCPA and
* reload manually.
*/
if (!linkback || curr_lcla == -EINVAL) {
unsigned int flags = 0;
if (d40d->lli_current + 1 < d40d->lli_len)
next_lcla = d40_lcla_alloc_one(d40c, d40d);
else
next_lcla = -EINVAL;
if (curr_lcla == -EINVAL)
flags |= LLI_TERM_INT;
lcla = d40c->base->lcla_pool.base +
d40c->phy_chan->num * 1024 +
8 * curr_lcla * 2;
d40_log_lli_lcpa_write(chan->lcpa,
&lli->dst[lli_current],
&lli->src[lli_current],
curr_lcla,
flags);
lli_current++;
}
d40_log_lli_lcla_write(lcla,
&d40d->lli_log.dst[d40d->lli_current],
&d40d->lli_log.src[d40d->lli_current],
next_lcla);
if (curr_lcla < 0)
goto out;
(void) dma_map_single(d40c->base->dev, lcla,
2 * sizeof(struct d40_log_lli),
DMA_TO_DEVICE);
for (; lli_current < lli_len; lli_current++) {
unsigned int lcla_offset = chan->phy_chan->num * 1024 +
8 * curr_lcla * 2;
struct d40_log_lli *lcla = pool->base + lcla_offset;
unsigned int flags = 0;
int next_lcla;
curr_lcla = next_lcla;
if (lli_current + 1 < lli_len)
next_lcla = d40_lcla_alloc_one(chan, desc);
else
next_lcla = linkback ? first_lcla : -EINVAL;
if (curr_lcla == -EINVAL) {
d40d->lli_current++;
break;
}
if (cyclic || next_lcla == -EINVAL)
flags |= LLI_TERM_INT;
if (linkback && curr_lcla == first_lcla) {
/* First link goes in both LCPA and LCLA */
d40_log_lli_lcpa_write(chan->lcpa,
&lli->dst[lli_current],
&lli->src[lli_current],
next_lcla, flags);
}
/*
* One unused LCLA in the cyclic case if the very first
* next_lcla fails...
*/
d40_log_lli_lcla_write(lcla,
&lli->dst[lli_current],
&lli->src[lli_current],
next_lcla, flags);
dma_sync_single_range_for_device(chan->base->dev,
pool->dma_addr, lcla_offset,
2 * sizeof(struct d40_log_lli),
DMA_TO_DEVICE);
curr_lcla = next_lcla;
if (curr_lcla == -EINVAL || curr_lcla == first_lcla) {
lli_current++;
break;
}
}
out:
desc->lli_current = lli_current;
}
static void d40_desc_load(struct d40_chan *d40c, struct d40_desc *d40d)
{
if (chan_is_physical(d40c)) {
d40_phy_lli_load(d40c, d40d);
d40d->lli_current = d40d->lli_len;
} else
d40_log_lli_to_lcxa(d40c, d40d);
}
static struct d40_desc *d40_first_active_get(struct d40_chan *d40c)
......@@ -543,18 +660,6 @@ static struct d40_desc *d40_first_queued(struct d40_chan *d40c)
return d;
}
static struct d40_desc *d40_last_queued(struct d40_chan *d40c)
{
struct d40_desc *d;
if (list_empty(&d40c->queue))
return NULL;
list_for_each_entry(d, &d40c->queue, node)
if (list_is_last(&d->node, &d40c->queue))
break;
return d;
}
static int d40_psize_2_burst_size(bool is_log, int psize)
{
if (is_log) {
......@@ -666,9 +771,9 @@ static int d40_channel_execute_command(struct d40_chan *d40c,
}
if (i == D40_SUSPEND_MAX_IT) {
dev_err(&d40c->chan.dev->device,
"[%s]: unable to suspend the chl %d (log: %d) status %x\n",
__func__, d40c->phy_chan->num, d40c->log_num,
chan_err(d40c,
"unable to suspend the chl %d (log: %d) status %x\n",
d40c->phy_chan->num, d40c->log_num,
status);
dump_stack();
ret = -EBUSY;
......@@ -701,17 +806,45 @@ static void d40_term_all(struct d40_chan *d40c)
d40c->busy = false;
}
static void __d40_config_set_event(struct d40_chan *d40c, bool enable,
u32 event, int reg)
{
void __iomem *addr = chan_base(d40c) + reg;
int tries;
if (!enable) {
writel((D40_DEACTIVATE_EVENTLINE << D40_EVENTLINE_POS(event))
| ~D40_EVENTLINE_MASK(event), addr);
return;
}
/*
* The hardware sometimes doesn't register the enable when src and dst
* event lines are active on the same logical channel. Retry to ensure
* it does. Usually only one retry is sufficient.
*/
tries = 100;
while (--tries) {
writel((D40_ACTIVATE_EVENTLINE << D40_EVENTLINE_POS(event))
| ~D40_EVENTLINE_MASK(event), addr);
if (readl(addr) & D40_EVENTLINE_MASK(event))
break;
}
if (tries != 99)
dev_dbg(chan2dev(d40c),
"[%s] workaround enable S%cLNK (%d tries)\n",
__func__, reg == D40_CHAN_REG_SSLNK ? 'S' : 'D',
100 - tries);
WARN_ON(!tries);
}
static void d40_config_set_event(struct d40_chan *d40c, bool do_enable)
{
u32 val;
unsigned long flags;
/* Notice, that disable requires the physical channel to be stopped */
if (do_enable)
val = D40_ACTIVATE_EVENTLINE;
else
val = D40_DEACTIVATE_EVENTLINE;
spin_lock_irqsave(&d40c->phy_chan->lock, flags);
/* Enable event line connected to device (or memcpy) */
......@@ -719,20 +852,15 @@ static void d40_config_set_event(struct d40_chan *d40c, bool do_enable)
(d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_PERIPH)) {
u32 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.src_dev_type);
writel((val << D40_EVENTLINE_POS(event)) |
~D40_EVENTLINE_MASK(event),
d40c->base->virtbase + D40_DREG_PCBASE +
d40c->phy_chan->num * D40_DREG_PCDELTA +
D40_CHAN_REG_SSLNK);
__d40_config_set_event(d40c, do_enable, event,
D40_CHAN_REG_SSLNK);
}
if (d40c->dma_cfg.dir != STEDMA40_PERIPH_TO_MEM) {
u32 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dst_dev_type);
writel((val << D40_EVENTLINE_POS(event)) |
~D40_EVENTLINE_MASK(event),
d40c->base->virtbase + D40_DREG_PCBASE +
d40c->phy_chan->num * D40_DREG_PCDELTA +
D40_CHAN_REG_SDLNK);
__d40_config_set_event(d40c, do_enable, event,
D40_CHAN_REG_SDLNK);
}
spin_unlock_irqrestore(&d40c->phy_chan->lock, flags);
......@@ -740,15 +868,12 @@ static void d40_config_set_event(struct d40_chan *d40c, bool do_enable)
static u32 d40_chan_has_events(struct d40_chan *d40c)
{
void __iomem *chanbase = chan_base(d40c);
u32 val;
val = readl(d40c->base->virtbase + D40_DREG_PCBASE +
d40c->phy_chan->num * D40_DREG_PCDELTA +
D40_CHAN_REG_SSLNK);
val = readl(chanbase + D40_CHAN_REG_SSLNK);
val |= readl(chanbase + D40_CHAN_REG_SDLNK);
val |= readl(d40c->base->virtbase + D40_DREG_PCBASE +
d40c->phy_chan->num * D40_DREG_PCDELTA +
D40_CHAN_REG_SDLNK);
return val;
}
......@@ -771,7 +896,7 @@ static u32 d40_get_prmo(struct d40_chan *d40c)
= D40_DREG_PRMO_LCHAN_SRC_LOG_DST_LOG,
};
if (d40c->log_num == D40_PHY_CHAN)
if (chan_is_physical(d40c))
return phy_map[d40c->dma_cfg.mode_opt];
else
return log_map[d40c->dma_cfg.mode_opt];
......@@ -785,7 +910,7 @@ static void d40_config_write(struct d40_chan *d40c)
/* Odd addresses are even addresses + 4 */
addr_base = (d40c->phy_chan->num % 2) * 4;
/* Setup channel mode to logical or physical */
var = ((u32)(d40c->log_num != D40_PHY_CHAN) + 1) <<
var = ((u32)(chan_is_logical(d40c)) + 1) <<
D40_CHAN_POS(d40c->phy_chan->num);
writel(var, d40c->base->virtbase + D40_DREG_PRMSE + addr_base);
......@@ -794,30 +919,18 @@ static void d40_config_write(struct d40_chan *d40c)
writel(var, d40c->base->virtbase + D40_DREG_PRMOE + addr_base);
if (d40c->log_num != D40_PHY_CHAN) {
if (chan_is_logical(d40c)) {
int lidx = (d40c->phy_chan->num << D40_SREG_ELEM_LOG_LIDX_POS)
& D40_SREG_ELEM_LOG_LIDX_MASK;
void __iomem *chanbase = chan_base(d40c);
/* Set default config for CFG reg */
writel(d40c->src_def_cfg,
d40c->base->virtbase + D40_DREG_PCBASE +
d40c->phy_chan->num * D40_DREG_PCDELTA +
D40_CHAN_REG_SSCFG);
writel(d40c->dst_def_cfg,
d40c->base->virtbase + D40_DREG_PCBASE +
d40c->phy_chan->num * D40_DREG_PCDELTA +
D40_CHAN_REG_SDCFG);
writel(d40c->src_def_cfg, chanbase + D40_CHAN_REG_SSCFG);
writel(d40c->dst_def_cfg, chanbase + D40_CHAN_REG_SDCFG);
/* Set LIDX for lcla */
writel((d40c->phy_chan->num << D40_SREG_ELEM_LOG_LIDX_POS) &
D40_SREG_ELEM_LOG_LIDX_MASK,
d40c->base->virtbase + D40_DREG_PCBASE +
d40c->phy_chan->num * D40_DREG_PCDELTA +
D40_CHAN_REG_SDELT);
writel((d40c->phy_chan->num << D40_SREG_ELEM_LOG_LIDX_POS) &
D40_SREG_ELEM_LOG_LIDX_MASK,
d40c->base->virtbase + D40_DREG_PCBASE +
d40c->phy_chan->num * D40_DREG_PCDELTA +
D40_CHAN_REG_SSELT);
writel(lidx, chanbase + D40_CHAN_REG_SSELT);
writel(lidx, chanbase + D40_CHAN_REG_SDELT);
}
}
......@@ -825,15 +938,15 @@ static u32 d40_residue(struct d40_chan *d40c)
{
u32 num_elt;
if (d40c->log_num != D40_PHY_CHAN)
if (chan_is_logical(d40c))
num_elt = (readl(&d40c->lcpa->lcsp2) & D40_MEM_LCSP2_ECNT_MASK)
>> D40_MEM_LCSP2_ECNT_POS;
else
num_elt = (readl(d40c->base->virtbase + D40_DREG_PCBASE +
d40c->phy_chan->num * D40_DREG_PCDELTA +
D40_CHAN_REG_SDELT) &
D40_SREG_ELEM_PHY_ECNT_MASK) >>
D40_SREG_ELEM_PHY_ECNT_POS;
else {
u32 val = readl(chan_base(d40c) + D40_CHAN_REG_SDELT);
num_elt = (val & D40_SREG_ELEM_PHY_ECNT_MASK)
>> D40_SREG_ELEM_PHY_ECNT_POS;
}
return num_elt * (1 << d40c->dma_cfg.dst_info.data_width);
}
......@@ -841,20 +954,17 @@ static bool d40_tx_is_linked(struct d40_chan *d40c)
{
bool is_link;
if (d40c->log_num != D40_PHY_CHAN)
if (chan_is_logical(d40c))
is_link = readl(&d40c->lcpa->lcsp3) & D40_MEM_LCSP3_DLOS_MASK;
else
is_link = readl(d40c->base->virtbase + D40_DREG_PCBASE +
d40c->phy_chan->num * D40_DREG_PCDELTA +
D40_CHAN_REG_SDLNK) &
D40_SREG_LNK_PHYS_LNK_MASK;
is_link = readl(chan_base(d40c) + D40_CHAN_REG_SDLNK)
& D40_SREG_LNK_PHYS_LNK_MASK;
return is_link;
}
static int d40_pause(struct dma_chan *chan)
static int d40_pause(struct d40_chan *d40c)
{
struct d40_chan *d40c =
container_of(chan, struct d40_chan, chan);
int res = 0;
unsigned long flags;
......@@ -865,7 +975,7 @@ static int d40_pause(struct dma_chan *chan)
res = d40_channel_execute_command(d40c, D40_DMA_SUSPEND_REQ);
if (res == 0) {
if (d40c->log_num != D40_PHY_CHAN) {
if (chan_is_logical(d40c)) {
d40_config_set_event(d40c, false);
/* Resume the other logical channels if any */
if (d40_chan_has_events(d40c))
......@@ -878,10 +988,8 @@ static int d40_pause(struct dma_chan *chan)
return res;
}
static int d40_resume(struct dma_chan *chan)
static int d40_resume(struct d40_chan *d40c)
{
struct d40_chan *d40c =
container_of(chan, struct d40_chan, chan);
int res = 0;
unsigned long flags;
......@@ -891,7 +999,7 @@ static int d40_resume(struct dma_chan *chan)
spin_lock_irqsave(&d40c->lock, flags);
if (d40c->base->rev == 0)
if (d40c->log_num != D40_PHY_CHAN) {
if (chan_is_logical(d40c)) {
res = d40_channel_execute_command(d40c,
D40_DMA_SUSPEND_REQ);
goto no_suspend;
......@@ -900,7 +1008,7 @@ static int d40_resume(struct dma_chan *chan)
/* If bytes left to transfer or linked tx resume job */
if (d40_residue(d40c) || d40_tx_is_linked(d40c)) {
if (d40c->log_num != D40_PHY_CHAN)
if (chan_is_logical(d40c))
d40_config_set_event(d40c, true);
res = d40_channel_execute_command(d40c, D40_DMA_RUN);
......@@ -911,75 +1019,20 @@ static int d40_resume(struct dma_chan *chan)
return res;
}
static void d40_tx_submit_log(struct d40_chan *d40c, struct d40_desc *d40d)
static int d40_terminate_all(struct d40_chan *chan)
{
/* TODO: Write */
}
static void d40_tx_submit_phy(struct d40_chan *d40c, struct d40_desc *d40d)
{
struct d40_desc *d40d_prev = NULL;
int i;
u32 val;
if (!list_empty(&d40c->queue))
d40d_prev = d40_last_queued(d40c);
else if (!list_empty(&d40c->active))
d40d_prev = d40_first_active_get(d40c);
if (!d40d_prev)
return;
/* Here we try to join this job with previous jobs */
val = readl(d40c->base->virtbase + D40_DREG_PCBASE +
d40c->phy_chan->num * D40_DREG_PCDELTA +
D40_CHAN_REG_SSLNK);
/* Figure out which link we're currently transmitting */
for (i = 0; i < d40d_prev->lli_len; i++)
if (val == d40d_prev->lli_phy.src[i].reg_lnk)
break;
val = readl(d40c->base->virtbase + D40_DREG_PCBASE +
d40c->phy_chan->num * D40_DREG_PCDELTA +
D40_CHAN_REG_SSELT) >> D40_SREG_ELEM_LOG_ECNT_POS;
if (i == (d40d_prev->lli_len - 1) && val > 0) {
/* Change the current one */
writel(virt_to_phys(d40d->lli_phy.src),
d40c->base->virtbase + D40_DREG_PCBASE +
d40c->phy_chan->num * D40_DREG_PCDELTA +
D40_CHAN_REG_SSLNK);
writel(virt_to_phys(d40d->lli_phy.dst),
d40c->base->virtbase + D40_DREG_PCBASE +
d40c->phy_chan->num * D40_DREG_PCDELTA +
D40_CHAN_REG_SDLNK);
d40d->is_hw_linked = true;
} else if (i < d40d_prev->lli_len) {
(void) dma_unmap_single(d40c->base->dev,
virt_to_phys(d40d_prev->lli_phy.src),
d40d_prev->lli_pool.size,
DMA_TO_DEVICE);
unsigned long flags;
int ret = 0;
/* Keep the settings */
val = d40d_prev->lli_phy.src[d40d_prev->lli_len - 1].reg_lnk &
~D40_SREG_LNK_PHYS_LNK_MASK;
d40d_prev->lli_phy.src[d40d_prev->lli_len - 1].reg_lnk =
val | virt_to_phys(d40d->lli_phy.src);
ret = d40_pause(chan);
if (!ret && chan_is_physical(chan))
ret = d40_channel_execute_command(chan, D40_DMA_STOP);
val = d40d_prev->lli_phy.dst[d40d_prev->lli_len - 1].reg_lnk &
~D40_SREG_LNK_PHYS_LNK_MASK;
d40d_prev->lli_phy.dst[d40d_prev->lli_len - 1].reg_lnk =
val | virt_to_phys(d40d->lli_phy.dst);
spin_lock_irqsave(&chan->lock, flags);
d40_term_all(chan);
spin_unlock_irqrestore(&chan->lock, flags);
(void) dma_map_single(d40c->base->dev,
d40d_prev->lli_phy.src,
d40d_prev->lli_pool.size,
DMA_TO_DEVICE);
d40d->is_hw_linked = true;
}
return ret;
}
static dma_cookie_t d40_tx_submit(struct dma_async_tx_descriptor *tx)
......@@ -990,8 +1043,6 @@ static dma_cookie_t d40_tx_submit(struct dma_async_tx_descriptor *tx)
struct d40_desc *d40d = container_of(tx, struct d40_desc, txd);
unsigned long flags;
(void) d40_pause(&d40c->chan);
spin_lock_irqsave(&d40c->lock, flags);
d40c->chan.cookie++;
......@@ -1001,17 +1052,10 @@ static dma_cookie_t d40_tx_submit(struct dma_async_tx_descriptor *tx)
d40d->txd.cookie = d40c->chan.cookie;
if (d40c->log_num == D40_PHY_CHAN)
d40_tx_submit_phy(d40c, d40d);
else
d40_tx_submit_log(d40c, d40d);
d40_desc_queue(d40c, d40d);
spin_unlock_irqrestore(&d40c->lock, flags);
(void) d40_resume(&d40c->chan);
return tx->cookie;
}
......@@ -1020,7 +1064,7 @@ static int d40_start(struct d40_chan *d40c)
if (d40c->base->rev == 0) {
int err;
if (d40c->log_num != D40_PHY_CHAN) {
if (chan_is_logical(d40c)) {
err = d40_channel_execute_command(d40c,
D40_DMA_SUSPEND_REQ);
if (err)
......@@ -1028,7 +1072,7 @@ static int d40_start(struct d40_chan *d40c)
}
}
if (d40c->log_num != D40_PHY_CHAN)
if (chan_is_logical(d40c))
d40_config_set_event(d40c, true);
return d40_channel_execute_command(d40c, D40_DMA_RUN);
......@@ -1051,21 +1095,14 @@ static struct d40_desc *d40_queue_start(struct d40_chan *d40c)
/* Add to active queue */
d40_desc_submit(d40c, d40d);
/*
* If this job is already linked in hw,
* do not submit it.
*/
if (!d40d->is_hw_linked) {
/* Initiate DMA job */
d40_desc_load(d40c, d40d);
/* Initiate DMA job */
d40_desc_load(d40c, d40d);
/* Start dma job */
err = d40_start(d40c);
/* Start dma job */
err = d40_start(d40c);
if (err)
return NULL;
}
if (err)
return NULL;
}
return d40d;
......@@ -1082,17 +1119,36 @@ static void dma_tc_handle(struct d40_chan *d40c)
if (d40d == NULL)
return;
d40_lcla_free_all(d40c, d40d);
if (d40d->cyclic) {
/*
* If this was a paritially loaded list, we need to reloaded
* it, and only when the list is completed. We need to check
* for done because the interrupt will hit for every link, and
* not just the last one.
*/
if (d40d->lli_current < d40d->lli_len
&& !d40_tx_is_linked(d40c)
&& !d40_residue(d40c)) {
d40_lcla_free_all(d40c, d40d);
d40_desc_load(d40c, d40d);
(void) d40_start(d40c);
if (d40d->lli_current < d40d->lli_len) {
d40_desc_load(d40c, d40d);
/* Start dma job */
(void) d40_start(d40c);
return;
}
if (d40d->lli_current == d40d->lli_len)
d40d->lli_current = 0;
}
} else {
d40_lcla_free_all(d40c, d40d);
if (d40_queue_start(d40c) == NULL)
d40c->busy = false;
if (d40d->lli_current < d40d->lli_len) {
d40_desc_load(d40c, d40d);
/* Start dma job */
(void) d40_start(d40c);
return;
}
if (d40_queue_start(d40c) == NULL)
d40c->busy = false;
}
d40c->pending_tx++;
tasklet_schedule(&d40c->tasklet);
......@@ -1111,11 +1167,11 @@ static void dma_tasklet(unsigned long data)
/* Get first active entry from list */
d40d = d40_first_active_get(d40c);
if (d40d == NULL)
goto err;
d40c->completed = d40d->txd.cookie;
if (!d40d->cyclic)
d40c->completed = d40d->txd.cookie;
/*
* If terminating a channel pending_tx is set to zero.
......@@ -1130,16 +1186,18 @@ static void dma_tasklet(unsigned long data)
callback = d40d->txd.callback;
callback_param = d40d->txd.callback_param;
if (async_tx_test_ack(&d40d->txd)) {
d40_pool_lli_free(d40d);
d40_desc_remove(d40d);
d40_desc_free(d40c, d40d);
} else {
if (!d40d->is_in_client_list) {
if (!d40d->cyclic) {
if (async_tx_test_ack(&d40d->txd)) {
d40_pool_lli_free(d40c, d40d);
d40_desc_remove(d40d);
d40_lcla_free_all(d40c, d40d);
list_add_tail(&d40d->node, &d40c->client);
d40d->is_in_client_list = true;
d40_desc_free(d40c, d40d);
} else {
if (!d40d->is_in_client_list) {
d40_desc_remove(d40d);
d40_lcla_free_all(d40c, d40d);
list_add_tail(&d40d->node, &d40c->client);
d40d->is_in_client_list = true;
}
}
}
......@@ -1216,9 +1274,8 @@ static irqreturn_t d40_handle_interrupt(int irq, void *data)
if (!il[row].is_error)
dma_tc_handle(d40c);
else
dev_err(base->dev,
"[%s] IRQ chan: %ld offset %d idx %d\n",
__func__, chan, il[row].offset, idx);
d40_err(base->dev, "IRQ chan: %ld offset %d idx %d\n",
chan, il[row].offset, idx);
spin_unlock(&d40c->lock);
}
......@@ -1237,8 +1294,7 @@ static int d40_validate_conf(struct d40_chan *d40c,
bool is_log = conf->mode == STEDMA40_MODE_LOGICAL;
if (!conf->dir) {
dev_err(&d40c->chan.dev->device, "[%s] Invalid direction.\n",
__func__);
chan_err(d40c, "Invalid direction.\n");
res = -EINVAL;
}
......@@ -1246,46 +1302,40 @@ static int d40_validate_conf(struct d40_chan *d40c,
d40c->base->plat_data->dev_tx[conf->dst_dev_type] == 0 &&
d40c->runtime_addr == 0) {
dev_err(&d40c->chan.dev->device,
"[%s] Invalid TX channel address (%d)\n",
__func__, conf->dst_dev_type);
chan_err(d40c, "Invalid TX channel address (%d)\n",
conf->dst_dev_type);
res = -EINVAL;
}
if (conf->src_dev_type != STEDMA40_DEV_SRC_MEMORY &&
d40c->base->plat_data->dev_rx[conf->src_dev_type] == 0 &&
d40c->runtime_addr == 0) {
dev_err(&d40c->chan.dev->device,
"[%s] Invalid RX channel address (%d)\n",
__func__, conf->src_dev_type);
chan_err(d40c, "Invalid RX channel address (%d)\n",
conf->src_dev_type);
res = -EINVAL;
}
if (conf->dir == STEDMA40_MEM_TO_PERIPH &&
dst_event_group == STEDMA40_DEV_DST_MEMORY) {
dev_err(&d40c->chan.dev->device, "[%s] Invalid dst\n",
__func__);
chan_err(d40c, "Invalid dst\n");
res = -EINVAL;
}
if (conf->dir == STEDMA40_PERIPH_TO_MEM &&
src_event_group == STEDMA40_DEV_SRC_MEMORY) {
dev_err(&d40c->chan.dev->device, "[%s] Invalid src\n",
__func__);
chan_err(d40c, "Invalid src\n");
res = -EINVAL;
}
if (src_event_group == STEDMA40_DEV_SRC_MEMORY &&
dst_event_group == STEDMA40_DEV_DST_MEMORY && is_log) {
dev_err(&d40c->chan.dev->device,
"[%s] No event line\n", __func__);
chan_err(d40c, "No event line\n");
res = -EINVAL;
}
if (conf->dir == STEDMA40_PERIPH_TO_PERIPH &&
(src_event_group != dst_event_group)) {
dev_err(&d40c->chan.dev->device,
"[%s] Invalid event group\n", __func__);
chan_err(d40c, "Invalid event group\n");
res = -EINVAL;
}
......@@ -1294,9 +1344,7 @@ static int d40_validate_conf(struct d40_chan *d40c,
* DMAC HW supports it. Will be added to this driver,
* in case any dma client requires it.
*/
dev_err(&d40c->chan.dev->device,
"[%s] periph to periph not supported\n",
__func__);
chan_err(d40c, "periph to periph not supported\n");
res = -EINVAL;
}
......@@ -1309,9 +1357,7 @@ static int d40_validate_conf(struct d40_chan *d40c,
* src (burst x width) == dst (burst x width)
*/
dev_err(&d40c->chan.dev->device,
"[%s] src (burst x width) != dst (burst x width)\n",
__func__);
chan_err(d40c, "src (burst x width) != dst (burst x width)\n");
res = -EINVAL;
}
......@@ -1514,8 +1560,7 @@ static int d40_config_memcpy(struct d40_chan *d40c)
dma_has_cap(DMA_SLAVE, cap)) {
d40c->dma_cfg = *d40c->base->plat_data->memcpy_conf_phy;
} else {
dev_err(&d40c->chan.dev->device, "[%s] No memcpy\n",
__func__);
chan_err(d40c, "No memcpy\n");
return -EINVAL;
}
......@@ -1540,21 +1585,19 @@ static int d40_free_dma(struct d40_chan *d40c)
/* Release client owned descriptors */
if (!list_empty(&d40c->client))
list_for_each_entry_safe(d, _d, &d40c->client, node) {
d40_pool_lli_free(d);
d40_pool_lli_free(d40c, d);
d40_desc_remove(d);
d40_desc_free(d40c, d);
}
if (phy == NULL) {
dev_err(&d40c->chan.dev->device, "[%s] phy == null\n",
__func__);
chan_err(d40c, "phy == null\n");
return -EINVAL;
}
if (phy->allocated_src == D40_ALLOC_FREE &&
phy->allocated_dst == D40_ALLOC_FREE) {
dev_err(&d40c->chan.dev->device, "[%s] channel already free\n",
__func__);
chan_err(d40c, "channel already free\n");
return -EINVAL;
}
......@@ -1566,19 +1609,17 @@ static int d40_free_dma(struct d40_chan *d40c)
event = D40_TYPE_TO_EVENT(d40c->dma_cfg.src_dev_type);
is_src = true;
} else {
dev_err(&d40c->chan.dev->device,
"[%s] Unknown direction\n", __func__);
chan_err(d40c, "Unknown direction\n");
return -EINVAL;
}
res = d40_channel_execute_command(d40c, D40_DMA_SUSPEND_REQ);
if (res) {
dev_err(&d40c->chan.dev->device, "[%s] suspend failed\n",
__func__);
chan_err(d40c, "suspend failed\n");
return res;
}
if (d40c->log_num != D40_PHY_CHAN) {
if (chan_is_logical(d40c)) {
/* Release logical channel, deactivate the event line */
d40_config_set_event(d40c, false);
......@@ -1594,9 +1635,8 @@ static int d40_free_dma(struct d40_chan *d40c)
res = d40_channel_execute_command(d40c,
D40_DMA_RUN);
if (res) {
dev_err(&d40c->chan.dev->device,
"[%s] Executing RUN command\n",
__func__);
chan_err(d40c,
"Executing RUN command\n");
return res;
}
}
......@@ -1609,8 +1649,7 @@ static int d40_free_dma(struct d40_chan *d40c)
/* Release physical channel */
res = d40_channel_execute_command(d40c, D40_DMA_STOP);
if (res) {
dev_err(&d40c->chan.dev->device,
"[%s] Failed to stop channel\n", __func__);
chan_err(d40c, "Failed to stop channel\n");
return res;
}
d40c->phy_chan = NULL;
......@@ -1622,6 +1661,7 @@ static int d40_free_dma(struct d40_chan *d40c)
static bool d40_is_paused(struct d40_chan *d40c)
{
void __iomem *chanbase = chan_base(d40c);
bool is_paused = false;
unsigned long flags;
void __iomem *active_reg;
......@@ -1630,7 +1670,7 @@ static bool d40_is_paused(struct d40_chan *d40c)
spin_lock_irqsave(&d40c->lock, flags);
if (d40c->log_num == D40_PHY_CHAN) {
if (chan_is_physical(d40c)) {
if (d40c->phy_chan->num % 2 == 0)
active_reg = d40c->base->virtbase + D40_DREG_ACTIVE;
else
......@@ -1648,17 +1688,12 @@ static bool d40_is_paused(struct d40_chan *d40c)
if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH ||
d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) {
event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dst_dev_type);
status = readl(d40c->base->virtbase + D40_DREG_PCBASE +
d40c->phy_chan->num * D40_DREG_PCDELTA +
D40_CHAN_REG_SDLNK);
status = readl(chanbase + D40_CHAN_REG_SDLNK);
} else if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) {
event = D40_TYPE_TO_EVENT(d40c->dma_cfg.src_dev_type);
status = readl(d40c->base->virtbase + D40_DREG_PCBASE +
d40c->phy_chan->num * D40_DREG_PCDELTA +
D40_CHAN_REG_SSLNK);
status = readl(chanbase + D40_CHAN_REG_SSLNK);
} else {
dev_err(&d40c->chan.dev->device,
"[%s] Unknown direction\n", __func__);
chan_err(d40c, "Unknown direction\n");
goto _exit;
}
......@@ -1688,114 +1723,184 @@ static u32 stedma40_residue(struct dma_chan *chan)
return bytes_left;
}
struct dma_async_tx_descriptor *stedma40_memcpy_sg(struct dma_chan *chan,
struct scatterlist *sgl_dst,
struct scatterlist *sgl_src,
unsigned int sgl_len,
unsigned long dma_flags)
static int
d40_prep_sg_log(struct d40_chan *chan, struct d40_desc *desc,
struct scatterlist *sg_src, struct scatterlist *sg_dst,
unsigned int sg_len, dma_addr_t src_dev_addr,
dma_addr_t dst_dev_addr)
{
int res;
struct d40_desc *d40d;
struct d40_chan *d40c = container_of(chan, struct d40_chan,
chan);
unsigned long flags;
struct stedma40_chan_cfg *cfg = &chan->dma_cfg;
struct stedma40_half_channel_info *src_info = &cfg->src_info;
struct stedma40_half_channel_info *dst_info = &cfg->dst_info;
int ret;
if (d40c->phy_chan == NULL) {
dev_err(&d40c->chan.dev->device,
"[%s] Unallocated channel.\n", __func__);
return ERR_PTR(-EINVAL);
}
ret = d40_log_sg_to_lli(sg_src, sg_len,
src_dev_addr,
desc->lli_log.src,
chan->log_def.lcsp1,
src_info->data_width,
dst_info->data_width);
spin_lock_irqsave(&d40c->lock, flags);
d40d = d40_desc_get(d40c);
ret = d40_log_sg_to_lli(sg_dst, sg_len,
dst_dev_addr,
desc->lli_log.dst,
chan->log_def.lcsp3,
dst_info->data_width,
src_info->data_width);
if (d40d == NULL)
return ret < 0 ? ret : 0;
}
static int
d40_prep_sg_phy(struct d40_chan *chan, struct d40_desc *desc,
struct scatterlist *sg_src, struct scatterlist *sg_dst,
unsigned int sg_len, dma_addr_t src_dev_addr,
dma_addr_t dst_dev_addr)
{
struct stedma40_chan_cfg *cfg = &chan->dma_cfg;
struct stedma40_half_channel_info *src_info = &cfg->src_info;
struct stedma40_half_channel_info *dst_info = &cfg->dst_info;
unsigned long flags = 0;
int ret;
if (desc->cyclic)
flags |= LLI_CYCLIC | LLI_TERM_INT;
ret = d40_phy_sg_to_lli(sg_src, sg_len, src_dev_addr,
desc->lli_phy.src,
virt_to_phys(desc->lli_phy.src),
chan->src_def_cfg,
src_info, dst_info, flags);
ret = d40_phy_sg_to_lli(sg_dst, sg_len, dst_dev_addr,
desc->lli_phy.dst,
virt_to_phys(desc->lli_phy.dst),
chan->dst_def_cfg,
dst_info, src_info, flags);
dma_sync_single_for_device(chan->base->dev, desc->lli_pool.dma_addr,
desc->lli_pool.size, DMA_TO_DEVICE);
return ret < 0 ? ret : 0;
}
static struct d40_desc *
d40_prep_desc(struct d40_chan *chan, struct scatterlist *sg,
unsigned int sg_len, unsigned long dma_flags)
{
struct stedma40_chan_cfg *cfg = &chan->dma_cfg;
struct d40_desc *desc;
int ret;
desc = d40_desc_get(chan);
if (!desc)
return NULL;
desc->lli_len = d40_sg_2_dmalen(sg, sg_len, cfg->src_info.data_width,
cfg->dst_info.data_width);
if (desc->lli_len < 0) {
chan_err(chan, "Unaligned size\n");
goto err;
}
d40d->lli_len = d40_sg_2_dmalen(sgl_dst, sgl_len,
d40c->dma_cfg.src_info.data_width,
d40c->dma_cfg.dst_info.data_width);
if (d40d->lli_len < 0) {
dev_err(&d40c->chan.dev->device,
"[%s] Unaligned size\n", __func__);
ret = d40_pool_lli_alloc(chan, desc, desc->lli_len);
if (ret < 0) {
chan_err(chan, "Could not allocate lli\n");
goto err;
}
d40d->lli_current = 0;
d40d->txd.flags = dma_flags;
if (d40c->log_num != D40_PHY_CHAN) {
desc->lli_current = 0;
desc->txd.flags = dma_flags;
desc->txd.tx_submit = d40_tx_submit;
if (d40_pool_lli_alloc(d40d, d40d->lli_len, true) < 0) {
dev_err(&d40c->chan.dev->device,
"[%s] Out of memory\n", __func__);
goto err;
}
dma_async_tx_descriptor_init(&desc->txd, &chan->chan);
(void) d40_log_sg_to_lli(sgl_src,
sgl_len,
d40d->lli_log.src,
d40c->log_def.lcsp1,
d40c->dma_cfg.src_info.data_width,
d40c->dma_cfg.dst_info.data_width);
(void) d40_log_sg_to_lli(sgl_dst,
sgl_len,
d40d->lli_log.dst,
d40c->log_def.lcsp3,
d40c->dma_cfg.dst_info.data_width,
d40c->dma_cfg.src_info.data_width);
} else {
if (d40_pool_lli_alloc(d40d, d40d->lli_len, false) < 0) {
dev_err(&d40c->chan.dev->device,
"[%s] Out of memory\n", __func__);
goto err;
}
return desc;
err:
d40_desc_free(chan, desc);
return NULL;
}
static dma_addr_t
d40_get_dev_addr(struct d40_chan *chan, enum dma_data_direction direction)
{
struct stedma40_platform_data *plat = chan->base->plat_data;
struct stedma40_chan_cfg *cfg = &chan->dma_cfg;
dma_addr_t addr;
res = d40_phy_sg_to_lli(sgl_src,
sgl_len,
0,
d40d->lli_phy.src,
virt_to_phys(d40d->lli_phy.src),
d40c->src_def_cfg,
d40c->dma_cfg.src_info.data_width,
d40c->dma_cfg.dst_info.data_width,
d40c->dma_cfg.src_info.psize);
if (chan->runtime_addr)
return chan->runtime_addr;
if (res < 0)
goto err;
if (direction == DMA_FROM_DEVICE)
addr = plat->dev_rx[cfg->src_dev_type];
else if (direction == DMA_TO_DEVICE)
addr = plat->dev_tx[cfg->dst_dev_type];
res = d40_phy_sg_to_lli(sgl_dst,
sgl_len,
0,
d40d->lli_phy.dst,
virt_to_phys(d40d->lli_phy.dst),
d40c->dst_def_cfg,
d40c->dma_cfg.dst_info.data_width,
d40c->dma_cfg.src_info.data_width,
d40c->dma_cfg.dst_info.psize);
return addr;
}
if (res < 0)
goto err;
static struct dma_async_tx_descriptor *
d40_prep_sg(struct dma_chan *dchan, struct scatterlist *sg_src,
struct scatterlist *sg_dst, unsigned int sg_len,
enum dma_data_direction direction, unsigned long dma_flags)
{
struct d40_chan *chan = container_of(dchan, struct d40_chan, chan);
dma_addr_t src_dev_addr = 0;
dma_addr_t dst_dev_addr = 0;
struct d40_desc *desc;
unsigned long flags;
int ret;
(void) dma_map_single(d40c->base->dev, d40d->lli_phy.src,
d40d->lli_pool.size, DMA_TO_DEVICE);
if (!chan->phy_chan) {
chan_err(chan, "Cannot prepare unallocated channel\n");
return NULL;
}
dma_async_tx_descriptor_init(&d40d->txd, chan);
d40d->txd.tx_submit = d40_tx_submit;
spin_lock_irqsave(&chan->lock, flags);
spin_unlock_irqrestore(&d40c->lock, flags);
desc = d40_prep_desc(chan, sg_src, sg_len, dma_flags);
if (desc == NULL)
goto err;
if (sg_next(&sg_src[sg_len - 1]) == sg_src)
desc->cyclic = true;
if (direction != DMA_NONE) {
dma_addr_t dev_addr = d40_get_dev_addr(chan, direction);
if (direction == DMA_FROM_DEVICE)
src_dev_addr = dev_addr;
else if (direction == DMA_TO_DEVICE)
dst_dev_addr = dev_addr;
}
if (chan_is_logical(chan))
ret = d40_prep_sg_log(chan, desc, sg_src, sg_dst,
sg_len, src_dev_addr, dst_dev_addr);
else
ret = d40_prep_sg_phy(chan, desc, sg_src, sg_dst,
sg_len, src_dev_addr, dst_dev_addr);
if (ret) {
chan_err(chan, "Failed to prepare %s sg job: %d\n",
chan_is_logical(chan) ? "log" : "phy", ret);
goto err;
}
spin_unlock_irqrestore(&chan->lock, flags);
return &desc->txd;
return &d40d->txd;
err:
if (d40d)
d40_desc_free(d40c, d40d);
spin_unlock_irqrestore(&d40c->lock, flags);
if (desc)
d40_desc_free(chan, desc);
spin_unlock_irqrestore(&chan->lock, flags);
return NULL;
}
EXPORT_SYMBOL(stedma40_memcpy_sg);
bool stedma40_filter(struct dma_chan *chan, void *data)
{
......@@ -1818,6 +1923,38 @@ bool stedma40_filter(struct dma_chan *chan, void *data)
}
EXPORT_SYMBOL(stedma40_filter);
static void __d40_set_prio_rt(struct d40_chan *d40c, int dev_type, bool src)
{
bool realtime = d40c->dma_cfg.realtime;
bool highprio = d40c->dma_cfg.high_priority;
u32 prioreg = highprio ? D40_DREG_PSEG1 : D40_DREG_PCEG1;
u32 rtreg = realtime ? D40_DREG_RSEG1 : D40_DREG_RCEG1;
u32 event = D40_TYPE_TO_EVENT(dev_type);
u32 group = D40_TYPE_TO_GROUP(dev_type);
u32 bit = 1 << event;
/* Destination event lines are stored in the upper halfword */
if (!src)
bit <<= 16;
writel(bit, d40c->base->virtbase + prioreg + group * 4);
writel(bit, d40c->base->virtbase + rtreg + group * 4);
}
static void d40_set_prio_realtime(struct d40_chan *d40c)
{
if (d40c->base->rev < 3)
return;
if ((d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) ||
(d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_PERIPH))
__d40_set_prio_rt(d40c, d40c->dma_cfg.src_dev_type, true);
if ((d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH) ||
(d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_PERIPH))
__d40_set_prio_rt(d40c, d40c->dma_cfg.dst_dev_type, false);
}
/* DMA ENGINE functions */
static int d40_alloc_chan_resources(struct dma_chan *chan)
{
......@@ -1834,9 +1971,7 @@ static int d40_alloc_chan_resources(struct dma_chan *chan)
if (!d40c->configured) {
err = d40_config_memcpy(d40c);
if (err) {
dev_err(&d40c->chan.dev->device,
"[%s] Failed to configure memcpy channel\n",
__func__);
chan_err(d40c, "Failed to configure memcpy channel\n");
goto fail;
}
}
......@@ -1844,16 +1979,17 @@ static int d40_alloc_chan_resources(struct dma_chan *chan)
err = d40_allocate_channel(d40c);
if (err) {
dev_err(&d40c->chan.dev->device,
"[%s] Failed to allocate channel\n", __func__);
chan_err(d40c, "Failed to allocate channel\n");
goto fail;
}
/* Fill in basic CFG register values */
d40_phy_cfg(&d40c->dma_cfg, &d40c->src_def_cfg,
&d40c->dst_def_cfg, d40c->log_num != D40_PHY_CHAN);
&d40c->dst_def_cfg, chan_is_logical(d40c));
if (d40c->log_num != D40_PHY_CHAN) {
d40_set_prio_realtime(d40c);
if (chan_is_logical(d40c)) {
d40_log_cfg(&d40c->dma_cfg,
&d40c->log_def.lcsp1, &d40c->log_def.lcsp3);
......@@ -1886,8 +2022,7 @@ static void d40_free_chan_resources(struct dma_chan *chan)
unsigned long flags;
if (d40c->phy_chan == NULL) {
dev_err(&d40c->chan.dev->device,
"[%s] Cannot free unallocated channel\n", __func__);
chan_err(d40c, "Cannot free unallocated channel\n");
return;
}
......@@ -1897,8 +2032,7 @@ static void d40_free_chan_resources(struct dma_chan *chan)
err = d40_free_dma(d40c);
if (err)
dev_err(&d40c->chan.dev->device,
"[%s] Failed to free channel\n", __func__);
chan_err(d40c, "Failed to free channel\n");
spin_unlock_irqrestore(&d40c->lock, flags);
}
......@@ -1908,251 +2042,31 @@ static struct dma_async_tx_descriptor *d40_prep_memcpy(struct dma_chan *chan,
size_t size,
unsigned long dma_flags)
{
struct d40_desc *d40d;
struct d40_chan *d40c = container_of(chan, struct d40_chan,
chan);
unsigned long flags;
if (d40c->phy_chan == NULL) {
dev_err(&d40c->chan.dev->device,
"[%s] Channel is not allocated.\n", __func__);
return ERR_PTR(-EINVAL);
}
spin_lock_irqsave(&d40c->lock, flags);
d40d = d40_desc_get(d40c);
if (d40d == NULL) {
dev_err(&d40c->chan.dev->device,
"[%s] Descriptor is NULL\n", __func__);
goto err;
}
struct scatterlist dst_sg;
struct scatterlist src_sg;
d40d->txd.flags = dma_flags;
d40d->lli_len = d40_size_2_dmalen(size,
d40c->dma_cfg.src_info.data_width,
d40c->dma_cfg.dst_info.data_width);
if (d40d->lli_len < 0) {
dev_err(&d40c->chan.dev->device,
"[%s] Unaligned size\n", __func__);
goto err;
}
sg_init_table(&dst_sg, 1);
sg_init_table(&src_sg, 1);
sg_dma_address(&dst_sg) = dst;
sg_dma_address(&src_sg) = src;
dma_async_tx_descriptor_init(&d40d->txd, chan);
sg_dma_len(&dst_sg) = size;
sg_dma_len(&src_sg) = size;
d40d->txd.tx_submit = d40_tx_submit;
if (d40c->log_num != D40_PHY_CHAN) {
if (d40_pool_lli_alloc(d40d, d40d->lli_len, true) < 0) {
dev_err(&d40c->chan.dev->device,
"[%s] Out of memory\n", __func__);
goto err;
}
d40d->lli_current = 0;
if (d40_log_buf_to_lli(d40d->lli_log.src,
src,
size,
d40c->log_def.lcsp1,
d40c->dma_cfg.src_info.data_width,
d40c->dma_cfg.dst_info.data_width,
true) == NULL)
goto err;
if (d40_log_buf_to_lli(d40d->lli_log.dst,
dst,
size,
d40c->log_def.lcsp3,
d40c->dma_cfg.dst_info.data_width,
d40c->dma_cfg.src_info.data_width,
true) == NULL)
goto err;
} else {
if (d40_pool_lli_alloc(d40d, d40d->lli_len, false) < 0) {
dev_err(&d40c->chan.dev->device,
"[%s] Out of memory\n", __func__);
goto err;
}
if (d40_phy_buf_to_lli(d40d->lli_phy.src,
src,
size,
d40c->dma_cfg.src_info.psize,
0,
d40c->src_def_cfg,
true,
d40c->dma_cfg.src_info.data_width,
d40c->dma_cfg.dst_info.data_width,
false) == NULL)
goto err;
if (d40_phy_buf_to_lli(d40d->lli_phy.dst,
dst,
size,
d40c->dma_cfg.dst_info.psize,
0,
d40c->dst_def_cfg,
true,
d40c->dma_cfg.dst_info.data_width,
d40c->dma_cfg.src_info.data_width,
false) == NULL)
goto err;
(void) dma_map_single(d40c->base->dev, d40d->lli_phy.src,
d40d->lli_pool.size, DMA_TO_DEVICE);
}
spin_unlock_irqrestore(&d40c->lock, flags);
return &d40d->txd;
err:
if (d40d)
d40_desc_free(d40c, d40d);
spin_unlock_irqrestore(&d40c->lock, flags);
return NULL;
return d40_prep_sg(chan, &src_sg, &dst_sg, 1, DMA_NONE, dma_flags);
}
static struct dma_async_tx_descriptor *
d40_prep_sg(struct dma_chan *chan,
struct scatterlist *dst_sg, unsigned int dst_nents,
struct scatterlist *src_sg, unsigned int src_nents,
unsigned long dma_flags)
d40_prep_memcpy_sg(struct dma_chan *chan,
struct scatterlist *dst_sg, unsigned int dst_nents,
struct scatterlist *src_sg, unsigned int src_nents,
unsigned long dma_flags)
{
if (dst_nents != src_nents)
return NULL;
return stedma40_memcpy_sg(chan, dst_sg, src_sg, dst_nents, dma_flags);
}
static int d40_prep_slave_sg_log(struct d40_desc *d40d,
struct d40_chan *d40c,
struct scatterlist *sgl,
unsigned int sg_len,
enum dma_data_direction direction,
unsigned long dma_flags)
{
dma_addr_t dev_addr = 0;
int total_size;
d40d->lli_len = d40_sg_2_dmalen(sgl, sg_len,
d40c->dma_cfg.src_info.data_width,
d40c->dma_cfg.dst_info.data_width);
if (d40d->lli_len < 0) {
dev_err(&d40c->chan.dev->device,
"[%s] Unaligned size\n", __func__);
return -EINVAL;
}
if (d40_pool_lli_alloc(d40d, d40d->lli_len, true) < 0) {
dev_err(&d40c->chan.dev->device,
"[%s] Out of memory\n", __func__);
return -ENOMEM;
}
d40d->lli_current = 0;
if (direction == DMA_FROM_DEVICE)
if (d40c->runtime_addr)
dev_addr = d40c->runtime_addr;
else
dev_addr = d40c->base->plat_data->dev_rx[d40c->dma_cfg.src_dev_type];
else if (direction == DMA_TO_DEVICE)
if (d40c->runtime_addr)
dev_addr = d40c->runtime_addr;
else
dev_addr = d40c->base->plat_data->dev_tx[d40c->dma_cfg.dst_dev_type];
else
return -EINVAL;
total_size = d40_log_sg_to_dev(sgl, sg_len,
&d40d->lli_log,
&d40c->log_def,
d40c->dma_cfg.src_info.data_width,
d40c->dma_cfg.dst_info.data_width,
direction,
dev_addr);
if (total_size < 0)
return -EINVAL;
return 0;
}
static int d40_prep_slave_sg_phy(struct d40_desc *d40d,
struct d40_chan *d40c,
struct scatterlist *sgl,
unsigned int sgl_len,
enum dma_data_direction direction,
unsigned long dma_flags)
{
dma_addr_t src_dev_addr;
dma_addr_t dst_dev_addr;
int res;
d40d->lli_len = d40_sg_2_dmalen(sgl, sgl_len,
d40c->dma_cfg.src_info.data_width,
d40c->dma_cfg.dst_info.data_width);
if (d40d->lli_len < 0) {
dev_err(&d40c->chan.dev->device,
"[%s] Unaligned size\n", __func__);
return -EINVAL;
}
if (d40_pool_lli_alloc(d40d, d40d->lli_len, false) < 0) {
dev_err(&d40c->chan.dev->device,
"[%s] Out of memory\n", __func__);
return -ENOMEM;
}
d40d->lli_current = 0;
if (direction == DMA_FROM_DEVICE) {
dst_dev_addr = 0;
if (d40c->runtime_addr)
src_dev_addr = d40c->runtime_addr;
else
src_dev_addr = d40c->base->plat_data->dev_rx[d40c->dma_cfg.src_dev_type];
} else if (direction == DMA_TO_DEVICE) {
if (d40c->runtime_addr)
dst_dev_addr = d40c->runtime_addr;
else
dst_dev_addr = d40c->base->plat_data->dev_tx[d40c->dma_cfg.dst_dev_type];
src_dev_addr = 0;
} else
return -EINVAL;
res = d40_phy_sg_to_lli(sgl,
sgl_len,
src_dev_addr,
d40d->lli_phy.src,
virt_to_phys(d40d->lli_phy.src),
d40c->src_def_cfg,
d40c->dma_cfg.src_info.data_width,
d40c->dma_cfg.dst_info.data_width,
d40c->dma_cfg.src_info.psize);
if (res < 0)
return res;
res = d40_phy_sg_to_lli(sgl,
sgl_len,
dst_dev_addr,
d40d->lli_phy.dst,
virt_to_phys(d40d->lli_phy.dst),
d40c->dst_def_cfg,
d40c->dma_cfg.dst_info.data_width,
d40c->dma_cfg.src_info.data_width,
d40c->dma_cfg.dst_info.psize);
if (res < 0)
return res;
(void) dma_map_single(d40c->base->dev, d40d->lli_phy.src,
d40d->lli_pool.size, DMA_TO_DEVICE);
return 0;
return d40_prep_sg(chan, src_sg, dst_sg, src_nents, DMA_NONE, dma_flags);
}
static struct dma_async_tx_descriptor *d40_prep_slave_sg(struct dma_chan *chan,
......@@ -2161,52 +2075,40 @@ static struct dma_async_tx_descriptor *d40_prep_slave_sg(struct dma_chan *chan,
enum dma_data_direction direction,
unsigned long dma_flags)
{
struct d40_desc *d40d;
struct d40_chan *d40c = container_of(chan, struct d40_chan,
chan);
unsigned long flags;
int err;
if (d40c->phy_chan == NULL) {
dev_err(&d40c->chan.dev->device,
"[%s] Cannot prepare unallocated channel\n", __func__);
return ERR_PTR(-EINVAL);
}
if (direction != DMA_FROM_DEVICE && direction != DMA_TO_DEVICE)
return NULL;
spin_lock_irqsave(&d40c->lock, flags);
d40d = d40_desc_get(d40c);
return d40_prep_sg(chan, sgl, sgl, sg_len, direction, dma_flags);
}
if (d40d == NULL)
goto err;
static struct dma_async_tx_descriptor *
dma40_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t dma_addr,
size_t buf_len, size_t period_len,
enum dma_data_direction direction)
{
unsigned int periods = buf_len / period_len;
struct dma_async_tx_descriptor *txd;
struct scatterlist *sg;
int i;
if (d40c->log_num != D40_PHY_CHAN)
err = d40_prep_slave_sg_log(d40d, d40c, sgl, sg_len,
direction, dma_flags);
else
err = d40_prep_slave_sg_phy(d40d, d40c, sgl, sg_len,
direction, dma_flags);
if (err) {
dev_err(&d40c->chan.dev->device,
"[%s] Failed to prepare %s slave sg job: %d\n",
__func__,
d40c->log_num != D40_PHY_CHAN ? "log" : "phy", err);
goto err;
sg = kcalloc(periods + 1, sizeof(struct scatterlist), GFP_KERNEL);
for (i = 0; i < periods; i++) {
sg_dma_address(&sg[i]) = dma_addr;
sg_dma_len(&sg[i]) = period_len;
dma_addr += period_len;
}
d40d->txd.flags = dma_flags;
sg[periods].offset = 0;
sg[periods].length = 0;
sg[periods].page_link =
((unsigned long)sg | 0x01) & ~0x02;
dma_async_tx_descriptor_init(&d40d->txd, chan);
txd = d40_prep_sg(chan, sg, sg, periods, direction,
DMA_PREP_INTERRUPT);
d40d->txd.tx_submit = d40_tx_submit;
kfree(sg);
spin_unlock_irqrestore(&d40c->lock, flags);
return &d40d->txd;
err:
if (d40d)
d40_desc_free(d40c, d40d);
spin_unlock_irqrestore(&d40c->lock, flags);
return NULL;
return txd;
}
static enum dma_status d40_tx_status(struct dma_chan *chan,
......@@ -2219,9 +2121,7 @@ static enum dma_status d40_tx_status(struct dma_chan *chan,
int ret;
if (d40c->phy_chan == NULL) {
dev_err(&d40c->chan.dev->device,
"[%s] Cannot read status of unallocated channel\n",
__func__);
chan_err(d40c, "Cannot read status of unallocated channel\n");
return -EINVAL;
}
......@@ -2245,8 +2145,7 @@ static void d40_issue_pending(struct dma_chan *chan)
unsigned long flags;
if (d40c->phy_chan == NULL) {
dev_err(&d40c->chan.dev->device,
"[%s] Channel is not allocated!\n", __func__);
chan_err(d40c, "Channel is not allocated!\n");
return;
}
......@@ -2339,7 +2238,7 @@ static void d40_set_runtime_config(struct dma_chan *chan,
return;
}
if (d40c->log_num != D40_PHY_CHAN) {
if (chan_is_logical(d40c)) {
if (config_maxburst >= 16)
psize = STEDMA40_PSIZE_LOG_16;
else if (config_maxburst >= 8)
......@@ -2372,7 +2271,7 @@ static void d40_set_runtime_config(struct dma_chan *chan,
cfg->dst_info.flow_ctrl = STEDMA40_NO_FLOW_CTRL;
/* Fill in register values */
if (d40c->log_num != D40_PHY_CHAN)
if (chan_is_logical(d40c))
d40_log_cfg(cfg, &d40c->log_def.lcsp1, &d40c->log_def.lcsp3);
else
d40_phy_cfg(cfg, &d40c->src_def_cfg,
......@@ -2393,25 +2292,20 @@ static void d40_set_runtime_config(struct dma_chan *chan,
static int d40_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
unsigned long arg)
{
unsigned long flags;
struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
if (d40c->phy_chan == NULL) {
dev_err(&d40c->chan.dev->device,
"[%s] Channel is not allocated!\n", __func__);
chan_err(d40c, "Channel is not allocated!\n");
return -EINVAL;
}
switch (cmd) {
case DMA_TERMINATE_ALL:
spin_lock_irqsave(&d40c->lock, flags);
d40_term_all(d40c);
spin_unlock_irqrestore(&d40c->lock, flags);
return 0;
return d40_terminate_all(d40c);
case DMA_PAUSE:
return d40_pause(chan);
return d40_pause(d40c);
case DMA_RESUME:
return d40_resume(chan);
return d40_resume(d40c);
case DMA_SLAVE_CONFIG:
d40_set_runtime_config(chan,
(struct dma_slave_config *) arg);
......@@ -2456,6 +2350,35 @@ static void __init d40_chan_init(struct d40_base *base, struct dma_device *dma,
}
}
static void d40_ops_init(struct d40_base *base, struct dma_device *dev)
{
if (dma_has_cap(DMA_SLAVE, dev->cap_mask))
dev->device_prep_slave_sg = d40_prep_slave_sg;
if (dma_has_cap(DMA_MEMCPY, dev->cap_mask)) {
dev->device_prep_dma_memcpy = d40_prep_memcpy;
/*
* This controller can only access address at even
* 32bit boundaries, i.e. 2^2
*/
dev->copy_align = 2;
}
if (dma_has_cap(DMA_SG, dev->cap_mask))
dev->device_prep_dma_sg = d40_prep_memcpy_sg;
if (dma_has_cap(DMA_CYCLIC, dev->cap_mask))
dev->device_prep_dma_cyclic = dma40_prep_dma_cyclic;
dev->device_alloc_chan_resources = d40_alloc_chan_resources;
dev->device_free_chan_resources = d40_free_chan_resources;
dev->device_issue_pending = d40_issue_pending;
dev->device_tx_status = d40_tx_status;
dev->device_control = d40_control;
dev->dev = base->dev;
}
static int __init d40_dmaengine_init(struct d40_base *base,
int num_reserved_chans)
{
......@@ -2466,23 +2389,14 @@ static int __init d40_dmaengine_init(struct d40_base *base,
dma_cap_zero(base->dma_slave.cap_mask);
dma_cap_set(DMA_SLAVE, base->dma_slave.cap_mask);
dma_cap_set(DMA_CYCLIC, base->dma_slave.cap_mask);
base->dma_slave.device_alloc_chan_resources = d40_alloc_chan_resources;
base->dma_slave.device_free_chan_resources = d40_free_chan_resources;
base->dma_slave.device_prep_dma_memcpy = d40_prep_memcpy;
base->dma_slave.device_prep_dma_sg = d40_prep_sg;
base->dma_slave.device_prep_slave_sg = d40_prep_slave_sg;
base->dma_slave.device_tx_status = d40_tx_status;
base->dma_slave.device_issue_pending = d40_issue_pending;
base->dma_slave.device_control = d40_control;
base->dma_slave.dev = base->dev;
d40_ops_init(base, &base->dma_slave);
err = dma_async_device_register(&base->dma_slave);
if (err) {
dev_err(base->dev,
"[%s] Failed to register slave channels\n",
__func__);
d40_err(base->dev, "Failed to register slave channels\n");
goto failure1;
}
......@@ -2491,29 +2405,15 @@ static int __init d40_dmaengine_init(struct d40_base *base,
dma_cap_zero(base->dma_memcpy.cap_mask);
dma_cap_set(DMA_MEMCPY, base->dma_memcpy.cap_mask);
dma_cap_set(DMA_SG, base->dma_slave.cap_mask);
base->dma_memcpy.device_alloc_chan_resources = d40_alloc_chan_resources;
base->dma_memcpy.device_free_chan_resources = d40_free_chan_resources;
base->dma_memcpy.device_prep_dma_memcpy = d40_prep_memcpy;
base->dma_slave.device_prep_dma_sg = d40_prep_sg;
base->dma_memcpy.device_prep_slave_sg = d40_prep_slave_sg;
base->dma_memcpy.device_tx_status = d40_tx_status;
base->dma_memcpy.device_issue_pending = d40_issue_pending;
base->dma_memcpy.device_control = d40_control;
base->dma_memcpy.dev = base->dev;
/*
* This controller can only access address at even
* 32bit boundaries, i.e. 2^2
*/
base->dma_memcpy.copy_align = 2;
dma_cap_set(DMA_SG, base->dma_memcpy.cap_mask);
d40_ops_init(base, &base->dma_memcpy);
err = dma_async_device_register(&base->dma_memcpy);
if (err) {
dev_err(base->dev,
"[%s] Failed to regsiter memcpy only channels\n",
__func__);
d40_err(base->dev,
"Failed to regsiter memcpy only channels\n");
goto failure2;
}
......@@ -2523,24 +2423,15 @@ static int __init d40_dmaengine_init(struct d40_base *base,
dma_cap_zero(base->dma_both.cap_mask);
dma_cap_set(DMA_SLAVE, base->dma_both.cap_mask);
dma_cap_set(DMA_MEMCPY, base->dma_both.cap_mask);
dma_cap_set(DMA_SG, base->dma_slave.cap_mask);
base->dma_both.device_alloc_chan_resources = d40_alloc_chan_resources;
base->dma_both.device_free_chan_resources = d40_free_chan_resources;
base->dma_both.device_prep_dma_memcpy = d40_prep_memcpy;
base->dma_slave.device_prep_dma_sg = d40_prep_sg;
base->dma_both.device_prep_slave_sg = d40_prep_slave_sg;
base->dma_both.device_tx_status = d40_tx_status;
base->dma_both.device_issue_pending = d40_issue_pending;
base->dma_both.device_control = d40_control;
base->dma_both.dev = base->dev;
base->dma_both.copy_align = 2;
dma_cap_set(DMA_SG, base->dma_both.cap_mask);
dma_cap_set(DMA_CYCLIC, base->dma_slave.cap_mask);
d40_ops_init(base, &base->dma_both);
err = dma_async_device_register(&base->dma_both);
if (err) {
dev_err(base->dev,
"[%s] Failed to register logical and physical capable channels\n",
__func__);
d40_err(base->dev,
"Failed to register logical and physical capable channels\n");
goto failure3;
}
return 0;
......@@ -2616,9 +2507,10 @@ static struct d40_base * __init d40_hw_detect_init(struct platform_device *pdev)
{ .reg = D40_DREG_PERIPHID1, .val = 0x0000},
/*
* D40_DREG_PERIPHID2 Depends on HW revision:
* MOP500/HREF ED has 0x0008,
* DB8500ed has 0x0008,
* ? has 0x0018,
* HREF V1 has 0x0028
* DB8500v1 has 0x0028
* DB8500v2 has 0x0038
*/
{ .reg = D40_DREG_PERIPHID3, .val = 0x0000},
......@@ -2642,8 +2534,7 @@ static struct d40_base * __init d40_hw_detect_init(struct platform_device *pdev)
clk = clk_get(&pdev->dev, NULL);
if (IS_ERR(clk)) {
dev_err(&pdev->dev, "[%s] No matching clock found\n",
__func__);
d40_err(&pdev->dev, "No matching clock found\n");
goto failure;
}
......@@ -2666,9 +2557,8 @@ static struct d40_base * __init d40_hw_detect_init(struct platform_device *pdev)
for (i = 0; i < ARRAY_SIZE(dma_id_regs); i++) {
if (dma_id_regs[i].val !=
readl(virtbase + dma_id_regs[i].reg)) {
dev_err(&pdev->dev,
"[%s] Unknown hardware! Expected 0x%x at 0x%x but got 0x%x\n",
__func__,
d40_err(&pdev->dev,
"Unknown hardware! Expected 0x%x at 0x%x but got 0x%x\n",
dma_id_regs[i].val,
dma_id_regs[i].reg,
readl(virtbase + dma_id_regs[i].reg));
......@@ -2681,9 +2571,8 @@ static struct d40_base * __init d40_hw_detect_init(struct platform_device *pdev)
if ((val & D40_DREG_PERIPHID2_DESIGNER_MASK) !=
D40_HW_DESIGNER) {
dev_err(&pdev->dev,
"[%s] Unknown designer! Got %x wanted %x\n",
__func__, val & D40_DREG_PERIPHID2_DESIGNER_MASK,
d40_err(&pdev->dev, "Unknown designer! Got %x wanted %x\n",
val & D40_DREG_PERIPHID2_DESIGNER_MASK,
D40_HW_DESIGNER);
goto failure;
}
......@@ -2713,7 +2602,7 @@ static struct d40_base * __init d40_hw_detect_init(struct platform_device *pdev)
sizeof(struct d40_chan), GFP_KERNEL);
if (base == NULL) {
dev_err(&pdev->dev, "[%s] Out of memory\n", __func__);
d40_err(&pdev->dev, "Out of memory\n");
goto failure;
}
......@@ -2860,6 +2749,7 @@ static void __init d40_hw_init(struct d40_base *base)
static int __init d40_lcla_allocate(struct d40_base *base)
{
struct d40_lcla_pool *pool = &base->lcla_pool;
unsigned long *page_list;
int i, j;
int ret = 0;
......@@ -2885,9 +2775,8 @@ static int __init d40_lcla_allocate(struct d40_base *base)
base->lcla_pool.pages);
if (!page_list[i]) {
dev_err(base->dev,
"[%s] Failed to allocate %d pages.\n",
__func__, base->lcla_pool.pages);
d40_err(base->dev, "Failed to allocate %d pages.\n",
base->lcla_pool.pages);
for (j = 0; j < i; j++)
free_pages(page_list[j], base->lcla_pool.pages);
......@@ -2925,6 +2814,15 @@ static int __init d40_lcla_allocate(struct d40_base *base)
LCLA_ALIGNMENT);
}
pool->dma_addr = dma_map_single(base->dev, pool->base,
SZ_1K * base->num_phy_chans,
DMA_TO_DEVICE);
if (dma_mapping_error(base->dev, pool->dma_addr)) {
pool->dma_addr = 0;
ret = -ENOMEM;
goto failure;
}
writel(virt_to_phys(base->lcla_pool.base),
base->virtbase + D40_DREG_LCLA);
failure:
......@@ -2957,9 +2855,7 @@ static int __init d40_probe(struct platform_device *pdev)
res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "lcpa");
if (!res) {
ret = -ENOENT;
dev_err(&pdev->dev,
"[%s] No \"lcpa\" memory resource\n",
__func__);
d40_err(&pdev->dev, "No \"lcpa\" memory resource\n");
goto failure;
}
base->lcpa_size = resource_size(res);
......@@ -2968,9 +2864,9 @@ static int __init d40_probe(struct platform_device *pdev)
if (request_mem_region(res->start, resource_size(res),
D40_NAME " I/O lcpa") == NULL) {
ret = -EBUSY;
dev_err(&pdev->dev,
"[%s] Failed to request LCPA region 0x%x-0x%x\n",
__func__, res->start, res->end);
d40_err(&pdev->dev,
"Failed to request LCPA region 0x%x-0x%x\n",
res->start, res->end);
goto failure;
}
......@@ -2986,16 +2882,13 @@ static int __init d40_probe(struct platform_device *pdev)
base->lcpa_base = ioremap(res->start, resource_size(res));
if (!base->lcpa_base) {
ret = -ENOMEM;
dev_err(&pdev->dev,
"[%s] Failed to ioremap LCPA region\n",
__func__);
d40_err(&pdev->dev, "Failed to ioremap LCPA region\n");
goto failure;
}
ret = d40_lcla_allocate(base);
if (ret) {
dev_err(&pdev->dev, "[%s] Failed to allocate LCLA area\n",
__func__);
d40_err(&pdev->dev, "Failed to allocate LCLA area\n");
goto failure;
}
......@@ -3004,9 +2897,8 @@ static int __init d40_probe(struct platform_device *pdev)
base->irq = platform_get_irq(pdev, 0);
ret = request_irq(base->irq, d40_handle_interrupt, 0, D40_NAME, base);
if (ret) {
dev_err(&pdev->dev, "[%s] No IRQ defined\n", __func__);
d40_err(&pdev->dev, "No IRQ defined\n");
goto failure;
}
......@@ -3025,6 +2917,12 @@ static int __init d40_probe(struct platform_device *pdev)
kmem_cache_destroy(base->desc_slab);
if (base->virtbase)
iounmap(base->virtbase);
if (base->lcla_pool.dma_addr)
dma_unmap_single(base->dev, base->lcla_pool.dma_addr,
SZ_1K * base->num_phy_chans,
DMA_TO_DEVICE);
if (!base->lcla_pool.base_unaligned && base->lcla_pool.base)
free_pages((unsigned long)base->lcla_pool.base,
base->lcla_pool.pages);
......@@ -3049,7 +2947,7 @@ static int __init d40_probe(struct platform_device *pdev)
kfree(base);
}
dev_err(&pdev->dev, "[%s] probe failed\n", __func__);
d40_err(&pdev->dev, "probe failed\n");
return ret;
}
......@@ -3060,7 +2958,7 @@ static struct platform_driver d40_driver = {
},
};
int __init stedma40_init(void)
static int __init stedma40_init(void)
{
return platform_driver_probe(&d40_driver, d40_probe);
}
......
drivers/dma/ste_dma40_ll.c
View file @ 6447f55d
......@@ -125,13 +125,15 @@ void d40_phy_cfg(struct stedma40_chan_cfg *cfg,
static int d40_phy_fill_lli(struct d40_phy_lli *lli,
dma_addr_t data,
u32 data_size,
int psize,
dma_addr_t next_lli,
u32 reg_cfg,
bool term_int,
u32 data_width,
bool is_device)
struct stedma40_half_channel_info *info,
unsigned int flags)
{
bool addr_inc = flags & LLI_ADDR_INC;
bool term_int = flags & LLI_TERM_INT;
unsigned int data_width = info->data_width;
int psize = info->psize;
int num_elems;
if (psize == STEDMA40_PSIZE_PHY_1)
......@@ -154,7 +156,7 @@ static int d40_phy_fill_lli(struct d40_phy_lli *lli,
* Distance to next element sized entry.
* Usually the size of the element unless you want gaps.
*/
if (!is_device)
if (addr_inc)
lli->reg_elt |= (0x1 << data_width) <<
D40_SREG_ELEM_PHY_EIDX_POS;
......@@ -198,47 +200,51 @@ static int d40_seg_size(int size, int data_width1, int data_width2)
return seg_max;
}
struct d40_phy_lli *d40_phy_buf_to_lli(struct d40_phy_lli *lli,
dma_addr_t addr,
u32 size,
int psize,
dma_addr_t lli_phys,
u32 reg_cfg,
bool term_int,
u32 data_width1,
u32 data_width2,
bool is_device)
static struct d40_phy_lli *
d40_phy_buf_to_lli(struct d40_phy_lli *lli, dma_addr_t addr, u32 size,
dma_addr_t lli_phys, dma_addr_t first_phys, u32 reg_cfg,
struct stedma40_half_channel_info *info,
struct stedma40_half_channel_info *otherinfo,
unsigned long flags)
{
bool lastlink = flags & LLI_LAST_LINK;
bool addr_inc = flags & LLI_ADDR_INC;
bool term_int = flags & LLI_TERM_INT;
bool cyclic = flags & LLI_CYCLIC;
int err;
dma_addr_t next = lli_phys;
int size_rest = size;
int size_seg = 0;
/*
* This piece may be split up based on d40_seg_size(); we only want the
* term int on the last part.
*/
if (term_int)
flags &= ~LLI_TERM_INT;
do {
size_seg = d40_seg_size(size_rest, data_width1, data_width2);
size_seg = d40_seg_size(size_rest, info->data_width,
otherinfo->data_width);
size_rest -= size_seg;
if (term_int && size_rest == 0)
next = 0;
if (size_rest == 0 && term_int)
flags |= LLI_TERM_INT;
if (size_rest == 0 && lastlink)
next = cyclic ? first_phys : 0;
else
next = ALIGN(next + sizeof(struct d40_phy_lli),
D40_LLI_ALIGN);
err = d40_phy_fill_lli(lli,
addr,
size_seg,
psize,
next,
reg_cfg,
!next,
data_width1,
is_device);
err = d40_phy_fill_lli(lli, addr, size_seg, next,
reg_cfg, info, flags);
if (err)
goto err;
lli++;
if (!is_device)
if (addr_inc)
addr += size_seg;
} while (size_rest);
......@@ -254,39 +260,35 @@ int d40_phy_sg_to_lli(struct scatterlist *sg,
struct d40_phy_lli *lli_sg,
dma_addr_t lli_phys,
u32 reg_cfg,
u32 data_width1,
u32 data_width2,
int psize)
struct stedma40_half_channel_info *info,
struct stedma40_half_channel_info *otherinfo,
unsigned long flags)
{
int total_size = 0;
int i;
struct scatterlist *current_sg = sg;
dma_addr_t dst;
struct d40_phy_lli *lli = lli_sg;
dma_addr_t l_phys = lli_phys;
if (!target)
flags |= LLI_ADDR_INC;
for_each_sg(sg, current_sg, sg_len, i) {
dma_addr_t sg_addr = sg_dma_address(current_sg);
unsigned int len = sg_dma_len(current_sg);
dma_addr_t dst = target ?: sg_addr;
total_size += sg_dma_len(current_sg);
if (target)
dst = target;
else
dst = sg_phys(current_sg);
if (i == sg_len - 1)
flags |= LLI_TERM_INT | LLI_LAST_LINK;
l_phys = ALIGN(lli_phys + (lli - lli_sg) *
sizeof(struct d40_phy_lli), D40_LLI_ALIGN);
lli = d40_phy_buf_to_lli(lli,
dst,
sg_dma_len(current_sg),
psize,
l_phys,
reg_cfg,
sg_len - 1 == i,
data_width1,
data_width2,
target == dst);
lli = d40_phy_buf_to_lli(lli, dst, len, l_phys, lli_phys,
reg_cfg, info, otherinfo, flags);
if (lli == NULL)
return -EINVAL;
}
......@@ -295,45 +297,22 @@ int d40_phy_sg_to_lli(struct scatterlist *sg,
}
void d40_phy_lli_write(void __iomem *virtbase,
u32 phy_chan_num,
struct d40_phy_lli *lli_dst,
struct d40_phy_lli *lli_src)
{
writel(lli_src->reg_cfg, virtbase + D40_DREG_PCBASE +
phy_chan_num * D40_DREG_PCDELTA + D40_CHAN_REG_SSCFG);
writel(lli_src->reg_elt, virtbase + D40_DREG_PCBASE +
phy_chan_num * D40_DREG_PCDELTA + D40_CHAN_REG_SSELT);
writel(lli_src->reg_ptr, virtbase + D40_DREG_PCBASE +
phy_chan_num * D40_DREG_PCDELTA + D40_CHAN_REG_SSPTR);
writel(lli_src->reg_lnk, virtbase + D40_DREG_PCBASE +
phy_chan_num * D40_DREG_PCDELTA + D40_CHAN_REG_SSLNK);
writel(lli_dst->reg_cfg, virtbase + D40_DREG_PCBASE +
phy_chan_num * D40_DREG_PCDELTA + D40_CHAN_REG_SDCFG);
writel(lli_dst->reg_elt, virtbase + D40_DREG_PCBASE +
phy_chan_num * D40_DREG_PCDELTA + D40_CHAN_REG_SDELT);
writel(lli_dst->reg_ptr, virtbase + D40_DREG_PCBASE +
phy_chan_num * D40_DREG_PCDELTA + D40_CHAN_REG_SDPTR);
writel(lli_dst->reg_lnk, virtbase + D40_DREG_PCBASE +
phy_chan_num * D40_DREG_PCDELTA + D40_CHAN_REG_SDLNK);
}
/* DMA logical lli operations */
static void d40_log_lli_link(struct d40_log_lli *lli_dst,
struct d40_log_lli *lli_src,
int next)
int next, unsigned int flags)
{
bool interrupt = flags & LLI_TERM_INT;
u32 slos = 0;
u32 dlos = 0;
if (next != -EINVAL) {
slos = next * 2;
dlos = next * 2 + 1;
} else {
}
if (interrupt) {
lli_dst->lcsp13 |= D40_MEM_LCSP1_SCFG_TIM_MASK;
lli_dst->lcsp13 |= D40_MEM_LCSP3_DTCP_MASK;
}
......@@ -348,9 +327,9 @@ static void d40_log_lli_link(struct d40_log_lli *lli_dst,
void d40_log_lli_lcpa_write(struct d40_log_lli_full *lcpa,
struct d40_log_lli *lli_dst,
struct d40_log_lli *lli_src,
int next)
int next, unsigned int flags)
{
d40_log_lli_link(lli_dst, lli_src, next);
d40_log_lli_link(lli_dst, lli_src, next, flags);
writel(lli_src->lcsp02, &lcpa[0].lcsp0);
writel(lli_src->lcsp13, &lcpa[0].lcsp1);
......@@ -361,9 +340,9 @@ void d40_log_lli_lcpa_write(struct d40_log_lli_full *lcpa,
void d40_log_lli_lcla_write(struct d40_log_lli *lcla,
struct d40_log_lli *lli_dst,
struct d40_log_lli *lli_src,
int next)
int next, unsigned int flags)
{
d40_log_lli_link(lli_dst, lli_src, next);
d40_log_lli_link(lli_dst, lli_src, next, flags);
writel(lli_src->lcsp02, &lcla[0].lcsp02);
writel(lli_src->lcsp13, &lcla[0].lcsp13);
......@@ -375,8 +354,10 @@ static void d40_log_fill_lli(struct d40_log_lli *lli,
dma_addr_t data, u32 data_size,
u32 reg_cfg,
u32 data_width,
bool addr_inc)
unsigned int flags)
{
bool addr_inc = flags & LLI_ADDR_INC;
lli->lcsp13 = reg_cfg;
/* The number of elements to transfer */
......@@ -395,67 +376,15 @@ static void d40_log_fill_lli(struct d40_log_lli *lli,
}
int d40_log_sg_to_dev(struct scatterlist *sg,
int sg_len,
struct d40_log_lli_bidir *lli,
struct d40_def_lcsp *lcsp,
u32 src_data_width,
u32 dst_data_width,
enum dma_data_direction direction,
dma_addr_t dev_addr)
{
int total_size = 0;
struct scatterlist *current_sg = sg;
int i;
struct d40_log_lli *lli_src = lli->src;
struct d40_log_lli *lli_dst = lli->dst;
for_each_sg(sg, current_sg, sg_len, i) {
total_size += sg_dma_len(current_sg);
if (direction == DMA_TO_DEVICE) {
lli_src =
d40_log_buf_to_lli(lli_src,
sg_phys(current_sg),
sg_dma_len(current_sg),
lcsp->lcsp1, src_data_width,
dst_data_width,
true);
lli_dst =
d40_log_buf_to_lli(lli_dst,
dev_addr,
sg_dma_len(current_sg),
lcsp->lcsp3, dst_data_width,
src_data_width,
false);
} else {
lli_dst =
d40_log_buf_to_lli(lli_dst,
sg_phys(current_sg),
sg_dma_len(current_sg),
lcsp->lcsp3, dst_data_width,
src_data_width,
true);
lli_src =
d40_log_buf_to_lli(lli_src,
dev_addr,
sg_dma_len(current_sg),
lcsp->lcsp1, src_data_width,
dst_data_width,
false);
}
}
return total_size;
}
struct d40_log_lli *d40_log_buf_to_lli(struct d40_log_lli *lli_sg,
static struct d40_log_lli *d40_log_buf_to_lli(struct d40_log_lli *lli_sg,
dma_addr_t addr,
int size,
u32 lcsp13, /* src or dst*/
u32 data_width1,
u32 data_width2,
bool addr_inc)
unsigned int flags)
{
bool addr_inc = flags & LLI_ADDR_INC;
struct d40_log_lli *lli = lli_sg;
int size_rest = size;
int size_seg = 0;
......@@ -468,7 +397,7 @@ struct d40_log_lli *d40_log_buf_to_lli(struct d40_log_lli *lli_sg,
addr,
size_seg,
lcsp13, data_width1,
addr_inc);
flags);
if (addr_inc)
addr += size_seg;
lli++;
......@@ -479,6 +408,7 @@ struct d40_log_lli *d40_log_buf_to_lli(struct d40_log_lli *lli_sg,
int d40_log_sg_to_lli(struct scatterlist *sg,
int sg_len,
dma_addr_t dev_addr,
struct d40_log_lli *lli_sg,
u32 lcsp13, /* src or dst*/
u32 data_width1, u32 data_width2)
......@@ -487,14 +417,24 @@ int d40_log_sg_to_lli(struct scatterlist *sg,
struct scatterlist *current_sg = sg;
int i;
struct d40_log_lli *lli = lli_sg;
unsigned long flags = 0;
if (!dev_addr)
flags |= LLI_ADDR_INC;
for_each_sg(sg, current_sg, sg_len, i) {
dma_addr_t sg_addr = sg_dma_address(current_sg);
unsigned int len = sg_dma_len(current_sg);
dma_addr_t addr = dev_addr ?: sg_addr;
total_size += sg_dma_len(current_sg);
lli = d40_log_buf_to_lli(lli,
sg_phys(current_sg),
sg_dma_len(current_sg),
lli = d40_log_buf_to_lli(lli, addr, len,
lcsp13,
data_width1, data_width2, true);
data_width1,
data_width2,
flags);
}
return total_size;
}
drivers/dma/ste_dma40_ll.h
View file @ 6447f55d
......@@ -163,6 +163,22 @@
#define D40_DREG_LCEIS1 0x0B4
#define D40_DREG_LCEIS2 0x0B8
#define D40_DREG_LCEIS3 0x0BC
#define D40_DREG_PSEG1 0x110
#define D40_DREG_PSEG2 0x114
#define D40_DREG_PSEG3 0x118
#define D40_DREG_PSEG4 0x11C
#define D40_DREG_PCEG1 0x120
#define D40_DREG_PCEG2 0x124
#define D40_DREG_PCEG3 0x128
#define D40_DREG_PCEG4 0x12C
#define D40_DREG_RSEG1 0x130
#define D40_DREG_RSEG2 0x134
#define D40_DREG_RSEG3 0x138
#define D40_DREG_RSEG4 0x13C
#define D40_DREG_RCEG1 0x140
#define D40_DREG_RCEG2 0x144
#define D40_DREG_RCEG3 0x148
#define D40_DREG_RCEG4 0x14C
#define D40_DREG_STFU 0xFC8
#define D40_DREG_ICFG 0xFCC
#define D40_DREG_PERIPHID0 0xFE0
......@@ -277,6 +293,13 @@ struct d40_def_lcsp {
/* Physical channels */
enum d40_lli_flags {
LLI_ADDR_INC = 1 << 0,
LLI_TERM_INT = 1 << 1,
LLI_CYCLIC = 1 << 2,
LLI_LAST_LINK = 1 << 3,
};
void d40_phy_cfg(struct stedma40_chan_cfg *cfg,
u32 *src_cfg,
u32 *dst_cfg,
......@@ -292,46 +315,15 @@ int d40_phy_sg_to_lli(struct scatterlist *sg,
struct d40_phy_lli *lli,
dma_addr_t lli_phys,
u32 reg_cfg,
u32 data_width1,
u32 data_width2,
int psize);
struct d40_phy_lli *d40_phy_buf_to_lli(struct d40_phy_lli *lli,
dma_addr_t data,
u32 data_size,
int psize,
dma_addr_t next_lli,
u32 reg_cfg,
bool term_int,
u32 data_width1,
u32 data_width2,
bool is_device);
void d40_phy_lli_write(void __iomem *virtbase,
u32 phy_chan_num,
struct d40_phy_lli *lli_dst,
struct d40_phy_lli *lli_src);
struct stedma40_half_channel_info *info,
struct stedma40_half_channel_info *otherinfo,
unsigned long flags);
/* Logical channels */
struct d40_log_lli *d40_log_buf_to_lli(struct d40_log_lli *lli_sg,
dma_addr_t addr,
int size,
u32 lcsp13, /* src or dst*/
u32 data_width1, u32 data_width2,
bool addr_inc);
int d40_log_sg_to_dev(struct scatterlist *sg,
int sg_len,
struct d40_log_lli_bidir *lli,
struct d40_def_lcsp *lcsp,
u32 src_data_width,
u32 dst_data_width,
enum dma_data_direction direction,
dma_addr_t dev_addr);
int d40_log_sg_to_lli(struct scatterlist *sg,
int sg_len,
dma_addr_t dev_addr,
struct d40_log_lli *lli_sg,
u32 lcsp13, /* src or dst*/
u32 data_width1, u32 data_width2);
......@@ -339,11 +331,11 @@ int d40_log_sg_to_lli(struct scatterlist *sg,
void d40_log_lli_lcpa_write(struct d40_log_lli_full *lcpa,
struct d40_log_lli *lli_dst,
struct d40_log_lli *lli_src,
int next);
int next, unsigned int flags);
void d40_log_lli_lcla_write(struct d40_log_lli *lcla,
struct d40_log_lli *lli_dst,
struct d40_log_lli *lli_src,
int next);
int next, unsigned int flags);
#endif /* STE_DMA40_LLI_H */
include/linux/dw_dmac.h
View file @ 6447f55d
......@@ -16,9 +16,18 @@
/**
* struct dw_dma_platform_data - Controller configuration parameters
* @nr_channels: Number of channels supported by hardware (max 8)
* @is_private: The device channels should be marked as private and not for
* by the general purpose DMA channel allocator.
*/
struct dw_dma_platform_data {
unsigned int nr_channels;
bool is_private;
#define CHAN_ALLOCATION_ASCENDING 0 /* zero to seven */
#define CHAN_ALLOCATION_DESCENDING 1 /* seven to zero */
unsigned char chan_allocation_order;
#define CHAN_PRIORITY_ASCENDING 0 /* chan0 highest */
#define CHAN_PRIORITY_DESCENDING 1 /* chan7 highest */
unsigned char chan_priority;
};
/**
......@@ -33,6 +42,30 @@ enum dw_dma_slave_width {
DW_DMA_SLAVE_WIDTH_32BIT,
};
/* bursts size */
enum dw_dma_msize {
DW_DMA_MSIZE_1,
DW_DMA_MSIZE_4,
DW_DMA_MSIZE_8,
DW_DMA_MSIZE_16,
DW_DMA_MSIZE_32,
DW_DMA_MSIZE_64,
DW_DMA_MSIZE_128,
DW_DMA_MSIZE_256,
};
/* flow controller */
enum dw_dma_fc {
DW_DMA_FC_D_M2M,
DW_DMA_FC_D_M2P,
DW_DMA_FC_D_P2M,
DW_DMA_FC_D_P2P,
DW_DMA_FC_P_P2M,
DW_DMA_FC_SP_P2P,
DW_DMA_FC_P_M2P,
DW_DMA_FC_DP_P2P,
};
/**
* struct dw_dma_slave - Controller-specific information about a slave
*
......@@ -44,6 +77,11 @@ enum dw_dma_slave_width {
* @reg_width: peripheral register width
* @cfg_hi: Platform-specific initializer for the CFG_HI register
* @cfg_lo: Platform-specific initializer for the CFG_LO register
* @src_master: src master for transfers on allocated channel.
* @dst_master: dest master for transfers on allocated channel.
* @src_msize: src burst size.
* @dst_msize: dest burst size.
* @fc: flow controller for DMA transfer
*/
struct dw_dma_slave {
struct device *dma_dev;
......@@ -52,6 +90,11 @@ struct dw_dma_slave {
enum dw_dma_slave_width reg_width;
u32 cfg_hi;
u32 cfg_lo;
u8 src_master;
u8 dst_master;
u8 src_msize;
u8 dst_msize;
u8 fc;
};
/* Platform-configurable bits in CFG_HI */
......@@ -62,7 +105,6 @@ struct dw_dma_slave {
#define DWC_CFGH_DST_PER(x) ((x) << 11)
/* Platform-configurable bits in CFG_LO */
#define DWC_CFGL_PRIO(x) ((x) << 5) /* priority */
#define DWC_CFGL_LOCK_CH_XFER (0 << 12) /* scope of LOCK_CH */
#define DWC_CFGL_LOCK_CH_BLOCK (1 << 12)
#define DWC_CFGL_LOCK_CH_XACT (2 << 12)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7