Commit d360f1ee authored by David S. Miller's avatar David S. Miller

[SPARC64]: Fix PCI IOMMU invalid iopte handling.

Instead of marking them as invalid, point them
at a dummy page.  This handles buggy third-party
bridges that erroneously prefetch sometimes.
Signed-off-by: default avatarDavid S. Miller <davem@redhat.com>
parent 14729dbe
......@@ -56,6 +56,39 @@ static void __iommu_flushall(struct pci_iommu *iommu)
}
}
#define IOPTE_CONSISTENT(CTX) \
(IOPTE_VALID | IOPTE_CACHE | \
(((CTX) << 47) & IOPTE_CONTEXT))
#define IOPTE_STREAMING(CTX) \
(IOPTE_CONSISTENT(CTX) | IOPTE_STBUF)
/* Existing mappings are never marked invalid, instead they
* are pointed to a dummy page.
*/
#define IOPTE_IS_DUMMY(iommu, iopte) \
((iopte_val(*iopte) & IOPTE_PAGE) == (iommu)->dummy_page_pa)
static void inline iopte_make_dummy(struct pci_iommu *iommu, iopte_t *iopte)
{
unsigned long val = iopte_val(*iopte);
val &= ~IOPTE_PAGE;
val |= iommu->dummy_page_pa;
iopte_val(*iopte) = val;
}
void pci_iommu_table_init(struct pci_iommu *iommu, int tsbsize)
{
int i;
tsbsize /= sizeof(iopte_t);
for (i = 0; i < tsbsize; i++)
iopte_make_dummy(iommu, &iommu->page_table[i]);
}
static iopte_t *alloc_streaming_cluster(struct pci_iommu *iommu, unsigned long npages)
{
iopte_t *iopte, *limit, *first;
......@@ -79,7 +112,7 @@ static iopte_t *alloc_streaming_cluster(struct pci_iommu *iommu, unsigned long n
first = iopte;
for (;;) {
if (iopte_val(*iopte) == 0UL) {
if (IOPTE_IS_DUMMY(iommu, iopte)) {
if ((iopte + (1 << cnum)) >= limit)
ent = 0;
else
......@@ -142,12 +175,12 @@ static iopte_t *alloc_consistent_cluster(struct pci_iommu *iommu, unsigned long
iopte = iommu->page_table + (1 << (iommu->page_table_sz_bits - PBM_LOGCLUSTERS));
while (iopte > iommu->page_table) {
iopte--;
if (!(iopte_val(*iopte) & IOPTE_VALID)) {
if (IOPTE_IS_DUMMY(iommu, iopte)) {
unsigned long tmp = npages;
while (--tmp) {
iopte--;
if (iopte_val(*iopte) & IOPTE_VALID)
if (!IOPTE_IS_DUMMY(iommu, iopte))
break;
}
if (tmp == 0) {
......@@ -162,15 +195,6 @@ static iopte_t *alloc_consistent_cluster(struct pci_iommu *iommu, unsigned long
return NULL;
}
#define IOPTE_CONSISTENT(CTX) \
(IOPTE_VALID | IOPTE_CACHE | \
(((CTX) << 47) & IOPTE_CONTEXT))
#define IOPTE_STREAMING(CTX) \
(IOPTE_CONSISTENT(CTX) | IOPTE_STBUF)
#define IOPTE_INVALID 0UL
/* Allocate and map kernel buffer of size SIZE using consistent mode
* DMA for PCI device PDEV. Return non-NULL cpu-side address if
* successful and set *DMA_ADDRP to the PCI side dma address.
......@@ -261,7 +285,7 @@ void pci_free_consistent(struct pci_dev *pdev, size_t size, void *cpu, dma_addr_
limit = (iommu->page_table +
(1 << (iommu->page_table_sz_bits - PBM_LOGCLUSTERS)));
while (walk < limit) {
if (iopte_val(*walk) != IOPTE_INVALID)
if (!IOPTE_IS_DUMMY(iommu, walk))
break;
walk++;
}
......@@ -280,7 +304,7 @@ void pci_free_consistent(struct pci_dev *pdev, size_t size, void *cpu, dma_addr_
ctx = (iopte_val(*iopte) & IOPTE_CONTEXT) >> 47UL;
for (i = 0; i < npages; i++, iopte++)
iopte_val(*iopte) = IOPTE_INVALID;
iopte_make_dummy(iommu, iopte);
if (iommu->iommu_ctxflush) {
pci_iommu_write(iommu->iommu_ctxflush, ctx);
......@@ -376,7 +400,7 @@ void pci_unmap_single(struct pci_dev *pdev, dma_addr_t bus_addr, size_t sz, int
base = iommu->page_table +
((bus_addr - iommu->page_table_map_base) >> IO_PAGE_SHIFT);
#ifdef DEBUG_PCI_IOMMU
if (iopte_val(*base) == IOPTE_INVALID)
if (IOPTE_IS_DUMMY(iommu, base))
printk("pci_unmap_single called on non-mapped region %08x,%08x from %016lx\n",
bus_addr, sz, __builtin_return_address(0));
#endif
......@@ -415,7 +439,7 @@ void pci_unmap_single(struct pci_dev *pdev, dma_addr_t bus_addr, size_t sz, int
}
/* Step 2: Clear out first TSB entry. */
iopte_val(*base) = IOPTE_INVALID;
iopte_make_dummy(iommu, base);
free_streaming_cluster(iommu, bus_addr - iommu->page_table_map_base,
npages, ctx);
......@@ -611,7 +635,7 @@ void pci_unmap_sg(struct pci_dev *pdev, struct scatterlist *sglist, int nelems,
((bus_addr - iommu->page_table_map_base) >> IO_PAGE_SHIFT);
#ifdef DEBUG_PCI_IOMMU
if (iopte_val(*base) == IOPTE_INVALID)
if (IOPTE_IS_DUMMY(iommu, base))
printk("pci_unmap_sg called on non-mapped region %016lx,%d from %016lx\n", sglist->dma_address, nelems, __builtin_return_address(0));
#endif
......@@ -648,7 +672,7 @@ void pci_unmap_sg(struct pci_dev *pdev, struct scatterlist *sglist, int nelems,
}
/* Step 2: Clear out first TSB entry. */
iopte_val(*base) = IOPTE_INVALID;
iopte_make_dummy(iommu, base);
free_streaming_cluster(iommu, bus_addr - iommu->page_table_map_base,
npages, ctx);
......
......@@ -1241,6 +1241,14 @@ static void __init psycho_iommu_init(struct pci_controller_info *p)
* in pci_iommu.c
*/
iommu->dummy_page = __get_free_pages(GFP_KERNEL, 0);
if (!iommu->dummy_page) {
prom_printf("PSYCHO_IOMMU: Error, gfp(dummy_page) failed.\n");
prom_halt();
}
memset((void *)iommu->dummy_page, 0, PAGE_SIZE);
iommu->dummy_page_pa = (unsigned long) __pa(iommu->dummy_page);
/* Using assumed page size 8K with 128K entries we need 1MB iommu page
* table (128K ioptes * 8 bytes per iopte). This is
* page order 7 on UltraSparc.
......@@ -1254,7 +1262,7 @@ static void __init psycho_iommu_init(struct pci_controller_info *p)
iommu->page_table_sz_bits = 17;
iommu->page_table_map_base = 0xc0000000;
iommu->dma_addr_mask = 0xffffffff;
memset((char *)tsbbase, 0, IO_TSB_SIZE);
pci_iommu_table_init(iommu, IO_TSB_SIZE);
/* We start with no consistent mappings. */
iommu->lowest_consistent_map =
......
......@@ -1293,6 +1293,14 @@ static void __init sabre_iommu_init(struct pci_controller_info *p,
* in pci_iommu.c
*/
iommu->dummy_page = __get_free_pages(GFP_KERNEL, 0);
if (!iommu->dummy_page) {
prom_printf("PSYCHO_IOMMU: Error, gfp(dummy_page) failed.\n");
prom_halt();
}
memset((void *)iommu->dummy_page, 0, PAGE_SIZE);
iommu->dummy_page_pa = (unsigned long) __pa(iommu->dummy_page);
tsbbase = __get_free_pages(GFP_KERNEL, order = get_order(tsbsize * 1024 * 8));
if (!tsbbase) {
prom_printf("SABRE_IOMMU: Error, gfp(tsb) failed.\n");
......@@ -1301,7 +1309,7 @@ static void __init sabre_iommu_init(struct pci_controller_info *p,
iommu->page_table = (iopte_t *)tsbbase;
iommu->page_table_map_base = dvma_offset;
iommu->dma_addr_mask = dma_mask;
memset((char *)tsbbase, 0, PAGE_SIZE << order);
pci_iommu_table_init(iommu, PAGE_SIZE << order);
sabre_write(p->pbm_A.controller_regs + SABRE_IOMMU_TSBBASE, __pa(tsbbase));
......
......@@ -1766,6 +1766,14 @@ static void schizo_pbm_iommu_init(struct pci_pbm_info *pbm)
* in pci_iommu.c
*/
iommu->dummy_page = __get_free_pages(GFP_KERNEL, 0);
if (!iommu->dummy_page) {
prom_printf("PSYCHO_IOMMU: Error, gfp(dummy_page) failed.\n");
prom_halt();
}
memset((void *)iommu->dummy_page, 0, PAGE_SIZE);
iommu->dummy_page_pa = (unsigned long) __pa(iommu->dummy_page);
/* Using assumed page size 8K with 128K entries we need 1MB iommu page
* table (128K ioptes * 8 bytes per iopte). This is
* page order 7 on UltraSparc.
......@@ -1780,7 +1788,7 @@ static void schizo_pbm_iommu_init(struct pci_pbm_info *pbm)
iommu->page_table = (iopte_t *)tsbbase;
iommu->page_table_map_base = vdma[0];
iommu->dma_addr_mask = dma_mask;
memset((char *)tsbbase, 0, PAGE_SIZE << order);
pci_iommu_table_init(iommu, PAGE_SIZE << order);
switch (tsbsize) {
case 64:
......
......@@ -70,6 +70,13 @@ struct pci_iommu {
*/
u32 lowest_consistent_map;
/* In order to deal with some buggy third-party PCI bridges that
* do wrong prefetching, we never mark valid mappings as invalid.
* Instead we point them at this dummy page.
*/
unsigned long dummy_page;
unsigned long dummy_page_pa;
/* If PBM_NCLUSTERS is ever decreased to 4 or lower,
* or if largest supported page_table_sz * 8K goes above
* 2GB, you must increase the size of the type of
......@@ -93,6 +100,8 @@ struct pci_iommu {
u32 dma_addr_mask;
};
extern void pci_iommu_table_init(struct pci_iommu *, int);
/* This describes a PCI bus module's streaming buffer. */
struct pci_strbuf {
int strbuf_enabled; /* Present and using it? */
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment