s390/pci: PCI adapter interrupts for MSI/MSI-X

Support PCI adapter interrupts using the Single-IRQ-mode. Single-IRQ-mode
disables an adapter IRQ automatically after delivering it until the SIC
instruction enables it again. This is used to reduce the number of IRQs
for streaming workloads.

Up to 64 MSI handlers can be registered per PCI function.
A hash table is used to map interrupt numbers to MSI descriptors.
The interrupt vector is scanned using the flogr instruction.
Only MSI/MSI-X interrupts are supported, no legacy INTs.
Signed-off-by: Jan Glauber <jang@linux.vnet.ibm.com>
Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>

arch/s390/include/asm/hw_irq.h

+#ifndef _HW_IRQ_H
+#define _HW_IRQ_H
+
+#include <linux/msi.h>
+#include <linux/pci.h>
+
+static inline struct msi_desc *irq_get_msi_desc(unsigned int irq)
+{
+	return __irq_get_msi_desc(irq);
+}
+
+/* Must be called with msi map lock held */
+static inline int irq_set_msi_desc(unsigned int irq, struct msi_desc *msi)
+{
+	if (!msi)
+		return -EINVAL;
+
+	msi->irq = irq;
+	return 0;
+}
+
+#endif

arch/s390/include/asm/irq.h

@@ -33,6 +33,8 @@ enum interruption_class {
 	IOINT_APB,
 	IOINT_ADM,
 	IOINT_CSC,
+	IOINT_PCI,
+	IOINT_MSI,
 	NMI_NMI,
 	NR_IRQS,
 };
@@ -51,4 +53,14 @@ void service_subclass_irq_unregister(void);
 void measurement_alert_subclass_register(void);
 void measurement_alert_subclass_unregister(void);
 
+#ifdef CONFIG_LOCKDEP
+#  define disable_irq_nosync_lockdep(irq)	disable_irq_nosync(irq)
+#  define disable_irq_nosync_lockdep_irqsave(irq, flags) \
+						disable_irq_nosync(irq)
+#  define disable_irq_lockdep(irq)		disable_irq(irq)
+#  define enable_irq_lockdep(irq)		enable_irq(irq)
+#  define enable_irq_lockdep_irqrestore(irq, flags) \
+						enable_irq(irq)
+#endif
+
 #endif /* _ASM_IRQ_H */

arch/s390/include/asm/isc.h

@@ -18,6 +18,7 @@
 #define CHSC_SCH_ISC 7			/* CHSC subchannels */
 /* Adapter interrupts. */
 #define QDIO_AIRQ_ISC IO_SCH_ISC	/* I/O subchannel in qdio mode */
+#define PCI_ISC 2			/* PCI I/O subchannels */
 #define AP_ISC 6			/* adjunct processor (crypto) devices */
 
 /* Functions for registration of I/O interruption subclasses */

arch/s390/include/asm/pci.h

@@ -20,6 +20,10 @@ void pci_iounmap(struct pci_dev *, void __iomem *);
 int pci_domain_nr(struct pci_bus *);
 int pci_proc_domain(struct pci_bus *);
 
+/* MSI arch hooks */
+#define arch_setup_msi_irqs	arch_setup_msi_irqs
+#define arch_teardown_msi_irqs	arch_teardown_msi_irqs
+
 #define ZPCI_BUS_NR			0	/* default bus number */
 #define ZPCI_DEVFN			0	/* default device number */
 
@@ -29,6 +33,15 @@ int pci_proc_domain(struct pci_bus *);
 #define ZPCI_FC_BLOCKED			0x20
 #define ZPCI_FC_DMA_ENABLED		0x10
 
+struct msi_map {
+	unsigned long irq;
+	struct msi_desc *msi;
+	struct hlist_node msi_chain;
+};
+
+#define ZPCI_NR_MSI_VECS	64
+#define ZPCI_MSI_MASK		(ZPCI_NR_MSI_VECS - 1)
+
 enum zpci_state {
 	ZPCI_FN_STATE_RESERVED,
 	ZPCI_FN_STATE_STANDBY,
@@ -56,6 +69,12 @@ struct zpci_dev {
 	u8		pfgid;		/* function group ID */
 	u16		domain;
 
+	/* IRQ stuff */
+	u64		msi_addr;	/* MSI address */
+	struct zdev_irq_map *irq_map;
+	struct msi_map *msi_map[ZPCI_NR_MSI_VECS];
+	unsigned int	aisb;		/* number of the summary bit */
+
 	struct zpci_bar_struct bars[PCI_BAR_COUNT];
 
 	enum pci_bus_speed max_bus_speed;
@@ -83,6 +102,14 @@ int clp_add_pci_device(u32, u32, int);
 int clp_enable_fh(struct zpci_dev *, u8);
 int clp_disable_fh(struct zpci_dev *);
 
+/* MSI */
+struct msi_desc *__irq_get_msi_desc(unsigned int);
+int zpci_msi_set_mask_bits(struct msi_desc *, u32, u32);
+int zpci_setup_msi_irq(struct zpci_dev *, struct msi_desc *, unsigned int, int);
+void zpci_teardown_msi_irq(struct zpci_dev *, struct msi_desc *);
+int zpci_msihash_init(void);
+void zpci_msihash_exit(void);
+
 /* Helpers */
 struct zpci_dev *get_zdev(struct pci_dev *);
 struct zpci_dev *get_zdev_by_fid(u32);

arch/s390/kernel/irq.c

@@ -58,6 +58,8 @@ static const struct irq_class intrclass_names[] = {
 	[IOINT_APB]  = {.name = "APB", .desc = "[I/O] AP Bus"},
 	[IOINT_ADM]  = {.name = "ADM", .desc = "[I/O] EADM Subchannel"},
 	[IOINT_CSC]  = {.name = "CSC", .desc = "[I/O] CHSC Subchannel"},
+	[IOINT_PCI]  = {.name = "PCI", .desc = "[I/O] PCI Interrupt" },
+	[IOINT_MSI] =  {.name = "MSI", .desc = "[I/O] MSI Interrupt" },
 	[NMI_NMI]    = {.name = "NMI", .desc = "[NMI] Machine Check"},
 };
 

arch/s390/pci/Makefile

@@ -2,4 +2,4 @@
 # Makefile for the s390 PCI subsystem.
 #
 
-obj-$(CONFIG_PCI)	+= pci.o pci_clp.o
+obj-$(CONFIG_PCI)	+= pci.o pci_clp.o pci_msi.o

arch/s390/pci/pci_clp.c

@@ -48,6 +48,9 @@ static void clp_free_block(void *ptr)
 static void clp_store_query_pci_fngrp(struct zpci_dev *zdev,
 				      struct clp_rsp_query_pci_grp *response)
 {
+	zdev->msi_addr = response->msia;
+
+	pr_debug("Supported number of MSI vectors: %u\n", response->noi);
 	switch (response->version) {
 	case 1:
 		zdev->max_bus_speed = PCIE_SPEED_5_0GT;

arch/s390/pci/pci_msi.c

+/*
+ * Copyright IBM Corp. 2012
+ *
+ * Author(s):
+ *   Jan Glauber <jang@linux.vnet.ibm.com>
+ */
+
+#define COMPONENT "zPCI"
+#define pr_fmt(fmt) COMPONENT ": " fmt
+
+#include <linux/kernel.h>
+#include <linux/err.h>
+#include <linux/rculist.h>
+#include <linux/hash.h>
+#include <linux/pci.h>
+#include <linux/msi.h>
+#include <asm/hw_irq.h>
+
+/* mapping of irq numbers to msi_desc */
+static struct hlist_head *msi_hash;
+static unsigned int msihash_shift = 6;
+#define msi_hashfn(nr)	hash_long(nr, msihash_shift)
+
+static DEFINE_SPINLOCK(msi_map_lock);
+
+struct msi_desc *__irq_get_msi_desc(unsigned int irq)
+{
+	struct hlist_node *entry;
+	struct msi_map *map;
+
+	hlist_for_each_entry_rcu(map, entry,
+			&msi_hash[msi_hashfn(irq)], msi_chain)
+		if (map->irq == irq)
+			return map->msi;
+	return NULL;
+}
+
+int zpci_msi_set_mask_bits(struct msi_desc *msi, u32 mask, u32 flag)
+{
+	if (msi->msi_attrib.is_msix) {
+		int offset = msi->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE +
+			PCI_MSIX_ENTRY_VECTOR_CTRL;
+		msi->masked = readl(msi->mask_base + offset);
+		writel(flag, msi->mask_base + offset);
+	} else {
+		if (msi->msi_attrib.maskbit) {
+			int pos;
+			u32 mask_bits;
+
+			pos = (long) msi->mask_base;
+			pci_read_config_dword(msi->dev, pos, &mask_bits);
+			mask_bits &= ~(mask);
+			mask_bits |= flag & mask;
+			pci_write_config_dword(msi->dev, pos, mask_bits);
+		} else {
+			return 0;
+		}
+	}
+
+	msi->msi_attrib.maskbit = !!flag;
+	return 1;
+}
+
+int zpci_setup_msi_irq(struct zpci_dev *zdev, struct msi_desc *msi,
+			unsigned int nr, int offset)
+{
+	struct msi_map *map;
+	struct msi_msg msg;
+	int rc;
+
+	map = kmalloc(sizeof(*map), GFP_KERNEL);
+	if (map == NULL)
+		return -ENOMEM;
+
+	map->irq = nr;
+	map->msi = msi;
+	zdev->msi_map[nr & ZPCI_MSI_MASK] = map;
+
+	pr_debug("%s hashing irq: %u  to bucket nr: %llu\n",
+		__func__, nr, msi_hashfn(nr));
+	hlist_add_head_rcu(&map->msi_chain, &msi_hash[msi_hashfn(nr)]);
+
+	spin_lock(&msi_map_lock);
+	rc = irq_set_msi_desc(nr, msi);
+	if (rc) {
+		spin_unlock(&msi_map_lock);
+		hlist_del_rcu(&map->msi_chain);
+		kfree(map);
+		zdev->msi_map[nr & ZPCI_MSI_MASK] = NULL;
+		return rc;
+	}
+	spin_unlock(&msi_map_lock);
+
+	msg.data = nr - offset;
+	msg.address_lo = zdev->msi_addr & 0xffffffff;
+	msg.address_hi = zdev->msi_addr >> 32;
+	write_msi_msg(nr, &msg);
+	return 0;
+}
+
+void zpci_teardown_msi_irq(struct zpci_dev *zdev, struct msi_desc *msi)
+{
+	int irq = msi->irq & ZPCI_MSI_MASK;
+	struct msi_map *map;
+
+	msi->msg.address_lo = 0;
+	msi->msg.address_hi = 0;
+	msi->msg.data = 0;
+	msi->irq = 0;
+	zpci_msi_set_mask_bits(msi, 1, 1);
+
+	spin_lock(&msi_map_lock);
+	map = zdev->msi_map[irq];
+	hlist_del_rcu(&map->msi_chain);
+	kfree(map);
+	zdev->msi_map[irq] = NULL;
+	spin_unlock(&msi_map_lock);
+}
+
+/*
+ * The msi hash table has 256 entries which is good for 4..20
+ * devices (a typical device allocates 10 + CPUs MSI's). Maybe make
+ * the hash table size adjustable later.
+ */
+int __init zpci_msihash_init(void)
+{
+	unsigned int i;
+
+	msi_hash = kmalloc(256 * sizeof(*msi_hash), GFP_KERNEL);
+	if (!msi_hash)
+		return -ENOMEM;
+
+	for (i = 0; i < (1U << msihash_shift); i++)
+		INIT_HLIST_HEAD(&msi_hash[i]);
+	return 0;
+}
+
+void __init zpci_msihash_exit(void)
+{
+	kfree(msi_hash);
+}

drivers/pci/msi.c

@@ -207,6 +207,8 @@ static void msix_mask_irq(struct msi_desc *desc, u32 flag)
 	desc->masked = __msix_mask_irq(desc, flag);
 }
 
+#ifdef CONFIG_GENERIC_HARDIRQS
+
 static void msi_set_mask_bit(struct irq_data *data, u32 flag)
 {
 	struct msi_desc *desc = irq_data_get_msi(data);
@@ -230,6 +232,8 @@ void unmask_msi_irq(struct irq_data *data)
 	msi_set_mask_bit(data, 0);
 }
 
+#endif /* CONFIG_GENERIC_HARDIRQS */
+
 void __read_msi_msg(struct msi_desc *entry, struct msi_msg *msg)
 {
 	BUG_ON(entry->dev->current_state != PCI_D0);
@@ -337,8 +341,10 @@ static void free_msi_irqs(struct pci_dev *dev)
 		if (!entry->irq)
 			continue;
 		nvec = 1 << entry->msi_attrib.multiple;
+#ifdef CONFIG_GENERIC_HARDIRQS
 		for (i = 0; i < nvec; i++)
 			BUG_ON(irq_has_action(entry->irq + i));
+#endif
 	}
 
 	arch_teardown_msi_irqs(dev);

include/linux/irq.h

@@ -10,9 +10,6 @@
  */
 
 #include <linux/smp.h>
-
-#ifndef CONFIG_S390
-
 #include <linux/linkage.h>
 #include <linux/cache.h>
 #include <linux/spinlock.h>
@@ -737,8 +734,11 @@ static inline void irq_gc_lock(struct irq_chip_generic *gc) { }
 static inline void irq_gc_unlock(struct irq_chip_generic *gc) { }
 #endif
 
-#endif /* CONFIG_GENERIC_HARDIRQS */
+#else /* !CONFIG_GENERIC_HARDIRQS */
 
-#endif /* !CONFIG_S390 */
+extern struct msi_desc *irq_get_msi_desc(unsigned int irq);
+extern int irq_set_msi_desc(unsigned int irq, struct msi_desc *entry);
+
+#endif /* CONFIG_GENERIC_HARDIRQS */
 
 #endif /* _LINUX_IRQ_H */