Commit ebb09b33 authored by Leonid Ravich's avatar Leonid Ravich Committed by Jon Mason

NTB: add new parameter to peer_db_addr() db_bit and db_data

NTB door bell usage depends on NTB hardware.

ex: intel NTB gen1 has one peer door bell register which can be controlled
by the bitmap writen to it, while Intel NTB gen3 has a registers
per door bell and the data trigering the each door bell is always 1.

therefore exposing only peer door bell address forcing the user
to be aware of such low level details
Signed-off-by: default avatarLeonid Ravich <Leonid.Ravich@emc.com>
Acked-by: default avatarLogan Gunthorpe <logang@deltatee.com>
Acked-by: default avatarDave Jiang <dave.jiang@intel.com>
Acked-by: default avatarAllen Hubbe <allenbh@gmail.com>
Signed-off-by: default avatarJon Mason <jdmason@kudzu.us>
parent c59666bb
...@@ -180,7 +180,7 @@ int ndev_mw_to_bar(struct intel_ntb_dev *ndev, int idx) ...@@ -180,7 +180,7 @@ int ndev_mw_to_bar(struct intel_ntb_dev *ndev, int idx)
return ndev->reg->mw_bar[idx]; return ndev->reg->mw_bar[idx];
} }
static inline int ndev_db_addr(struct intel_ntb_dev *ndev, void ndev_db_addr(struct intel_ntb_dev *ndev,
phys_addr_t *db_addr, resource_size_t *db_size, phys_addr_t *db_addr, resource_size_t *db_size,
phys_addr_t reg_addr, unsigned long reg) phys_addr_t reg_addr, unsigned long reg)
{ {
...@@ -196,8 +196,6 @@ static inline int ndev_db_addr(struct intel_ntb_dev *ndev, ...@@ -196,8 +196,6 @@ static inline int ndev_db_addr(struct intel_ntb_dev *ndev,
*db_size = ndev->reg->db_size; *db_size = ndev->reg->db_size;
dev_dbg(&ndev->ntb.pdev->dev, "Peer db size %llx\n", *db_size); dev_dbg(&ndev->ntb.pdev->dev, "Peer db size %llx\n", *db_size);
} }
return 0;
} }
u64 ndev_db_read(struct intel_ntb_dev *ndev, u64 ndev_db_read(struct intel_ntb_dev *ndev,
...@@ -1111,13 +1109,28 @@ int intel_ntb_db_clear_mask(struct ntb_dev *ntb, u64 db_bits) ...@@ -1111,13 +1109,28 @@ int intel_ntb_db_clear_mask(struct ntb_dev *ntb, u64 db_bits)
ndev->self_reg->db_mask); ndev->self_reg->db_mask);
} }
int intel_ntb_peer_db_addr(struct ntb_dev *ntb, phys_addr_t *db_addr, static int intel_ntb_peer_db_addr(struct ntb_dev *ntb, phys_addr_t *db_addr,
resource_size_t *db_size) resource_size_t *db_size, u64 *db_data, int db_bit)
{ {
u64 db_bits;
struct intel_ntb_dev *ndev = ntb_ndev(ntb); struct intel_ntb_dev *ndev = ntb_ndev(ntb);
return ndev_db_addr(ndev, db_addr, db_size, ndev->peer_addr, if (unlikely(db_bit >= BITS_PER_LONG_LONG))
return -EINVAL;
db_bits = BIT_ULL(db_bit);
if (unlikely(db_bits & ~ntb_ndev(ntb)->db_valid_mask))
return -EINVAL;
ndev_db_addr(ndev, db_addr, db_size, ndev->peer_addr,
ndev->peer_reg->db_bell); ndev->peer_reg->db_bell);
if (db_data)
*db_data = db_bits;
return 0;
} }
static int intel_ntb_peer_db_set(struct ntb_dev *ntb, u64 db_bits) static int intel_ntb_peer_db_set(struct ntb_dev *ntb, u64 db_bits)
......
...@@ -147,6 +147,9 @@ extern struct intel_b2b_addr xeon_b2b_dsd_addr; ...@@ -147,6 +147,9 @@ extern struct intel_b2b_addr xeon_b2b_dsd_addr;
int ndev_init_isr(struct intel_ntb_dev *ndev, int msix_min, int msix_max, int ndev_init_isr(struct intel_ntb_dev *ndev, int msix_min, int msix_max,
int msix_shift, int total_shift); int msix_shift, int total_shift);
enum ntb_topo xeon_ppd_topo(struct intel_ntb_dev *ndev, u8 ppd); enum ntb_topo xeon_ppd_topo(struct intel_ntb_dev *ndev, u8 ppd);
void ndev_db_addr(struct intel_ntb_dev *ndev,
phys_addr_t *db_addr, resource_size_t *db_size,
phys_addr_t reg_addr, unsigned long reg);
u64 ndev_db_read(struct intel_ntb_dev *ndev, void __iomem *mmio); u64 ndev_db_read(struct intel_ntb_dev *ndev, void __iomem *mmio);
int ndev_db_write(struct intel_ntb_dev *ndev, u64 db_bits, int ndev_db_write(struct intel_ntb_dev *ndev, u64 db_bits,
void __iomem *mmio); void __iomem *mmio);
...@@ -166,8 +169,6 @@ int intel_ntb_db_vector_count(struct ntb_dev *ntb); ...@@ -166,8 +169,6 @@ int intel_ntb_db_vector_count(struct ntb_dev *ntb);
u64 intel_ntb_db_vector_mask(struct ntb_dev *ntb, int db_vector); u64 intel_ntb_db_vector_mask(struct ntb_dev *ntb, int db_vector);
int intel_ntb_db_set_mask(struct ntb_dev *ntb, u64 db_bits); int intel_ntb_db_set_mask(struct ntb_dev *ntb, u64 db_bits);
int intel_ntb_db_clear_mask(struct ntb_dev *ntb, u64 db_bits); int intel_ntb_db_clear_mask(struct ntb_dev *ntb, u64 db_bits);
int intel_ntb_peer_db_addr(struct ntb_dev *ntb, phys_addr_t *db_addr,
resource_size_t *db_size);
int intel_ntb_spad_is_unsafe(struct ntb_dev *ntb); int intel_ntb_spad_is_unsafe(struct ntb_dev *ntb);
int intel_ntb_spad_count(struct ntb_dev *ntb); int intel_ntb_spad_count(struct ntb_dev *ntb);
u32 intel_ntb_spad_read(struct ntb_dev *ntb, int idx); u32 intel_ntb_spad_read(struct ntb_dev *ntb, int idx);
......
...@@ -532,6 +532,37 @@ static int intel_ntb3_mw_set_trans(struct ntb_dev *ntb, int pidx, int idx, ...@@ -532,6 +532,37 @@ static int intel_ntb3_mw_set_trans(struct ntb_dev *ntb, int pidx, int idx,
return 0; return 0;
} }
int intel_ntb3_peer_db_addr(struct ntb_dev *ntb, phys_addr_t *db_addr,
resource_size_t *db_size,
u64 *db_data, int db_bit)
{
phys_addr_t db_addr_base;
struct intel_ntb_dev *ndev = ntb_ndev(ntb);
if (unlikely(db_bit >= BITS_PER_LONG_LONG))
return -EINVAL;
if (unlikely(BIT_ULL(db_bit) & ~ntb_ndev(ntb)->db_valid_mask))
return -EINVAL;
ndev_db_addr(ndev, &db_addr_base, db_size, ndev->peer_addr,
ndev->peer_reg->db_bell);
if (db_addr) {
*db_addr = db_addr_base + (db_bit * 4);
dev_dbg(&ndev->ntb.pdev->dev, "Peer db addr %llx db bit %d\n",
*db_addr, db_bit);
}
if (db_data) {
*db_data = 1;
dev_dbg(&ndev->ntb.pdev->dev, "Peer db data %llx db bit %d\n",
*db_data, db_bit);
}
return 0;
}
static int intel_ntb3_peer_db_set(struct ntb_dev *ntb, u64 db_bits) static int intel_ntb3_peer_db_set(struct ntb_dev *ntb, u64 db_bits)
{ {
struct intel_ntb_dev *ndev = ntb_ndev(ntb); struct intel_ntb_dev *ndev = ntb_ndev(ntb);
...@@ -584,7 +615,7 @@ const struct ntb_dev_ops intel_ntb3_ops = { ...@@ -584,7 +615,7 @@ const struct ntb_dev_ops intel_ntb3_ops = {
.db_clear = intel_ntb3_db_clear, .db_clear = intel_ntb3_db_clear,
.db_set_mask = intel_ntb_db_set_mask, .db_set_mask = intel_ntb_db_set_mask,
.db_clear_mask = intel_ntb_db_clear_mask, .db_clear_mask = intel_ntb_db_clear_mask,
.peer_db_addr = intel_ntb_peer_db_addr, .peer_db_addr = intel_ntb3_peer_db_addr,
.peer_db_set = intel_ntb3_peer_db_set, .peer_db_set = intel_ntb3_peer_db_set,
.spad_is_unsafe = intel_ntb_spad_is_unsafe, .spad_is_unsafe = intel_ntb_spad_is_unsafe,
.spad_count = intel_ntb_spad_count, .spad_count = intel_ntb_spad_count,
......
...@@ -710,11 +710,16 @@ static u64 switchtec_ntb_db_read_mask(struct ntb_dev *ntb) ...@@ -710,11 +710,16 @@ static u64 switchtec_ntb_db_read_mask(struct ntb_dev *ntb)
static int switchtec_ntb_peer_db_addr(struct ntb_dev *ntb, static int switchtec_ntb_peer_db_addr(struct ntb_dev *ntb,
phys_addr_t *db_addr, phys_addr_t *db_addr,
resource_size_t *db_size) resource_size_t *db_size,
u64 *db_data,
int db_bit)
{ {
struct switchtec_ntb *sndev = ntb_sndev(ntb); struct switchtec_ntb *sndev = ntb_sndev(ntb);
unsigned long offset; unsigned long offset;
if (unlikely(db_bit >= BITS_PER_LONG_LONG))
return -EINVAL;
offset = (unsigned long)sndev->mmio_peer_dbmsg->odb - offset = (unsigned long)sndev->mmio_peer_dbmsg->odb -
(unsigned long)sndev->stdev->mmio; (unsigned long)sndev->stdev->mmio;
...@@ -724,6 +729,8 @@ static int switchtec_ntb_peer_db_addr(struct ntb_dev *ntb, ...@@ -724,6 +729,8 @@ static int switchtec_ntb_peer_db_addr(struct ntb_dev *ntb,
*db_addr = pci_resource_start(ntb->pdev, 0) + offset; *db_addr = pci_resource_start(ntb->pdev, 0) + offset;
if (db_size) if (db_size)
*db_size = sizeof(u32); *db_size = sizeof(u32);
if (db_data)
*db_data = BIT_ULL(db_bit) << sndev->db_peer_shift;
return 0; return 0;
} }
......
...@@ -296,7 +296,8 @@ struct ntb_dev_ops { ...@@ -296,7 +296,8 @@ struct ntb_dev_ops {
int (*db_clear_mask)(struct ntb_dev *ntb, u64 db_bits); int (*db_clear_mask)(struct ntb_dev *ntb, u64 db_bits);
int (*peer_db_addr)(struct ntb_dev *ntb, int (*peer_db_addr)(struct ntb_dev *ntb,
phys_addr_t *db_addr, resource_size_t *db_size); phys_addr_t *db_addr, resource_size_t *db_size,
u64 *db_data, int db_bit);
u64 (*peer_db_read)(struct ntb_dev *ntb); u64 (*peer_db_read)(struct ntb_dev *ntb);
int (*peer_db_set)(struct ntb_dev *ntb, u64 db_bits); int (*peer_db_set)(struct ntb_dev *ntb, u64 db_bits);
int (*peer_db_clear)(struct ntb_dev *ntb, u64 db_bits); int (*peer_db_clear)(struct ntb_dev *ntb, u64 db_bits);
...@@ -1078,6 +1079,8 @@ static inline int ntb_db_clear_mask(struct ntb_dev *ntb, u64 db_bits) ...@@ -1078,6 +1079,8 @@ static inline int ntb_db_clear_mask(struct ntb_dev *ntb, u64 db_bits)
* @ntb: NTB device context. * @ntb: NTB device context.
* @db_addr: OUT - The address of the peer doorbell register. * @db_addr: OUT - The address of the peer doorbell register.
* @db_size: OUT - The number of bytes to write the peer doorbell register. * @db_size: OUT - The number of bytes to write the peer doorbell register.
* @db_data: OUT - The data of peer doorbell register
* @db_bit: door bell bit number
* *
* Return the address of the peer doorbell register. This may be used, for * Return the address of the peer doorbell register. This may be used, for
* example, by drivers that offload memory copy operations to a dma engine. * example, by drivers that offload memory copy operations to a dma engine.
...@@ -1091,12 +1094,13 @@ static inline int ntb_db_clear_mask(struct ntb_dev *ntb, u64 db_bits) ...@@ -1091,12 +1094,13 @@ static inline int ntb_db_clear_mask(struct ntb_dev *ntb, u64 db_bits)
*/ */
static inline int ntb_peer_db_addr(struct ntb_dev *ntb, static inline int ntb_peer_db_addr(struct ntb_dev *ntb,
phys_addr_t *db_addr, phys_addr_t *db_addr,
resource_size_t *db_size) resource_size_t *db_size,
u64 *db_data, int db_bit)
{ {
if (!ntb->ops->peer_db_addr) if (!ntb->ops->peer_db_addr)
return -EINVAL; return -EINVAL;
return ntb->ops->peer_db_addr(ntb, db_addr, db_size); return ntb->ops->peer_db_addr(ntb, db_addr, db_size, db_data, db_bit);
} }
/** /**
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment