Commit 8ad11fb6 authored by Jack Morgenstein's avatar Jack Morgenstein Committed by Roland Dreier

IB/mlx4: Implement FMRs

Implement FMRs for mlx4.  This is an adaptation of code from mthca.
Signed-off-by: default avatarJack Morgenstein <jackm@dev.mellanox.co.il>
Signed-off-by: default avatarMichael S. Tsirkin <mst@dev.mellanox.co.il>
Signed-off-by: default avatarRoland Dreier <rolandd@cisco.com>
parent d7bb58fb
......@@ -607,6 +607,11 @@ static void *mlx4_ib_add(struct mlx4_dev *dev)
ibdev->ib_dev.detach_mcast = mlx4_ib_mcg_detach;
ibdev->ib_dev.process_mad = mlx4_ib_process_mad;
ibdev->ib_dev.alloc_fmr = mlx4_ib_fmr_alloc;
ibdev->ib_dev.map_phys_fmr = mlx4_ib_map_phys_fmr;
ibdev->ib_dev.unmap_fmr = mlx4_ib_unmap_fmr;
ibdev->ib_dev.dealloc_fmr = mlx4_ib_fmr_dealloc;
if (init_node_data(ibdev))
goto err_map;
......
......@@ -93,6 +93,11 @@ struct mlx4_ib_mr {
struct ib_umem *umem;
};
struct mlx4_ib_fmr {
struct ib_fmr ibfmr;
struct mlx4_fmr mfmr;
};
struct mlx4_ib_wq {
u64 *wrid;
spinlock_t lock;
......@@ -199,6 +204,10 @@ static inline struct mlx4_ib_mr *to_mmr(struct ib_mr *ibmr)
return container_of(ibmr, struct mlx4_ib_mr, ibmr);
}
static inline struct mlx4_ib_fmr *to_mfmr(struct ib_fmr *ibfmr)
{
return container_of(ibfmr, struct mlx4_ib_fmr, ibfmr);
}
static inline struct mlx4_ib_qp *to_mqp(struct ib_qp *ibqp)
{
return container_of(ibqp, struct mlx4_ib_qp, ibqp);
......@@ -284,6 +293,13 @@ int mlx4_ib_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num,
int mlx4_ib_mad_init(struct mlx4_ib_dev *dev);
void mlx4_ib_mad_cleanup(struct mlx4_ib_dev *dev);
struct ib_fmr *mlx4_ib_fmr_alloc(struct ib_pd *pd, int mr_access_flags,
struct ib_fmr_attr *fmr_attr);
int mlx4_ib_map_phys_fmr(struct ib_fmr *ibfmr, u64 *page_list, int npages,
u64 iova);
int mlx4_ib_unmap_fmr(struct list_head *fmr_list);
int mlx4_ib_fmr_dealloc(struct ib_fmr *fmr);
static inline int mlx4_ib_ah_grh_present(struct mlx4_ib_ah *ah)
{
return !!(ah->av.g_slid & 0x80);
......
......@@ -181,3 +181,96 @@ int mlx4_ib_dereg_mr(struct ib_mr *ibmr)
return 0;
}
struct ib_fmr *mlx4_ib_fmr_alloc(struct ib_pd *pd, int acc,
struct ib_fmr_attr *fmr_attr)
{
struct mlx4_ib_dev *dev = to_mdev(pd->device);
struct mlx4_ib_fmr *fmr;
int err = -ENOMEM;
fmr = kmalloc(sizeof *fmr, GFP_KERNEL);
if (!fmr)
return ERR_PTR(-ENOMEM);
err = mlx4_fmr_alloc(dev->dev, to_mpd(pd)->pdn, convert_access(acc),
fmr_attr->max_pages, fmr_attr->max_maps,
fmr_attr->page_shift, &fmr->mfmr);
if (err)
goto err_free;
err = mlx4_mr_enable(to_mdev(pd->device)->dev, &fmr->mfmr.mr);
if (err)
goto err_mr;
fmr->ibfmr.rkey = fmr->ibfmr.lkey = fmr->mfmr.mr.key;
return &fmr->ibfmr;
err_mr:
mlx4_mr_free(to_mdev(pd->device)->dev, &fmr->mfmr.mr);
err_free:
kfree(fmr);
return ERR_PTR(err);
}
int mlx4_ib_map_phys_fmr(struct ib_fmr *ibfmr, u64 *page_list,
int npages, u64 iova)
{
struct mlx4_ib_fmr *ifmr = to_mfmr(ibfmr);
struct mlx4_ib_dev *dev = to_mdev(ifmr->ibfmr.device);
return mlx4_map_phys_fmr(dev->dev, &ifmr->mfmr, page_list, npages, iova,
&ifmr->ibfmr.lkey, &ifmr->ibfmr.rkey);
}
int mlx4_ib_unmap_fmr(struct list_head *fmr_list)
{
struct ib_fmr *ibfmr;
int err;
struct mlx4_dev *mdev = NULL;
list_for_each_entry(ibfmr, fmr_list, list) {
if (mdev && to_mdev(ibfmr->device)->dev != mdev)
return -EINVAL;
mdev = to_mdev(ibfmr->device)->dev;
}
if (!mdev)
return 0;
list_for_each_entry(ibfmr, fmr_list, list) {
struct mlx4_ib_fmr *ifmr = to_mfmr(ibfmr);
mlx4_fmr_unmap(mdev, &ifmr->mfmr, &ifmr->ibfmr.lkey, &ifmr->ibfmr.rkey);
}
/*
* Make sure all MPT status updates are visible before issuing
* SYNC_TPT firmware command.
*/
wmb();
err = mlx4_SYNC_TPT(mdev);
if (err)
printk(KERN_WARNING "mlx4_ib: SYNC_TPT error %d when "
"unmapping FMRs\n", err);
return 0;
}
int mlx4_ib_fmr_dealloc(struct ib_fmr *ibfmr)
{
struct mlx4_ib_fmr *ifmr = to_mfmr(ibfmr);
struct mlx4_ib_dev *dev = to_mdev(ibfmr->device);
int err;
err = mlx4_fmr_free(dev->dev, &ifmr->mfmr);
if (!err)
kfree(ifmr);
return err;
}
......@@ -68,6 +68,9 @@ struct mlx4_mpt_entry {
#define MLX4_MTT_FLAG_PRESENT 1
#define MLX4_MPT_STATUS_SW 0xF0
#define MLX4_MPT_STATUS_HW 0x00
static u32 mlx4_buddy_alloc(struct mlx4_buddy *buddy, int order)
{
int o;
......@@ -469,3 +472,165 @@ void mlx4_cleanup_mr_table(struct mlx4_dev *dev)
mlx4_buddy_cleanup(&mr_table->mtt_buddy);
mlx4_bitmap_cleanup(&mr_table->mpt_bitmap);
}
static inline int mlx4_check_fmr(struct mlx4_fmr *fmr, u64 *page_list,
int npages, u64 iova)
{
int i, page_mask;
if (npages > fmr->max_pages)
return -EINVAL;
page_mask = (1 << fmr->page_shift) - 1;
/* We are getting page lists, so va must be page aligned. */
if (iova & page_mask)
return -EINVAL;
/* Trust the user not to pass misaligned data in page_list */
if (0)
for (i = 0; i < npages; ++i) {
if (page_list[i] & ~page_mask)
return -EINVAL;
}
if (fmr->maps >= fmr->max_maps)
return -EINVAL;
return 0;
}
int mlx4_map_phys_fmr(struct mlx4_dev *dev, struct mlx4_fmr *fmr, u64 *page_list,
int npages, u64 iova, u32 *lkey, u32 *rkey)
{
u32 key;
int i, err;
err = mlx4_check_fmr(fmr, page_list, npages, iova);
if (err)
return err;
++fmr->maps;
key = key_to_hw_index(fmr->mr.key);
key += dev->caps.num_mpts;
*lkey = *rkey = fmr->mr.key = hw_index_to_key(key);
*(u8 *) fmr->mpt = MLX4_MPT_STATUS_SW;
/* Make sure MPT status is visible before writing MTT entries */
wmb();
for (i = 0; i < npages; ++i)
fmr->mtts[i] = cpu_to_be64(page_list[i] | MLX4_MTT_FLAG_PRESENT);
dma_sync_single(&dev->pdev->dev, fmr->dma_handle,
npages * sizeof(u64), DMA_TO_DEVICE);
fmr->mpt->key = cpu_to_be32(key);
fmr->mpt->lkey = cpu_to_be32(key);
fmr->mpt->length = cpu_to_be64(npages * (1ull << fmr->page_shift));
fmr->mpt->start = cpu_to_be64(iova);
/* Make MTT entries are visible before setting MPT status */
wmb();
*(u8 *) fmr->mpt = MLX4_MPT_STATUS_HW;
/* Make sure MPT status is visible before consumer can use FMR */
wmb();
return 0;
}
EXPORT_SYMBOL_GPL(mlx4_map_phys_fmr);
int mlx4_fmr_alloc(struct mlx4_dev *dev, u32 pd, u32 access, int max_pages,
int max_maps, u8 page_shift, struct mlx4_fmr *fmr)
{
struct mlx4_priv *priv = mlx4_priv(dev);
u64 mtt_seg;
int err = -ENOMEM;
if (page_shift < 12 || page_shift >= 32)
return -EINVAL;
/* All MTTs must fit in the same page */
if (max_pages * sizeof *fmr->mtts > PAGE_SIZE)
return -EINVAL;
fmr->page_shift = page_shift;
fmr->max_pages = max_pages;
fmr->max_maps = max_maps;
fmr->maps = 0;
err = mlx4_mr_alloc(dev, pd, 0, 0, access, max_pages,
page_shift, &fmr->mr);
if (err)
return err;
mtt_seg = fmr->mr.mtt.first_seg * dev->caps.mtt_entry_sz;
fmr->mtts = mlx4_table_find(&priv->mr_table.mtt_table,
fmr->mr.mtt.first_seg,
&fmr->dma_handle);
if (!fmr->mtts) {
err = -ENOMEM;
goto err_free;
}
fmr->mpt = mlx4_table_find(&priv->mr_table.dmpt_table,
key_to_hw_index(fmr->mr.key), NULL);
if (!fmr->mpt) {
err = -ENOMEM;
goto err_free;
}
return 0;
err_free:
mlx4_mr_free(dev, &fmr->mr);
return err;
}
EXPORT_SYMBOL_GPL(mlx4_fmr_alloc);
int mlx4_fmr_enable(struct mlx4_dev *dev, struct mlx4_fmr *fmr)
{
return mlx4_mr_enable(dev, &fmr->mr);
}
EXPORT_SYMBOL_GPL(mlx4_fmr_enable);
void mlx4_fmr_unmap(struct mlx4_dev *dev, struct mlx4_fmr *fmr,
u32 *lkey, u32 *rkey)
{
u32 key;
if (!fmr->maps)
return;
key = key_to_hw_index(fmr->mr.key);
key &= dev->caps.num_mpts - 1;
*lkey = *rkey = fmr->mr.key = hw_index_to_key(key);
fmr->maps = 0;
*(u8 *) fmr->mpt = MLX4_MPT_STATUS_SW;
}
EXPORT_SYMBOL_GPL(mlx4_fmr_unmap);
int mlx4_fmr_free(struct mlx4_dev *dev, struct mlx4_fmr *fmr)
{
if (fmr->maps)
return -EBUSY;
fmr->mr.enabled = 0;
mlx4_mr_free(dev, &fmr->mr);
return 0;
}
EXPORT_SYMBOL_GPL(mlx4_fmr_free);
int mlx4_SYNC_TPT(struct mlx4_dev *dev)
{
return mlx4_cmd(dev, 0, 0, 0, MLX4_CMD_SYNC_TPT, 1000);
}
EXPORT_SYMBOL_GPL(mlx4_SYNC_TPT);
......@@ -214,6 +214,17 @@ struct mlx4_mr {
int enabled;
};
struct mlx4_fmr {
struct mlx4_mr mr;
struct mlx4_mpt_entry *mpt;
__be64 *mtts;
dma_addr_t dma_handle;
int max_pages;
int max_maps;
int maps;
u8 page_shift;
};
struct mlx4_uar {
unsigned long pfn;
int index;
......@@ -337,4 +348,14 @@ int mlx4_CLOSE_PORT(struct mlx4_dev *dev, int port);
int mlx4_multicast_attach(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16]);
int mlx4_multicast_detach(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16]);
int mlx4_map_phys_fmr(struct mlx4_dev *dev, struct mlx4_fmr *fmr, u64 *page_list,
int npages, u64 iova, u32 *lkey, u32 *rkey);
int mlx4_fmr_alloc(struct mlx4_dev *dev, u32 pd, u32 access, int max_pages,
int max_maps, u8 page_shift, struct mlx4_fmr *fmr);
int mlx4_fmr_enable(struct mlx4_dev *dev, struct mlx4_fmr *fmr);
void mlx4_fmr_unmap(struct mlx4_dev *dev, struct mlx4_fmr *fmr,
u32 *lkey, u32 *rkey);
int mlx4_fmr_free(struct mlx4_dev *dev, struct mlx4_fmr *fmr);
int mlx4_SYNC_TPT(struct mlx4_dev *dev);
#endif /* MLX4_DEVICE_H */
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment