Commit 49780d42 authored by Artemy Kovalyov's avatar Artemy Kovalyov Committed by Doug Ledford

IB/mlx5: Expose MR cache for mlx5_ib

Allow other parts of mlx5_ib to use MR cache mechanism.
* Add new functions mlx5_mr_cache_alloc and mlx5_mr_cache_free
* Traditional MTT MKey buckets are limited by MAX_UMR_CACHE_ENTRY
  Additinal buckets may be added above.
Signed-off-by: default avatarArtemy Kovalyov <artemyko@mellanox.com>
Signed-off-by: default avatarLeon Romanovsky <leon@kernel.org>
Signed-off-by: default avatarDoug Ledford <dledford@redhat.com>
parent 94990b49
...@@ -541,6 +541,10 @@ struct mlx5_cache_ent { ...@@ -541,6 +541,10 @@ struct mlx5_cache_ent {
struct dentry *dir; struct dentry *dir;
char name[4]; char name[4];
u32 order; u32 order;
u32 xlt;
u32 access_mode;
u32 page;
u32 size; u32 size;
u32 cur; u32 cur;
u32 miss; u32 miss;
...@@ -555,6 +559,7 @@ struct mlx5_cache_ent { ...@@ -555,6 +559,7 @@ struct mlx5_cache_ent {
struct work_struct work; struct work_struct work;
struct delayed_work dwork; struct delayed_work dwork;
int pending; int pending;
struct completion compl;
}; };
struct mlx5_mr_cache { struct mlx5_mr_cache {
...@@ -837,7 +842,9 @@ void mlx5_ib_copy_pas(u64 *old, u64 *new, int step, int num); ...@@ -837,7 +842,9 @@ void mlx5_ib_copy_pas(u64 *old, u64 *new, int step, int num);
int mlx5_ib_get_cqe_size(struct mlx5_ib_dev *dev, struct ib_cq *ibcq); int mlx5_ib_get_cqe_size(struct mlx5_ib_dev *dev, struct ib_cq *ibcq);
int mlx5_mr_cache_init(struct mlx5_ib_dev *dev); int mlx5_mr_cache_init(struct mlx5_ib_dev *dev);
int mlx5_mr_cache_cleanup(struct mlx5_ib_dev *dev); int mlx5_mr_cache_cleanup(struct mlx5_ib_dev *dev);
int mlx5_mr_ib_cont_pages(struct ib_umem *umem, u64 addr, int *count, int *shift);
struct mlx5_ib_mr *mlx5_mr_cache_alloc(struct mlx5_ib_dev *dev, int entry);
void mlx5_mr_cache_free(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr);
int mlx5_ib_check_mr_status(struct ib_mr *ibmr, u32 check_mask, int mlx5_ib_check_mr_status(struct ib_mr *ibmr, u32 check_mask,
struct ib_mr_status *mr_status); struct ib_mr_status *mr_status);
struct ib_wq *mlx5_ib_create_wq(struct ib_pd *pd, struct ib_wq *mlx5_ib_create_wq(struct ib_pd *pd,
......
...@@ -49,6 +49,7 @@ enum { ...@@ -49,6 +49,7 @@ enum {
static int clean_mr(struct mlx5_ib_mr *mr); static int clean_mr(struct mlx5_ib_mr *mr);
static int use_umr(struct mlx5_ib_dev *dev, int order); static int use_umr(struct mlx5_ib_dev *dev, int order);
static int unreg_umr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr);
static int destroy_mkey(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr) static int destroy_mkey(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr)
{ {
...@@ -149,6 +150,9 @@ static void reg_mr_callback(int status, void *context) ...@@ -149,6 +150,9 @@ static void reg_mr_callback(int status, void *context)
if (err) if (err)
pr_err("Error inserting to mkey tree. 0x%x\n", -err); pr_err("Error inserting to mkey tree. 0x%x\n", -err);
write_unlock_irqrestore(&table->lock, flags); write_unlock_irqrestore(&table->lock, flags);
if (!completion_done(&ent->compl))
complete(&ent->compl);
} }
static int add_keys(struct mlx5_ib_dev *dev, int c, int num) static int add_keys(struct mlx5_ib_dev *dev, int c, int num)
...@@ -157,7 +161,6 @@ static int add_keys(struct mlx5_ib_dev *dev, int c, int num) ...@@ -157,7 +161,6 @@ static int add_keys(struct mlx5_ib_dev *dev, int c, int num)
struct mlx5_cache_ent *ent = &cache->ent[c]; struct mlx5_cache_ent *ent = &cache->ent[c];
int inlen = MLX5_ST_SZ_BYTES(create_mkey_in); int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
struct mlx5_ib_mr *mr; struct mlx5_ib_mr *mr;
int npages = 1 << ent->order;
void *mkc; void *mkc;
u32 *in; u32 *in;
int err = 0; int err = 0;
...@@ -185,11 +188,11 @@ static int add_keys(struct mlx5_ib_dev *dev, int c, int num) ...@@ -185,11 +188,11 @@ static int add_keys(struct mlx5_ib_dev *dev, int c, int num)
MLX5_SET(mkc, mkc, free, 1); MLX5_SET(mkc, mkc, free, 1);
MLX5_SET(mkc, mkc, umr_en, 1); MLX5_SET(mkc, mkc, umr_en, 1);
MLX5_SET(mkc, mkc, access_mode, MLX5_MKC_ACCESS_MODE_MTT); MLX5_SET(mkc, mkc, access_mode, ent->access_mode);
MLX5_SET(mkc, mkc, qpn, 0xffffff); MLX5_SET(mkc, mkc, qpn, 0xffffff);
MLX5_SET(mkc, mkc, translations_octword_size, (npages + 1) / 2); MLX5_SET(mkc, mkc, translations_octword_size, ent->xlt);
MLX5_SET(mkc, mkc, log_page_size, 12); MLX5_SET(mkc, mkc, log_page_size, ent->page);
spin_lock_irq(&ent->lock); spin_lock_irq(&ent->lock);
ent->pending++; ent->pending++;
...@@ -447,6 +450,42 @@ static void cache_work_func(struct work_struct *work) ...@@ -447,6 +450,42 @@ static void cache_work_func(struct work_struct *work)
__cache_work_func(ent); __cache_work_func(ent);
} }
struct mlx5_ib_mr *mlx5_mr_cache_alloc(struct mlx5_ib_dev *dev, int entry)
{
struct mlx5_mr_cache *cache = &dev->cache;
struct mlx5_cache_ent *ent;
struct mlx5_ib_mr *mr;
int err;
if (entry < 0 || entry >= MAX_MR_CACHE_ENTRIES) {
mlx5_ib_err(dev, "cache entry %d is out of range\n", entry);
return NULL;
}
ent = &cache->ent[entry];
while (1) {
spin_lock_irq(&ent->lock);
if (list_empty(&ent->head)) {
spin_unlock_irq(&ent->lock);
err = add_keys(dev, entry, 1);
if (err)
return ERR_PTR(err);
wait_for_completion(&ent->compl);
} else {
mr = list_first_entry(&ent->head, struct mlx5_ib_mr,
list);
list_del(&mr->list);
ent->cur--;
spin_unlock_irq(&ent->lock);
if (ent->cur < ent->limit)
queue_work(cache->wq, &ent->work);
return mr;
}
}
}
static struct mlx5_ib_mr *alloc_cached_mr(struct mlx5_ib_dev *dev, int order) static struct mlx5_ib_mr *alloc_cached_mr(struct mlx5_ib_dev *dev, int order)
{ {
struct mlx5_mr_cache *cache = &dev->cache; struct mlx5_mr_cache *cache = &dev->cache;
...@@ -456,12 +495,12 @@ static struct mlx5_ib_mr *alloc_cached_mr(struct mlx5_ib_dev *dev, int order) ...@@ -456,12 +495,12 @@ static struct mlx5_ib_mr *alloc_cached_mr(struct mlx5_ib_dev *dev, int order)
int i; int i;
c = order2idx(dev, order); c = order2idx(dev, order);
if (c < 0 || c >= MAX_MR_CACHE_ENTRIES) { if (c < 0 || c > MAX_UMR_CACHE_ENTRY) {
mlx5_ib_warn(dev, "order %d, cache index %d\n", order, c); mlx5_ib_warn(dev, "order %d, cache index %d\n", order, c);
return NULL; return NULL;
} }
for (i = c; i < MAX_MR_CACHE_ENTRIES; i++) { for (i = c; i < MAX_UMR_CACHE_ENTRY; i++) {
ent = &cache->ent[i]; ent = &cache->ent[i];
mlx5_ib_dbg(dev, "order %d, cache index %d\n", ent->order, i); mlx5_ib_dbg(dev, "order %d, cache index %d\n", ent->order, i);
...@@ -488,7 +527,7 @@ static struct mlx5_ib_mr *alloc_cached_mr(struct mlx5_ib_dev *dev, int order) ...@@ -488,7 +527,7 @@ static struct mlx5_ib_mr *alloc_cached_mr(struct mlx5_ib_dev *dev, int order)
return mr; return mr;
} }
static void free_cached_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr) void mlx5_mr_cache_free(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr)
{ {
struct mlx5_mr_cache *cache = &dev->cache; struct mlx5_mr_cache *cache = &dev->cache;
struct mlx5_cache_ent *ent; struct mlx5_cache_ent *ent;
...@@ -500,6 +539,10 @@ static void free_cached_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr) ...@@ -500,6 +539,10 @@ static void free_cached_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr)
mlx5_ib_warn(dev, "order %d, cache index %d\n", mr->order, c); mlx5_ib_warn(dev, "order %d, cache index %d\n", mr->order, c);
return; return;
} }
if (unreg_umr(dev, mr))
return;
ent = &cache->ent[c]; ent = &cache->ent[c];
spin_lock_irq(&ent->lock); spin_lock_irq(&ent->lock);
list_add_tail(&mr->list, &ent->head); list_add_tail(&mr->list, &ent->head);
...@@ -602,7 +645,6 @@ int mlx5_mr_cache_init(struct mlx5_ib_dev *dev) ...@@ -602,7 +645,6 @@ int mlx5_mr_cache_init(struct mlx5_ib_dev *dev)
{ {
struct mlx5_mr_cache *cache = &dev->cache; struct mlx5_mr_cache *cache = &dev->cache;
struct mlx5_cache_ent *ent; struct mlx5_cache_ent *ent;
int limit;
int err; int err;
int i; int i;
...@@ -615,26 +657,33 @@ int mlx5_mr_cache_init(struct mlx5_ib_dev *dev) ...@@ -615,26 +657,33 @@ int mlx5_mr_cache_init(struct mlx5_ib_dev *dev)
setup_timer(&dev->delay_timer, delay_time_func, (unsigned long)dev); setup_timer(&dev->delay_timer, delay_time_func, (unsigned long)dev);
for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) { for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) {
INIT_LIST_HEAD(&cache->ent[i].head);
spin_lock_init(&cache->ent[i].lock);
ent = &cache->ent[i]; ent = &cache->ent[i];
INIT_LIST_HEAD(&ent->head); INIT_LIST_HEAD(&ent->head);
spin_lock_init(&ent->lock); spin_lock_init(&ent->lock);
ent->order = i + 2; ent->order = i + 2;
ent->dev = dev; ent->dev = dev;
ent->limit = 0;
if ((dev->mdev->profile->mask & MLX5_PROF_MASK_MR_CACHE) && init_completion(&ent->compl);
mlx5_core_is_pf(dev->mdev) &&
use_umr(dev, ent->order))
limit = dev->mdev->profile->mr_cache[i].limit;
else
limit = 0;
INIT_WORK(&ent->work, cache_work_func); INIT_WORK(&ent->work, cache_work_func);
INIT_DELAYED_WORK(&ent->dwork, delayed_cache_work_func); INIT_DELAYED_WORK(&ent->dwork, delayed_cache_work_func);
ent->limit = limit;
queue_work(cache->wq, &ent->work); queue_work(cache->wq, &ent->work);
if (i > MAX_UMR_CACHE_ENTRY)
continue;
if (!use_umr(dev, ent->order))
continue;
ent->page = PAGE_SHIFT;
ent->xlt = (1 << ent->order) * sizeof(struct mlx5_mtt) /
MLX5_IB_UMR_OCTOWORD;
ent->access_mode = MLX5_MKC_ACCESS_MODE_MTT;
if ((dev->mdev->profile->mask & MLX5_PROF_MASK_MR_CACHE) &&
mlx5_core_is_pf(dev->mdev))
ent->limit = dev->mdev->profile->mr_cache[i].limit;
else
ent->limit = 0;
} }
err = mlx5_mr_cache_debugfs_init(dev); err = mlx5_mr_cache_debugfs_init(dev);
...@@ -758,7 +807,7 @@ static int get_octo_len(u64 addr, u64 len, int page_size) ...@@ -758,7 +807,7 @@ static int get_octo_len(u64 addr, u64 len, int page_size)
static int use_umr(struct mlx5_ib_dev *dev, int order) static int use_umr(struct mlx5_ib_dev *dev, int order)
{ {
if (MLX5_CAP_GEN(dev->mdev, umr_extended_translation_offset)) if (MLX5_CAP_GEN(dev->mdev, umr_extended_translation_offset))
return order < MAX_MR_CACHE_ENTRIES + 2; return order <= MAX_UMR_CACHE_ENTRY + 2;
return order <= MLX5_MAX_UMR_SHIFT; return order <= MLX5_MAX_UMR_SHIFT;
} }
...@@ -871,7 +920,7 @@ static struct mlx5_ib_mr *reg_umr(struct ib_pd *pd, struct ib_umem *umem, ...@@ -871,7 +920,7 @@ static struct mlx5_ib_mr *reg_umr(struct ib_pd *pd, struct ib_umem *umem,
MLX5_IB_UPD_XLT_ENABLE); MLX5_IB_UPD_XLT_ENABLE);
if (err) { if (err) {
free_cached_mr(dev, mr); mlx5_mr_cache_free(dev, mr);
return ERR_PTR(err); return ERR_PTR(err);
} }
...@@ -1091,6 +1140,7 @@ static struct mlx5_ib_mr *reg_create(struct ib_mr *ibmr, struct ib_pd *pd, ...@@ -1091,6 +1140,7 @@ static struct mlx5_ib_mr *reg_create(struct ib_mr *ibmr, struct ib_pd *pd,
goto err_2; goto err_2;
} }
mr->mmkey.type = MLX5_MKEY_MR; mr->mmkey.type = MLX5_MKEY_MR;
mr->desc_size = sizeof(struct mlx5_mtt);
mr->umem = umem; mr->umem = umem;
mr->dev = dev; mr->dev = dev;
mr->live = 1; mr->live = 1;
...@@ -1398,12 +1448,7 @@ static int clean_mr(struct mlx5_ib_mr *mr) ...@@ -1398,12 +1448,7 @@ static int clean_mr(struct mlx5_ib_mr *mr)
return err; return err;
} }
} else { } else {
err = unreg_umr(dev, mr); mlx5_mr_cache_free(dev, mr);
if (err) {
mlx5_ib_warn(dev, "failed unregister\n");
return err;
}
free_cached_mr(dev, mr);
} }
if (!umred) if (!umred)
......
...@@ -1052,7 +1052,8 @@ enum { ...@@ -1052,7 +1052,8 @@ enum {
}; };
enum { enum {
MAX_MR_CACHE_ENTRIES = 21, MAX_UMR_CACHE_ENTRY = 20,
MAX_MR_CACHE_ENTRIES
}; };
enum { enum {
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment