Commit b58fc804 authored by Mike Marciniszyn's avatar Mike Marciniszyn Committed by Doug Ledford

IB/hfi1: Eliminate synchronize_rcu() in mr delete

The synchronize_rcu() call can be eliminated to improve memory deregistration
performance.

There are two key fields involved:
- The rcu pointer itself
- the lkey_published field

To close the window between the rcu read of the mregion pointer and the
reference count the code should:

1. To lkey/rkey validation (reader)

Read the rcu pointer.  If the pointer is non-NULL, get a reference.

To the current validation tests use a READ_ONCE() on the lkey_published.

Upon any failure release the reference.

2. To the remove logic (delete)

Insure the published is zeroed prior to setting the pointer to NULL.
This requires using rcu_assign_pointer() to insure lkey_published
is written prior to the NULL.

3. To the insert logic (add)

Insure the published is set use an rcu_assign_pointer() to insure the
pointer is after all MR fields.
Reviewed-by: default avatarDennis Dalessandro <dennis.dalessandro@intel.com>
Signed-off-by: default avatarMike Marciniszyn <mike.marciniszyn@intel.com>
Signed-off-by: default avatarDennis Dalessandro <dennis.dalessandro@intel.com>
Signed-off-by: default avatarDoug Ledford <dledford@redhat.com>
parent 243d9f43
...@@ -191,8 +191,9 @@ static int rvt_alloc_lkey(struct rvt_mregion *mr, int dma_region) ...@@ -191,8 +191,9 @@ static int rvt_alloc_lkey(struct rvt_mregion *mr, int dma_region)
tmr = rcu_access_pointer(dev->dma_mr); tmr = rcu_access_pointer(dev->dma_mr);
if (!tmr) { if (!tmr) {
rcu_assign_pointer(dev->dma_mr, mr);
mr->lkey_published = 1; mr->lkey_published = 1;
/* Insure published written first */
rcu_assign_pointer(dev->dma_mr, mr);
rvt_get_mr(mr); rvt_get_mr(mr);
} }
goto success; goto success;
...@@ -224,8 +225,9 @@ static int rvt_alloc_lkey(struct rvt_mregion *mr, int dma_region) ...@@ -224,8 +225,9 @@ static int rvt_alloc_lkey(struct rvt_mregion *mr, int dma_region)
mr->lkey |= 1 << 8; mr->lkey |= 1 << 8;
rkt->gen++; rkt->gen++;
} }
rcu_assign_pointer(rkt->table[r], mr);
mr->lkey_published = 1; mr->lkey_published = 1;
/* Insure published written first */
rcu_assign_pointer(rkt->table[r], mr);
success: success:
spin_unlock_irqrestore(&rkt->lock, flags); spin_unlock_irqrestore(&rkt->lock, flags);
out: out:
...@@ -253,23 +255,24 @@ static void rvt_free_lkey(struct rvt_mregion *mr) ...@@ -253,23 +255,24 @@ static void rvt_free_lkey(struct rvt_mregion *mr)
spin_lock_irqsave(&rkt->lock, flags); spin_lock_irqsave(&rkt->lock, flags);
if (!lkey) { if (!lkey) {
if (mr->lkey_published) { if (mr->lkey_published) {
RCU_INIT_POINTER(dev->dma_mr, NULL); mr->lkey_published = 0;
/* insure published is written before pointer */
rcu_assign_pointer(dev->dma_mr, NULL);
rvt_put_mr(mr); rvt_put_mr(mr);
} }
} else { } else {
if (!mr->lkey_published) if (!mr->lkey_published)
goto out; goto out;
r = lkey >> (32 - dev->dparms.lkey_table_size); r = lkey >> (32 - dev->dparms.lkey_table_size);
RCU_INIT_POINTER(rkt->table[r], NULL); mr->lkey_published = 0;
/* insure published is written before pointer */
rcu_assign_pointer(rkt->table[r], NULL);
} }
mr->lkey_published = 0;
freed++; freed++;
out: out:
spin_unlock_irqrestore(&rkt->lock, flags); spin_unlock_irqrestore(&rkt->lock, flags);
if (freed) { if (freed)
synchronize_rcu();
percpu_ref_kill(&mr->refcount); percpu_ref_kill(&mr->refcount);
}
} }
static struct rvt_mr *__rvt_alloc_mr(int count, struct ib_pd *pd) static struct rvt_mr *__rvt_alloc_mr(int count, struct ib_pd *pd)
...@@ -822,16 +825,21 @@ int rvt_lkey_ok(struct rvt_lkey_table *rkt, struct rvt_pd *pd, ...@@ -822,16 +825,21 @@ int rvt_lkey_ok(struct rvt_lkey_table *rkt, struct rvt_pd *pd,
goto ok; goto ok;
} }
mr = rcu_dereference(rkt->table[sge->lkey >> rkt->shift]); mr = rcu_dereference(rkt->table[sge->lkey >> rkt->shift]);
if (unlikely(!mr || atomic_read(&mr->lkey_invalid) || if (!mr)
mr->lkey != sge->lkey || mr->pd != &pd->ibpd))
goto bail; goto bail;
rvt_get_mr(mr);
if (!READ_ONCE(mr->lkey_published))
goto bail_unref;
if (unlikely(atomic_read(&mr->lkey_invalid) ||
mr->lkey != sge->lkey || mr->pd != &pd->ibpd))
goto bail_unref;
off = sge->addr - mr->user_base; off = sge->addr - mr->user_base;
if (unlikely(sge->addr < mr->user_base || if (unlikely(sge->addr < mr->user_base ||
off + sge->length > mr->length || off + sge->length > mr->length ||
(mr->access_flags & acc) != acc)) (mr->access_flags & acc) != acc))
goto bail; goto bail_unref;
rvt_get_mr(mr);
rcu_read_unlock(); rcu_read_unlock();
off += mr->offset; off += mr->offset;
...@@ -867,6 +875,8 @@ int rvt_lkey_ok(struct rvt_lkey_table *rkt, struct rvt_pd *pd, ...@@ -867,6 +875,8 @@ int rvt_lkey_ok(struct rvt_lkey_table *rkt, struct rvt_pd *pd,
isge->n = n; isge->n = n;
ok: ok:
return 1; return 1;
bail_unref:
rvt_put_mr(mr);
bail: bail:
rcu_read_unlock(); rcu_read_unlock();
return 0; return 0;
...@@ -922,15 +932,20 @@ int rvt_rkey_ok(struct rvt_qp *qp, struct rvt_sge *sge, ...@@ -922,15 +932,20 @@ int rvt_rkey_ok(struct rvt_qp *qp, struct rvt_sge *sge,
} }
mr = rcu_dereference(rkt->table[rkey >> rkt->shift]); mr = rcu_dereference(rkt->table[rkey >> rkt->shift]);
if (unlikely(!mr || atomic_read(&mr->lkey_invalid) || if (!mr)
mr->lkey != rkey || qp->ibqp.pd != mr->pd))
goto bail; goto bail;
rvt_get_mr(mr);
/* insure mr read is before test */
if (!READ_ONCE(mr->lkey_published))
goto bail_unref;
if (unlikely(atomic_read(&mr->lkey_invalid) ||
mr->lkey != rkey || qp->ibqp.pd != mr->pd))
goto bail_unref;
off = vaddr - mr->iova; off = vaddr - mr->iova;
if (unlikely(vaddr < mr->iova || off + len > mr->length || if (unlikely(vaddr < mr->iova || off + len > mr->length ||
(mr->access_flags & acc) == 0)) (mr->access_flags & acc) == 0))
goto bail; goto bail_unref;
rvt_get_mr(mr);
rcu_read_unlock(); rcu_read_unlock();
off += mr->offset; off += mr->offset;
...@@ -966,6 +981,8 @@ int rvt_rkey_ok(struct rvt_qp *qp, struct rvt_sge *sge, ...@@ -966,6 +981,8 @@ int rvt_rkey_ok(struct rvt_qp *qp, struct rvt_sge *sge,
sge->n = n; sge->n = n;
ok: ok:
return 1; return 1;
bail_unref:
rvt_put_mr(mr);
bail: bail:
rcu_read_unlock(); rcu_read_unlock();
return 0; return 0;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment