Skip to content

L
linux
Commit 2804d0f1 authored by Kent Overstreet's avatar Kent Overstreet
Browse files
Options

six locks: Split out seq, use atomic_t instead of atomic64_t 

Signed-off-by: default avatarKent Overstreet <kent.overstreet@linux.dev>
parent a4e9e1f0
Show whitespace changes
Inline Side-by-side
Showing with 58 additions and 71 deletions
fs/bcachefs/six.c
View file @ 2804d0f1
...@@ -32,25 +32,18 @@ static void do_six_unlock_type(struct six_lock *lock, enum six_lock_type type); ...@@ -32,25 +32,18 @@ static void do_six_unlock_type(struct six_lock *lock, enum six_lock_type type);
* bits 29-30 has read waiters * bits 29-30 has read waiters
* bits 30-31 has intent waiters * bits 30-31 has intent waiters
* bits 31-32 has write waiters * bits 31-32 has write waiters
* bits 32-64 sequence number: incremented on every write lock or
* unlock, thus bit 33 (sequence number odd) indicates
* lock is currently held for write
*/ */
#define SIX_STATE_READ_OFFSET 0 #define SIX_STATE_READ_OFFSET 0
#define SIX_STATE_READ_BITS 26 #define SIX_STATE_READ_BITS 26
#define SIX_STATE_READ_LOCK ~(~0ULL << 26) #define SIX_STATE_READ_LOCK ~(~0U << 26)
#define SIX_STATE_INTENT_HELD (1ULL << 26) #define SIX_STATE_INTENT_HELD (1U << 26)
#define SIX_STATE_WRITE_LOCK (1ULL << 27) #define SIX_STATE_WRITE_LOCK (1U << 27)
#define SIX_STATE_NOSPIN (1ULL << 28) #define SIX_STATE_NOSPIN (1U << 28)
#define SIX_STATE_WAITING_READ (1ULL << (29 + SIX_LOCK_read)) #define SIX_STATE_WAITING_READ (1U << (29 + SIX_LOCK_read))
#define SIX_STATE_WAITING_INTENT (1ULL << (29 + SIX_LOCK_intent)) #define SIX_STATE_WAITING_INTENT (1U << (29 + SIX_LOCK_intent))
#define SIX_STATE_WAITING_WRITE (1ULL << (29 + SIX_LOCK_write)) #define SIX_STATE_WAITING_WRITE (1U << (29 + SIX_LOCK_write))
#define SIX_STATE_SEQ_OFFSET 32
#define SIX_STATE_SEQ_BITS 32
#define SIX_STATE_SEQ (~0ULL << 32)
#define SIX_LOCK_HELD_read SIX_STATE_READ_LOCK #define SIX_LOCK_HELD_read SIX_STATE_READ_LOCK
#define SIX_LOCK_HELD_intent SIX_STATE_INTENT_HELD #define SIX_LOCK_HELD_intent SIX_STATE_INTENT_HELD
...@@ -58,13 +51,13 @@ static void do_six_unlock_type(struct six_lock *lock, enum six_lock_type type); ...@@ -58,13 +51,13 @@ static void do_six_unlock_type(struct six_lock *lock, enum six_lock_type type);
struct six_lock_vals { struct six_lock_vals {
/* Value we add to the lock in order to take the lock: */ /* Value we add to the lock in order to take the lock: */
u64 lock_val; u32 lock_val;
/* If the lock has this value (used as a mask), taking the lock fails: */ /* If the lock has this value (used as a mask), taking the lock fails: */
u64 lock_fail; u32 lock_fail;
/* Mask that indicates lock is held for this type: */ /* Mask that indicates lock is held for this type: */
u64 held_mask; u32 held_mask;
/* Waitlist we wakeup when releasing the lock: */ /* Waitlist we wakeup when releasing the lock: */
enum six_lock_type unlock_wakeup; enum six_lock_type unlock_wakeup;
...@@ -72,7 +65,7 @@ struct six_lock_vals { ...@@ -72,7 +65,7 @@ struct six_lock_vals {
static const struct six_lock_vals l[] = { static const struct six_lock_vals l[] = {
[SIX_LOCK_read] = { [SIX_LOCK_read] = {
.lock_val = 1ULL << SIX_STATE_READ_OFFSET, .lock_val = 1U << SIX_STATE_READ_OFFSET,
.lock_fail = SIX_LOCK_HELD_write, .lock_fail = SIX_LOCK_HELD_write,
.held_mask = SIX_LOCK_HELD_read, .held_mask = SIX_LOCK_HELD_read,
.unlock_wakeup = SIX_LOCK_write, .unlock_wakeup = SIX_LOCK_write,
...@@ -91,25 +84,20 @@ static const struct six_lock_vals l[] = { ...@@ -91,25 +84,20 @@ static const struct six_lock_vals l[] = {
}, },
}; };
static inline u32 six_state_seq(u64 state) static inline void six_set_bitmask(struct six_lock *lock, u32 mask)
{
return state >> SIX_STATE_SEQ_OFFSET;
}
static inline void six_set_bitmask(struct six_lock *lock, u64 mask)
{ {
if ((atomic64_read(&lock->state) & mask) != mask) if ((atomic_read(&lock->state) & mask) != mask)
atomic64_or(mask, &lock->state); atomic_or(mask, &lock->state);
} }
static inline void six_clear_bitmask(struct six_lock *lock, u64 mask) static inline void six_clear_bitmask(struct six_lock *lock, u32 mask)
{ {
if (atomic64_read(&lock->state) & mask) if (atomic_read(&lock->state) & mask)
atomic64_and(~mask, &lock->state); atomic_and(~mask, &lock->state);
} }
static inline void six_set_owner(struct six_lock *lock, enum six_lock_type type, static inline void six_set_owner(struct six_lock *lock, enum six_lock_type type,
u64 old, struct task_struct *owner) u32 old, struct task_struct *owner)
{ {
if (type != SIX_LOCK_intent) if (type != SIX_LOCK_intent)
return; return;
...@@ -145,13 +133,11 @@ static int __do_six_trylock(struct six_lock *lock, enum six_lock_type type, ...@@ -145,13 +133,11 @@ static int __do_six_trylock(struct six_lock *lock, enum six_lock_type type,
struct task_struct *task, bool try) struct task_struct *task, bool try)
{ {
int ret; int ret;
u64 old, new, v; u32 old, new, v;
EBUG_ON(type == SIX_LOCK_write && lock->owner != task); EBUG_ON(type == SIX_LOCK_write && lock->owner != task);
EBUG_ON(type == SIX_LOCK_write && EBUG_ON(type == SIX_LOCK_write &&
(try != !(atomic64_read(&lock->state) & SIX_LOCK_HELD_write))); (try != !(atomic_read(&lock->state) & SIX_STATE_WRITE_LOCK)));
EBUG_ON(type == SIX_LOCK_write &&
(try != !(atomic64_read(&lock->state) & SIX_STATE_WRITE_LOCK)));
/* /*
* Percpu reader mode: * Percpu reader mode:
...@@ -186,7 +172,7 @@ static int __do_six_trylock(struct six_lock *lock, enum six_lock_type type, ...@@ -186,7 +172,7 @@ static int __do_six_trylock(struct six_lock *lock, enum six_lock_type type,
smp_mb(); smp_mb();
old = atomic64_read(&lock->state); old = atomic_read(&lock->state);
ret = !(old & l[type].lock_fail); ret = !(old & l[type].lock_fail);
this_cpu_sub(*lock->readers, !ret); this_cpu_sub(*lock->readers, !ret);
...@@ -196,19 +182,19 @@ static int __do_six_trylock(struct six_lock *lock, enum six_lock_type type, ...@@ -196,19 +182,19 @@ static int __do_six_trylock(struct six_lock *lock, enum six_lock_type type,
ret = -1 - SIX_LOCK_write; ret = -1 - SIX_LOCK_write;
} else if (type == SIX_LOCK_write && lock->readers) { } else if (type == SIX_LOCK_write && lock->readers) {
if (try) { if (try) {
atomic64_add(SIX_STATE_WRITE_LOCK, &lock->state); atomic_add(SIX_STATE_WRITE_LOCK, &lock->state);
smp_mb__after_atomic(); smp_mb__after_atomic();
} }
ret = !pcpu_read_count(lock); ret = !pcpu_read_count(lock);
if (try && !ret) { if (try && !ret) {
old = atomic64_sub_return(SIX_STATE_WRITE_LOCK, &lock->state); old = atomic_sub_return(SIX_STATE_WRITE_LOCK, &lock->state);
if (old & SIX_STATE_WAITING_READ) if (old & SIX_STATE_WAITING_READ)
ret = -1 - SIX_LOCK_read; ret = -1 - SIX_LOCK_read;
} }
} else { } else {
v = atomic64_read(&lock->state); v = atomic_read(&lock->state);
do { do {
new = old = v; new = old = v;
...@@ -218,16 +204,16 @@ static int __do_six_trylock(struct six_lock *lock, enum six_lock_type type, ...@@ -218,16 +204,16 @@ static int __do_six_trylock(struct six_lock *lock, enum six_lock_type type,
break; break;
new += l[type].lock_val; new += l[type].lock_val;
} while ((v = atomic64_cmpxchg_acquire(&lock->state, old, new)) != old); } while ((v = atomic_cmpxchg_acquire(&lock->state, old, new)) != old);
EBUG_ON(ret && !(atomic64_read(&lock->state) & l[type].held_mask)); EBUG_ON(ret && !(atomic_read(&lock->state) & l[type].held_mask));
} }
if (ret > 0) if (ret > 0)
six_set_owner(lock, type, old, task); six_set_owner(lock, type, old, task);
EBUG_ON(type == SIX_LOCK_write && try && ret <= 0 && EBUG_ON(type == SIX_LOCK_write && try && ret <= 0 &&
(atomic64_read(&lock->state) & SIX_STATE_WRITE_LOCK)); (atomic_read(&lock->state) & SIX_STATE_WRITE_LOCK));
return ret; return ret;
} }
...@@ -277,7 +263,7 @@ static void __six_lock_wakeup(struct six_lock *lock, enum six_lock_type lock_typ ...@@ -277,7 +263,7 @@ static void __six_lock_wakeup(struct six_lock *lock, enum six_lock_type lock_typ
} }
__always_inline __always_inline
static void six_lock_wakeup(struct six_lock *lock, u64 state, static void six_lock_wakeup(struct six_lock *lock, u32 state,
enum six_lock_type lock_type) enum six_lock_type lock_type)
{ {
if (lock_type == SIX_LOCK_write && (state & SIX_LOCK_HELD_read)) if (lock_type == SIX_LOCK_write && (state & SIX_LOCK_HELD_read))
...@@ -317,7 +303,8 @@ bool six_trylock_ip(struct six_lock *lock, enum six_lock_type type, unsigned lon ...@@ -317,7 +303,8 @@ bool six_trylock_ip(struct six_lock *lock, enum six_lock_type type, unsigned lon
if (type != SIX_LOCK_write) if (type != SIX_LOCK_write)
six_acquire(&lock->dep_map, 1, type == SIX_LOCK_read, ip); six_acquire(&lock->dep_map, 1, type == SIX_LOCK_read, ip);
else else
atomic64_add(1ULL << SIX_STATE_SEQ_OFFSET, &lock->state); lock->seq++;
return true; return true;
} }
EXPORT_SYMBOL_GPL(six_trylock_ip); EXPORT_SYMBOL_GPL(six_trylock_ip);
...@@ -482,12 +469,12 @@ static int six_lock_slowpath(struct six_lock *lock, enum six_lock_type type, ...@@ -482,12 +469,12 @@ static int six_lock_slowpath(struct six_lock *lock, enum six_lock_type type,
six_lock_should_sleep_fn should_sleep_fn, void *p, six_lock_should_sleep_fn should_sleep_fn, void *p,
unsigned long ip) unsigned long ip)
{ {
u64 old; u32 old;
int ret = 0; int ret = 0;
if (type == SIX_LOCK_write) { if (type == SIX_LOCK_write) {
EBUG_ON(atomic64_read(&lock->state) & SIX_STATE_WRITE_LOCK); EBUG_ON(atomic_read(&lock->state) & SIX_STATE_WRITE_LOCK);
atomic64_add(SIX_STATE_WRITE_LOCK, &lock->state); atomic_add(SIX_STATE_WRITE_LOCK, &lock->state);
smp_mb__after_atomic(); smp_mb__after_atomic();
} }
...@@ -609,8 +596,7 @@ int six_lock_ip_waiter(struct six_lock *lock, enum six_lock_type type, ...@@ -609,8 +596,7 @@ int six_lock_ip_waiter(struct six_lock *lock, enum six_lock_type type,
ret = do_six_trylock(lock, type, true) ? 0 ret = do_six_trylock(lock, type, true) ? 0
: six_lock_slowpath(lock, type, wait, should_sleep_fn, p, ip); : six_lock_slowpath(lock, type, wait, should_sleep_fn, p, ip);
if (!ret && type == SIX_LOCK_write) lock->seq += !ret && type == SIX_LOCK_write;
atomic64_add(1ULL << SIX_STATE_SEQ_OFFSET, &lock->state);
if (ret && type != SIX_LOCK_write) if (ret && type != SIX_LOCK_write)
six_release(&lock->dep_map, ip); six_release(&lock->dep_map, ip);
...@@ -624,7 +610,7 @@ EXPORT_SYMBOL_GPL(six_lock_ip_waiter); ...@@ -624,7 +610,7 @@ EXPORT_SYMBOL_GPL(six_lock_ip_waiter);
__always_inline __always_inline
static void do_six_unlock_type(struct six_lock *lock, enum six_lock_type type) static void do_six_unlock_type(struct six_lock *lock, enum six_lock_type type)
{ {
u64 state; u32 state;
if (type == SIX_LOCK_intent) if (type == SIX_LOCK_intent)
lock->owner = NULL; lock->owner = NULL;
...@@ -634,15 +620,15 @@ static void do_six_unlock_type(struct six_lock *lock, enum six_lock_type type) ...@@ -634,15 +620,15 @@ static void do_six_unlock_type(struct six_lock *lock, enum six_lock_type type)
smp_mb(); /* unlock barrier */ smp_mb(); /* unlock barrier */
this_cpu_dec(*lock->readers); this_cpu_dec(*lock->readers);
smp_mb(); /* between unlocking and checking for waiters */ smp_mb(); /* between unlocking and checking for waiters */
state = atomic64_read(&lock->state); state = atomic_read(&lock->state);
} else { } else {
u64 v = l[type].lock_val; u32 v = l[type].lock_val;
if (type != SIX_LOCK_read) if (type != SIX_LOCK_read)
v += atomic64_read(&lock->state) & SIX_STATE_NOSPIN; v += atomic_read(&lock->state) & SIX_STATE_NOSPIN;
EBUG_ON(!(atomic64_read(&lock->state) & l[type].held_mask)); EBUG_ON(!(atomic_read(&lock->state) & l[type].held_mask));
state = atomic64_sub_return_release(v, &lock->state); state = atomic_sub_return_release(v, &lock->state);
} }
six_lock_wakeup(lock, state, l[type].unlock_wakeup); six_lock_wakeup(lock, state, l[type].unlock_wakeup);
...@@ -666,7 +652,7 @@ static void do_six_unlock_type(struct six_lock *lock, enum six_lock_type type) ...@@ -666,7 +652,7 @@ static void do_six_unlock_type(struct six_lock *lock, enum six_lock_type type)
void six_unlock_ip(struct six_lock *lock, enum six_lock_type type, unsigned long ip) void six_unlock_ip(struct six_lock *lock, enum six_lock_type type, unsigned long ip)
{ {
EBUG_ON(type == SIX_LOCK_write && EBUG_ON(type == SIX_LOCK_write &&
!(atomic64_read(&lock->state) & SIX_LOCK_HELD_intent)); !(atomic_read(&lock->state) & SIX_LOCK_HELD_intent));
EBUG_ON((type == SIX_LOCK_write || EBUG_ON((type == SIX_LOCK_write ||
type == SIX_LOCK_intent) && type == SIX_LOCK_intent) &&
lock->owner != current); lock->owner != current);
...@@ -674,7 +660,7 @@ void six_unlock_ip(struct six_lock *lock, enum six_lock_type type, unsigned long ...@@ -674,7 +660,7 @@ void six_unlock_ip(struct six_lock *lock, enum six_lock_type type, unsigned long
if (type != SIX_LOCK_write) if (type != SIX_LOCK_write)
six_release(&lock->dep_map, ip); six_release(&lock->dep_map, ip);
else else
atomic64_add(1ULL << SIX_STATE_SEQ_OFFSET, &lock->state); lock->seq++;
if (type == SIX_LOCK_intent && if (type == SIX_LOCK_intent &&
lock->intent_lock_recurse) { lock->intent_lock_recurse) {
...@@ -710,7 +696,7 @@ EXPORT_SYMBOL_GPL(six_lock_downgrade); ...@@ -710,7 +696,7 @@ EXPORT_SYMBOL_GPL(six_lock_downgrade);
*/ */
bool six_lock_tryupgrade(struct six_lock *lock) bool six_lock_tryupgrade(struct six_lock *lock)
{ {
u64 old, new, v = atomic64_read(&lock->state); u32 old, new, v = atomic_read(&lock->state);
do { do {
new = old = v; new = old = v;
...@@ -724,7 +710,7 @@ bool six_lock_tryupgrade(struct six_lock *lock) ...@@ -724,7 +710,7 @@ bool six_lock_tryupgrade(struct six_lock *lock)
} }
new |= SIX_LOCK_HELD_intent; new |= SIX_LOCK_HELD_intent;
} while ((v = atomic64_cmpxchg_acquire(&lock->state, old, new)) != old); } while ((v = atomic_cmpxchg_acquire(&lock->state, old, new)) != old);
if (lock->readers) if (lock->readers)
this_cpu_dec(*lock->readers); this_cpu_dec(*lock->readers);
...@@ -786,14 +772,14 @@ void six_lock_increment(struct six_lock *lock, enum six_lock_type type) ...@@ -786,14 +772,14 @@ void six_lock_increment(struct six_lock *lock, enum six_lock_type type)
if (lock->readers) { if (lock->readers) {
this_cpu_inc(*lock->readers); this_cpu_inc(*lock->readers);
} else { } else {
EBUG_ON(!(atomic64_read(&lock->state) & EBUG_ON(!(atomic_read(&lock->state) &
(SIX_LOCK_HELD_read| (SIX_LOCK_HELD_read|
SIX_LOCK_HELD_intent))); SIX_LOCK_HELD_intent)));
atomic64_add(l[type].lock_val, &lock->state); atomic_add(l[type].lock_val, &lock->state);
} }
break; break;
case SIX_LOCK_intent: case SIX_LOCK_intent:
EBUG_ON(!(atomic64_read(&lock->state) & SIX_LOCK_HELD_intent)); EBUG_ON(!(atomic_read(&lock->state) & SIX_LOCK_HELD_intent));
lock->intent_lock_recurse++; lock->intent_lock_recurse++;
break; break;
case SIX_LOCK_write: case SIX_LOCK_write:
...@@ -815,7 +801,7 @@ EXPORT_SYMBOL_GPL(six_lock_increment); ...@@ -815,7 +801,7 @@ EXPORT_SYMBOL_GPL(six_lock_increment);
*/ */
void six_lock_wakeup_all(struct six_lock *lock) void six_lock_wakeup_all(struct six_lock *lock)
{ {
u64 state = atomic64_read(&lock->state); u32 state = atomic_read(&lock->state);
struct six_lock_waiter *w; struct six_lock_waiter *w;
six_lock_wakeup(lock, state, SIX_LOCK_read); six_lock_wakeup(lock, state, SIX_LOCK_read);
...@@ -840,11 +826,11 @@ struct six_lock_count six_lock_counts(struct six_lock *lock) ...@@ -840,11 +826,11 @@ struct six_lock_count six_lock_counts(struct six_lock *lock)
struct six_lock_count ret; struct six_lock_count ret;
ret.n[SIX_LOCK_read] = !lock->readers ret.n[SIX_LOCK_read] = !lock->readers
? atomic64_read(&lock->state) & SIX_STATE_READ_LOCK ? atomic_read(&lock->state) & SIX_STATE_READ_LOCK
: pcpu_read_count(lock); : pcpu_read_count(lock);
ret.n[SIX_LOCK_intent] = !!(atomic64_read(&lock->state) & SIX_LOCK_HELD_intent) + ret.n[SIX_LOCK_intent] = !!(atomic_read(&lock->state) & SIX_LOCK_HELD_intent) +
lock->intent_lock_recurse; lock->intent_lock_recurse;
ret.n[SIX_LOCK_write] = !!(atomic64_read(&lock->state) & SIX_LOCK_HELD_write); ret.n[SIX_LOCK_write] = !!(atomic_read(&lock->state) & SIX_LOCK_HELD_write);
return ret; return ret;
} }
...@@ -875,9 +861,9 @@ void six_lock_readers_add(struct six_lock *lock, int nr) ...@@ -875,9 +861,9 @@ void six_lock_readers_add(struct six_lock *lock, int nr)
if (lock->readers) { if (lock->readers) {
this_cpu_add(*lock->readers, nr); this_cpu_add(*lock->readers, nr);
} else { } else {
EBUG_ON((int) (atomic64_read(&lock->state) & SIX_STATE_READ_LOCK) + nr < 0); EBUG_ON((int) (atomic_read(&lock->state) & SIX_STATE_READ_LOCK) + nr < 0);
/* reader count starts at bit 0 */ /* reader count starts at bit 0 */
atomic64_add(nr, &lock->state); atomic_add(nr, &lock->state);
} }
} }
EXPORT_SYMBOL_GPL(six_lock_readers_add); EXPORT_SYMBOL_GPL(six_lock_readers_add);
...@@ -892,7 +878,7 @@ EXPORT_SYMBOL_GPL(six_lock_readers_add); ...@@ -892,7 +878,7 @@ EXPORT_SYMBOL_GPL(six_lock_readers_add);
void six_lock_exit(struct six_lock *lock) void six_lock_exit(struct six_lock *lock)
{ {
WARN_ON(lock->readers && pcpu_read_count(lock)); WARN_ON(lock->readers && pcpu_read_count(lock));
WARN_ON(atomic64_read(&lock->state) & SIX_LOCK_HELD_read); WARN_ON(atomic_read(&lock->state) & SIX_LOCK_HELD_read);
free_percpu(lock->readers); free_percpu(lock->readers);
lock->readers = NULL; lock->readers = NULL;
...@@ -902,7 +888,7 @@ EXPORT_SYMBOL_GPL(six_lock_exit); ...@@ -902,7 +888,7 @@ EXPORT_SYMBOL_GPL(six_lock_exit);
void __six_lock_init(struct six_lock *lock, const char *name, void __six_lock_init(struct six_lock *lock, const char *name,
struct lock_class_key *key, enum six_lock_init_flags flags) struct lock_class_key *key, enum six_lock_init_flags flags)
{ {
atomic64_set(&lock->state, 0); atomic_set(&lock->state, 0);
raw_spin_lock_init(&lock->wait_lock); raw_spin_lock_init(&lock->wait_lock);
INIT_LIST_HEAD(&lock->wait_list); INIT_LIST_HEAD(&lock->wait_list);
#ifdef CONFIG_DEBUG_LOCK_ALLOC #ifdef CONFIG_DEBUG_LOCK_ALLOC
......
fs/bcachefs/six.h
View file @ 2804d0f1
...@@ -138,7 +138,8 @@ enum six_lock_type { ...@@ -138,7 +138,8 @@ enum six_lock_type {
}; };
struct six_lock { struct six_lock {
atomic64_t state; atomic_t state;
u32 seq;
unsigned intent_lock_recurse; unsigned intent_lock_recurse;
struct task_struct *owner; struct task_struct *owner;
unsigned __percpu *readers; unsigned __percpu *readers;
...@@ -196,7 +197,7 @@ do { \ ...@@ -196,7 +197,7 @@ do { \
*/ */
static inline u32 six_lock_seq(const struct six_lock *lock) static inline u32 six_lock_seq(const struct six_lock *lock)
{ {
return atomic64_read(&lock->state) >> 32; return lock->seq;
} }
bool six_trylock_ip(struct six_lock *lock, enum six_lock_type type, unsigned long ip); bool six_trylock_ip(struct six_lock *lock, enum six_lock_type type, unsigned long ip);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7