Skip to content

M
MariaDB
Commit 4e1ca388 authored by Marko Mäkelä's avatar Marko Mäkelä
Browse files
Options

MDEV-26781 InnoDB hangs when using SUX_LOCK_GENERIC 

The Shared/Update/Exclusive locks that were introduced in
commit 03ca6495 (MDEV-24142)
did not work correctly when a futex-like system call interface
was not available.

On all tested implementations (IBM AIX as well as FreeBSD and GNU/Linux
with the futex interface artificially disabled), the old implementation
would cause hangs in some SPATIAL INDEX tests (innodb_gis suite).
On FreeBSD, a hang was also observed in an encryption test.

We will simply emulate the futex system calls with a single mutex
and two condition variables, one for each wait queue. The condition
variables basically shadow the lock words and are used as wait queues,
just like the futex system calls would be.

The storage overhead of ssux_lock_impl will be increased by 32 bits
when using SUX_LOCK_GENERIC. Compared to the futex-based implementation,
the SUX_LOCK_GENERIC implementation has an overhead of
sizeof(pthread_mutex_t)+2*sizeof(pthread_cond_t).

rw_lock: Remove all SUX_LOCK_GENERIC extensions.

pthread_mutex_wrapper: A simple wrapper of pthread_mutex that
implements srw_spin_mutex and srw_mutex for SUX_LOCK_GENERIC.

srw_mutex_impl: Define this also for SUX_LOCK_GENERIC, but in
that case add the fields mutex, cond.

ssux_lock_impl: Define for SUX_LOCK_GENERIC with a minimal difference:
adding readers_cond.
parent ff994138
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 138 additions and 409 deletions
storage/innobase/include/rw_lock.h
View file @ 4e1ca388
......@@ -20,25 +20,7 @@ this program; if not, write to the Free Software Foundation, Inc.,
#include <atomic>
#include "my_dbug.h"
#if defined __linux__
/* futex(2): FUTEX_WAIT_PRIVATE, FUTEX_WAKE_PRIVATE */
#elif defined __OpenBSD__ || defined __FreeBSD__ || defined __DragonFly__
/* system calls similar to Linux futex(2) */
#elif defined _WIN32
/* SRWLOCK as well as WaitOnAddress(), WakeByAddressSingle() */
#else
# define SUX_LOCK_GENERIC /* fall back to generic synchronization primitives */
#endif
#if !defined SUX_LOCK_GENERIC && 0 /* defined SAFE_MUTEX */
# define SUX_LOCK_GENERIC /* Use dummy implementation for debugging purposes */
#endif
#ifdef SUX_LOCK_GENERIC
/** Simple read-update-write lock based on std::atomic */
#else
/** Simple read-write lock based on std::atomic */
#endif
class rw_lock
{
/** The lock word */
......@@ -53,10 +35,6 @@ class rw_lock
static constexpr uint32_t WRITER_WAITING= 1U << 30;
/** Flag to indicate that write_lock() or write_lock_wait() is pending */
static constexpr uint32_t WRITER_PENDING= WRITER | WRITER_WAITING;
#ifdef SUX_LOCK_GENERIC
/** Flag to indicate that an update lock exists */
static constexpr uint32_t UPDATER= 1U << 29;
#endif /* SUX_LOCK_GENERIC */
/** Start waiting for an exclusive lock. */
void write_lock_wait_start()
......@@ -84,12 +62,8 @@ class rw_lock
std::memory_order_relaxed);
}
/** Try to acquire a shared lock.
@tparam prioritize_updater whether to ignore WRITER_WAITING for UPDATER
@param l the value of the lock word
@return whether the lock was acquired */
#ifdef SUX_LOCK_GENERIC
template<bool prioritize_updater= false>
#endif /* SUX_LOCK_GENERIC */
bool read_trylock(uint32_t &l)
{
l= UNLOCKED;
......@@ -97,66 +71,11 @@ class rw_lock
std::memory_order_relaxed))
{
DBUG_ASSERT(!(WRITER & l) || !(~WRITER_PENDING & l));
#ifdef SUX_LOCK_GENERIC
DBUG_ASSERT((~(WRITER_PENDING | UPDATER) & l) < UPDATER);
if (prioritize_updater
? (WRITER & l) || ((WRITER_WAITING | UPDATER) & l) == WRITER_WAITING
: (WRITER_PENDING & l))
#else /* SUX_LOCK_GENERIC */
if (l & WRITER_PENDING)
#endif /* SUX_LOCK_GENERIC */
return false;
}
return true;
}
#ifdef SUX_LOCK_GENERIC
/** Try to acquire an update lock.
@param l the value of the lock word
@return whether the lock was acquired */
bool update_trylock(uint32_t &l)
{
l= UNLOCKED;
while (!lock.compare_exchange_strong(l, l | UPDATER,
std::memory_order_acquire,
std::memory_order_relaxed))
{
DBUG_ASSERT(!(WRITER & l) || !(~WRITER_PENDING & l));
DBUG_ASSERT((~(WRITER_PENDING | UPDATER) & l) < UPDATER);
if ((WRITER_PENDING | UPDATER) & l)
return false;
}
return true;
}
/** Try to upgrade an update lock to an exclusive lock.
@return whether the update lock was upgraded to exclusive */
bool upgrade_trylock()
{
auto l= UPDATER;
while (!lock.compare_exchange_strong(l, WRITER,
std::memory_order_acquire,
std::memory_order_relaxed))
{
/* Either conflicting (read) locks have been granted, or
the WRITER_WAITING flag was set by some thread that is waiting
to become WRITER. */
DBUG_ASSERT(((WRITER | UPDATER) & l) == UPDATER);
if (~(WRITER_WAITING | UPDATER) & l)
return false;
}
DBUG_ASSERT((l & ~WRITER_WAITING) == UPDATER);
/* Any thread that had set WRITER_WAITING will eventually be woken
up by ssux_lock_impl::x_unlock() or ssux_lock_impl::u_unlock()
(not ssux_lock_impl::wr_u_downgrade() to keep the code simple). */
return true;
}
/** Downgrade an exclusive lock to an update lock. */
void downgrade()
{
IF_DBUG_ASSERT(auto l=,)
lock.fetch_xor(WRITER | UPDATER, std::memory_order_relaxed);
DBUG_ASSERT((l & ~WRITER_WAITING) == WRITER);
}
#endif /* SUX_LOCK_GENERIC */
/** Wait for an exclusive lock.
@return whether the exclusive lock was acquired */
......@@ -183,24 +102,9 @@ class rw_lock
{
auto l= lock.fetch_sub(1, std::memory_order_release);
DBUG_ASSERT(!(l & WRITER)); /* no write lock must have existed */
#ifdef SUX_LOCK_GENERIC
DBUG_ASSERT(~(WRITER_PENDING | UPDATER) & l); /* at least one read lock */
return (~(WRITER_PENDING | UPDATER) & l) == 1;
#else /* SUX_LOCK_GENERIC */
DBUG_ASSERT(~(WRITER_PENDING) & l); /* at least one read lock */
return (~WRITER_PENDING & l) == 1;
#endif /* SUX_LOCK_GENERIC */
}
#ifdef SUX_LOCK_GENERIC
/** Release an update lock */
void update_unlock()
{
IF_DBUG_ASSERT(auto l=,)
lock.fetch_and(~UPDATER, std::memory_order_release);
/* the update lock must have existed */
DBUG_ASSERT((l & (WRITER | UPDATER)) == UPDATER);
}
#endif /* SUX_LOCK_GENERIC */
/** Release an exclusive lock */
void write_unlock()
{
......@@ -211,11 +115,7 @@ class rw_lock
static_assert(WRITER == 1U << 31, "compatibility");
IF_DBUG_ASSERT(auto l=,) lock.fetch_sub(WRITER, std::memory_order_release);
/* the write lock must have existed */
#ifdef SUX_LOCK_GENERIC
DBUG_ASSERT((l & (WRITER | UPDATER)) == WRITER);
#else /* SUX_LOCK_GENERIC */
DBUG_ASSERT(l & WRITER);
#endif /* SUX_LOCK_GENERIC */
}
/** Try to acquire a shared lock.
@return whether the lock was acquired */
......@@ -231,10 +131,6 @@ class rw_lock
/** @return whether an exclusive lock is being held by any thread */
bool is_write_locked() const { return !!(value() & WRITER); }
#ifdef SUX_LOCK_GENERIC
/** @return whether an update lock is being held by any thread */
bool is_update_locked() const { return !!(value() & UPDATER); }
#endif /* SUX_LOCK_GENERIC */
/** @return whether any lock is being held or waited for by any thread */
bool is_locked_or_waiting() const { return value() != 0; }
/** @return whether any lock is being held by any thread */
......
storage/innobase/include/srw_lock.h
View file @ 4e1ca388
/*****************************************************************************
Copyright (c) 2020, 2021, MariaDB Corporation.
Copyright (c) 2020, 2022, MariaDB Corporation.
This program is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free Software
......@@ -20,27 +20,58 @@ this program; if not, write to the Free Software Foundation, Inc.,
#include "univ.i"
#include "rw_lock.h"
#if defined __linux__
/* futex(2): FUTEX_WAIT_PRIVATE, FUTEX_WAKE_PRIVATE */
#elif defined __OpenBSD__ || defined __FreeBSD__ || defined __DragonFly__
/* system calls similar to Linux futex(2) */
#elif defined _WIN32
/* SRWLOCK as well as WaitOnAddress(), WakeByAddressSingle() */
#else
# define SUX_LOCK_GENERIC /* fall back to generic synchronization primitives */
#endif
#if !defined SUX_LOCK_GENERIC && 0 /* defined SAFE_MUTEX */
# define SUX_LOCK_GENERIC /* Use dummy implementation for debugging purposes */
#endif
#ifdef SUX_LOCK_GENERIC
/** An exclusive-only variant of srw_lock */
template<bool spinloop>
class srw_mutex_impl final
class pthread_mutex_wrapper final
{
pthread_mutex_t lock;
void wr_wait();
public:
void init() { pthread_mutex_init(&lock, nullptr); }
void init()
{
if (spinloop)
pthread_mutex_init(&lock, MY_MUTEX_INIT_FAST);
else
pthread_mutex_init(&lock, nullptr);
}
void destroy() { pthread_mutex_destroy(&lock); }
# ifdef PTHREAD_ADAPTIVE_MUTEX_INITIALIZER_NP
void wr_lock() { pthread_mutex_lock(&lock); }
# else
private:
void wr_wait();
public:
inline void wr_lock();
# endif
void wr_unlock() { pthread_mutex_unlock(&lock); }
bool wr_lock_try() { return !pthread_mutex_trylock(&lock); }
};
template<> void srw_mutex_impl<true>::wr_wait();
# ifndef PTHREAD_ADAPTIVE_MUTEX_INITIALIZER_NP
template<> void pthread_mutex_wrapper<true>::wr_wait();
template<>
inline void srw_mutex_impl<false>::wr_lock() { pthread_mutex_lock(&lock); }
inline void pthread_mutex_wrapper<false>::wr_lock()
{ pthread_mutex_lock(&lock); }
template<>
inline void srw_mutex_impl<true>::wr_lock() { if (!wr_lock_try()) wr_wait(); }
#else
inline void pthread_mutex_wrapper<true>::wr_lock()
{ if (!wr_lock_try()) wr_wait(); }
# endif
#endif
/** Futex-based mutex */
template<bool spinloop>
class srw_mutex_impl final
......@@ -51,6 +82,15 @@ class srw_mutex_impl final
/** Identifies that the lock is being held */
static constexpr uint32_t HOLDER= 1U << 31;
#ifdef SUX_LOCK_GENERIC
public:
/** The mutex for the condition variables. */
pthread_mutex_t mutex;
private:
/** Condition variable for the lock word. Used with mutex. */
pthread_cond_t cond;
#endif
/** Wait until the mutex has been acquired */
void wait_and_lock();
/** Wait for lock!=lk */
......@@ -65,8 +105,22 @@ class srw_mutex_impl final
bool is_locked() const
{ return (lock.load(std::memory_order_acquire) & HOLDER) != 0; }
void init() { DBUG_ASSERT(!is_locked_or_waiting()); }
void destroy() { DBUG_ASSERT(!is_locked_or_waiting()); }
void init()
{
DBUG_ASSERT(!is_locked_or_waiting());
#ifdef SUX_LOCK_GENERIC
pthread_mutex_init(&mutex, nullptr);
pthread_cond_init(&cond, nullptr);
#endif
}
void destroy()
{
DBUG_ASSERT(!is_locked_or_waiting());
#ifdef SUX_LOCK_GENERIC
pthread_mutex_destroy(&mutex);
pthread_cond_destroy(&cond);
#endif
}
/** @return whether the mutex was acquired */
bool wr_lock_try()
......@@ -88,19 +142,20 @@ class srw_mutex_impl final
}
}
};
#endif
#ifdef SUX_LOCK_GENERIC
typedef pthread_mutex_wrapper<true> srw_spin_mutex;
typedef pthread_mutex_wrapper<false> srw_mutex;
#else
typedef srw_mutex_impl<true> srw_spin_mutex;
typedef srw_mutex_impl<false> srw_mutex;
#endif
template<bool spinloop> class srw_lock_impl;
/** Slim shared-update-exclusive lock with no recursion */
template<bool spinloop>
class ssux_lock_impl final
#ifdef SUX_LOCK_GENERIC
: private rw_lock
#endif
{
#ifdef UNIV_PFS_RWLOCK
friend class ssux_lock;
......@@ -110,50 +165,12 @@ class ssux_lock_impl final
friend srw_lock_impl<spinloop>;
# endif
#endif
#ifdef SUX_LOCK_GENERIC
pthread_mutex_t mutex;
pthread_cond_t cond_shared;
pthread_cond_t cond_exclusive;
/** Wait for a read lock.
@param l lock word from a failed read_trylock() */
void read_lock(uint32_t l);
/** Wait for an update lock.
@param l lock word from a failed update_trylock() */
void update_lock(uint32_t l);
/** Wait for a write lock after a failed write_trylock() or upgrade_trylock()
@param holding_u whether we already hold u_lock() */
void write_lock(bool holding_u);
/** Wait for signal
@param l lock word from a failed acquisition */
inline void writer_wait(uint32_t l);
/** Wait for signal
@param l lock word from a failed acquisition */
inline void readers_wait(uint32_t l);
/** Wake waiters */
inline void wake();
public:
void init();
void destroy();
/** @return whether any writer is waiting */
bool is_waiting() const { return (value() & WRITER_WAITING) != 0; }
bool is_write_locked() const { return rw_lock::is_write_locked(); }
bool is_locked_or_waiting() const { return rw_lock::is_locked_or_waiting(); }
bool rd_lock_try() { uint32_t l; return read_trylock(l); }
bool wr_lock_try() { return write_trylock(); }
void rd_lock() { uint32_t l; if (!read_trylock(l)) read_lock(l); }
void u_lock() { uint32_t l; if (!update_trylock(l)) update_lock(l); }
bool u_lock_try() { uint32_t l; return update_trylock(l); }
void u_wr_upgrade() { if (!upgrade_trylock()) write_lock(true); }
void wr_u_downgrade() { downgrade(); }
void wr_lock() { if (!write_trylock()) write_lock(false); }
void rd_unlock();
void u_unlock();
void wr_unlock();
#else
/** mutex for synchronization; held by U or X lock holders */
srw_mutex_impl<spinloop> writer;
#ifdef SUX_LOCK_GENERIC
/** Condition variable for "readers"; used with writer.mutex. */
pthread_cond_t readers_cond;
#endif
/** S or U holders, and WRITER flag for X holder or waiter */
std::atomic<uint32_t> readers;
/** indicates an X request; readers=WRITER indicates granted X lock */
......@@ -169,15 +186,29 @@ class ssux_lock_impl final
/** Acquire a read lock */
void rd_wait();
public:
void init() { DBUG_ASSERT(is_vacant()); }
void destroy() { DBUG_ASSERT(is_vacant()); }
void init()
{
writer.init();
DBUG_ASSERT(is_vacant());
#ifdef SUX_LOCK_GENERIC
pthread_cond_init(&readers_cond, nullptr);
#endif
}
void destroy()
{
DBUG_ASSERT(is_vacant());
writer.destroy();
#ifdef SUX_LOCK_GENERIC
pthread_cond_destroy(&readers_cond);
#endif
}
/** @return whether any writer is waiting */
bool is_waiting() const
{ return (readers.load(std::memory_order_relaxed) & WRITER) != 0; }
# ifndef DBUG_OFF
#ifndef DBUG_OFF
/** @return whether the lock is being held or waited for */
bool is_vacant() const { return !is_locked_or_waiting(); }
# endif /* !DBUG_OFF */
#endif /* !DBUG_OFF */
bool rd_lock_try()
{
......@@ -288,7 +319,6 @@ class ssux_lock_impl final
void unlock_shared() { rd_unlock(); }
void lock() { wr_lock(); }
void unlock() { wr_unlock(); }
#endif
};
#if defined _WIN32 || defined SUX_LOCK_GENERIC
......
storage/innobase/sync/srw_lock.cc
View file @ 4e1ca388
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7