MDEV-24789: Reduce lock_sys mutex contention further

lock_sys_t::deadlock_check(): Assume that only lock_sys.wait_mutex
is being held by the caller.

lock_sys_t::rd_lock_try(): New function.

lock_sys_t::cancel(trx_t*): Kill an active transaction that may be
holding a lock.

lock_sys_t::cancel(trx_t*, lock_t*): Cancel a waiting lock request.

lock_trx_handle_wait(): Avoid acquiring mutexes in some cases,
and in never acquire lock_sys.latch in exclusive mode.
This function is only invoked in a semi-consistent read
(locking a clustered index record only if it matches the search condition).
Normally, lock_wait() will take care of lock waits.

lock_wait(): Invoke the new function lock_sys_t::cancel() at the end,
to avoid acquiring exclusive lock_sys.latch.

lock_rec_other_trx_holds_expl(): Use LockGuard instead of LockMutexGuard.

lock_release_autoinc_locks(): Explicitly acquire table->lock_mutex,
in case only a shared lock_sys.latch is being held. Deadlock::report()
will still hold exclusive lock_sys.latch while invoking
lock_cancel_waiting_and_release().

lock_cancel_waiting_and_release(): Acquire trx->mutex in this function,
instead of expecting the caller to do so.

lock_unlock_table_autoinc(): Only acquire shared lock_sys.latch.

lock_table_has_locks(): Do not acquire lock_sys.latch at all.

Deadlock::check_and_resolve(): Only acquire shared lock_sys.latchm
for invoking lock_sys_t::cancel(trx, wait_lock).

innobase_query_caching_table_check_low(),
row_drop_tables_for_mysql_in_background(): Do not acquire lock_sys.latch.

storage/innobase/handler/ha_innodb.cc

@@ -2688,8 +2688,7 @@ the query cache.
 @param[in]	trx	transaction object
 @return whether the storing or retrieving from the query cache is permitted */
 static bool innobase_query_caching_table_check_low(
-	const dict_table_t*	table,
-	trx_t*			trx)
+	dict_table_t* table, trx_t* trx)
 {
 	/* The following conditions will decide the query cache
 	retrieval or storing into:
@@ -2714,8 +2713,10 @@ static bool innobase_query_caching_table_check_low(
 		return false;
 	}
 
-	LockMutexGuard g{SRW_LOCK_CALL};
-	return UT_LIST_GET_LEN(table->locks) == 0;
+	table->lock_mutex_lock();
+	auto len= UT_LIST_GET_LEN(table->locks);
+	table->lock_mutex_unlock();
+	return len == 0;
 }
 
 /** Checks if MySQL at the moment is allowed for this table to retrieve a
@@ -4483,21 +4484,7 @@ static void innobase_kill_query(handlerton*, THD *thd, enum thd_kill_levels)
       DBUG_VOID_RETURN;
 #endif /* WITH_WSREP */
     if (trx->lock.wait_lock)
-    {
-      {
-        LockMutexGuard g{SRW_LOCK_CALL};
-        mysql_mutex_lock(&lock_sys.wait_mutex);
-        if (lock_t *lock= trx->lock.wait_lock)
-        {
-          trx->mutex_lock();
-          trx->error_state= DB_INTERRUPTED;
-          lock_cancel_waiting_and_release(lock);
-          trx->mutex_unlock();
-        }
-        lock_sys.deadlock_check(true);
-      }
-      mysql_mutex_unlock(&lock_sys.wait_mutex);
-    }
+      lock_sys_t::cancel(trx);
   }
 
   DBUG_VOID_RETURN;

storage/innobase/include/lock0lock.h

@@ -504,15 +504,9 @@ lock_rec_get_index(
 /*===============*/
 	const lock_t*	lock);	/*!< in: lock */
 
-/*******************************************************************//**
-Check if there are any locks (table or rec) against table.
-@return TRUE if locks exist */
-bool
-lock_table_has_locks(
-/*=================*/
-	const dict_table_t*	table);	/*!< in: check if there are any locks
-					held on records in this table or on the
-					table itself */
+/** Check if there are any locks on a table.
+@return true if table has either table or record locks. */
+bool lock_table_has_locks(dict_table_t *table);
 
 /** Wait for a lock to be released.
 @retval DB_DEADLOCK if this transaction was chosen as the deadlock victim
@@ -528,15 +522,15 @@ void
 lock_unlock_table_autoinc(
 /*======================*/
 	trx_t*	trx);			/*!< in/out: transaction */
-/*********************************************************************//**
-Check whether the transaction has already been rolled back because it
-was selected as a deadlock victim, or if it has to wait then cancel
-the wait lock.
-@return DB_DEADLOCK, DB_LOCK_WAIT or DB_SUCCESS */
-dberr_t
-lock_trx_handle_wait(
-/*=================*/
-	trx_t*	trx);	/*!< in/out: trx lock state */
+
+/** Handle a pending lock wait (DB_LOCK_WAIT) in a semi-consistent read
+while holding a clustered index leaf page latch.
+@param trx           transaction that is or was waiting for a lock
+@retval DB_SUCCESS   if the lock was granted
+@retval DB_DEADLOCK  if the transaction must be aborted due to a deadlock
+@retval DB_LOCK_WAIT if a lock wait would be necessary; the pending
+                     lock request was released */
+dberr_t lock_trx_handle_wait(trx_t *trx);
 
 /*********************************************************************//**
 Checks that a transaction id is sensible, i.e., not in the future.
@@ -775,6 +769,16 @@ class lock_sys_t
                            std::memory_order_relaxed));
     return true;
   }
+  /** Try to acquire shared lock_sys.latch
+  @return whether the latch was acquired */
+  bool rd_lock_try()
+  {
+    ut_ad(!is_writer());
+    if (!latch.rd_lock_try()) return false;
+    ut_ad(!writer.load(std::memory_order_relaxed));
+    ut_d(readers.fetch_add(1, std::memory_order_relaxed));
+    return true;
+  }
 
   /** Assert that wr_lock() has been invoked by this thread */
   void assert_locked() const { ut_ad(is_writer()); }
@@ -816,10 +820,15 @@ class lock_sys_t
   void close();
 
 
-  /** Check for deadlocks
-  @param locked lock_sys.is_writer() */
-  static void deadlock_check(bool locked);
+  /** Check for deadlocks while holding only lock_sys.wait_mutex. */
+  void deadlock_check();
 
+  /** Cancel a waiting lock request.
+  @param lock   waiting lock request
+  @param trx    active transaction */
+  static void cancel(trx_t *trx, lock_t *lock);
+  /** Cancel a waiting lock request (if any) when killing a transaction */
+  static void cancel(trx_t *trx);
 
   /** Note that a record lock wait started */
   inline void wait_start();
@@ -1034,9 +1043,6 @@ lock_rtr_move_rec_list(
 						moved */
 	ulint			num_move);	/*!< in: num of rec to move */
 
-/** Cancel a waiting lock request and release possibly waiting transactions */
-void lock_cancel_waiting_and_release(lock_t *lock);
-
 #include "lock0lock.ic"
 
 #endif

storage/innobase/row/row0mysql.cc

@@ -2613,10 +2613,9 @@ row_drop_tables_for_mysql_in_background(void)
 	}
 
 	if (!srv_fast_shutdown && !trx_sys.any_active_transactions()) {
-		{
-			LockMutexGuard g{SRW_LOCK_CALL};
+		table->lock_mutex_lock();
 		skip = UT_LIST_GET_LEN(table->locks) != 0;
-		}
+		table->lock_mutex_unlock();
 		if (skip) {
 			/* We cannot drop tables that are locked by XA
 			PREPARE transactions. */