Commit 1029b22f authored by Sergey Vojtovich's avatar Sergey Vojtovich

MDEV-13728 - Import MySQL 5.7 atomic operations for MSVC and Solaris

gcc_sync.h, solaris.h, generic-msvc.h copied verbatim.
parent 62fb0221
#ifndef ATOMIC_GCC_BUILTINS_INCLUDED #ifndef ATOMIC_GCC_BUILTINS_INCLUDED
#define ATOMIC_GCC_BUILTINS_INCLUDED #define ATOMIC_GCC_BUILTINS_INCLUDED
/* Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. /* Copyright (c) 2017 MariaDB Foundation
This program is free software; you can redistribute it and/or modify This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by it under the terms of the GNU General Public License as published by
...@@ -16,8 +16,6 @@ ...@@ -16,8 +16,6 @@
along with this program; if not, write to the Free Software along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */
#if defined(HAVE_GCC_C11_ATOMICS)
#define MY_ATOMIC_MODE "gcc-atomics-smp"
#define MY_MEMORY_ORDER_RELAXED __ATOMIC_RELAXED #define MY_MEMORY_ORDER_RELAXED __ATOMIC_RELAXED
#define MY_MEMORY_ORDER_CONSUME __ATOMIC_CONSUME #define MY_MEMORY_ORDER_CONSUME __ATOMIC_CONSUME
...@@ -76,21 +74,5 @@ ...@@ -76,21 +74,5 @@
__atomic_compare_exchange_n((P), (E), (D), 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST) __atomic_compare_exchange_n((P), (E), (D), 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)
#define my_atomic_casptr(P, E, D) \ #define my_atomic_casptr(P, E, D) \
__atomic_compare_exchange_n((P), (E), (D), 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST) __atomic_compare_exchange_n((P), (E), (D), 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)
#else
#define MY_ATOMIC_MODE "gcc-builtins-smp"
#define make_atomic_load_body(S) \
ret= __sync_fetch_and_or(a, 0);
#define make_atomic_store_body(S) \
(void) __sync_lock_test_and_set(a, v);
#define make_atomic_add_body(S) \
v= __sync_fetch_and_add(a, v);
#define make_atomic_fas_body(S) \
v= __sync_lock_test_and_set(a, v);
#define make_atomic_cas_body(S) \
int ## S sav; \
int ## S cmp_val= *cmp; \
sav= __sync_val_compare_and_swap(a, cmp_val, set);\
if (!(ret= (sav == cmp_val))) *cmp= sav
#endif
#endif /* ATOMIC_GCC_BUILTINS_INCLUDED */ #endif /* ATOMIC_GCC_BUILTINS_INCLUDED */
#ifndef GCC_SYNC_INCLUDED
#define GCC_SYNC_INCLUDED
/* Copyright (c) 2008, 2016, Oracle and/or its affiliates. All rights reserved.
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; version 2 of the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */
/* Old GCC __sync builtins introduced in GCC 4.1 */
static inline int my_atomic_cas32(int32 volatile *a, int32 *cmp, int32 set)
{
int32 cmp_val= *cmp;
int32 sav= __sync_val_compare_and_swap(a, cmp_val, set);
int ret= (sav == cmp_val);
if (!ret)
*cmp = sav;
return ret;
}
static inline int my_atomic_cas64(int64 volatile *a, int64 *cmp, int64 set)
{
int64 cmp_val= *cmp;
int64 sav= __sync_val_compare_and_swap(a, cmp_val, set);
int ret= (sav == cmp_val);
if (!ret)
*cmp = sav;
return ret;
}
static inline int my_atomic_casptr(void * volatile *a, void **cmp, void *set)
{
void *cmp_val= *cmp;
void *sav= __sync_val_compare_and_swap(a, cmp_val, set);
int ret= (sav == cmp_val);
if (!ret)
*cmp = sav;
return ret;
}
static inline int32 my_atomic_add32(int32 volatile *a, int32 v)
{
return __sync_fetch_and_add(a, v);
}
static inline int64 my_atomic_add64(int64 volatile *a, int64 v)
{
return __sync_fetch_and_add(a, v);
}
static inline int32 my_atomic_fas32(int32 volatile *a, int32 v)
{
return __sync_lock_test_and_set(a, v);
}
static inline int64 my_atomic_fas64(int64 volatile *a, int64 v)
{
return __sync_lock_test_and_set(a, v);
}
static inline void * my_atomic_fasptr(void * volatile *a, void * v)
{
return __sync_lock_test_and_set(a, v);
}
static inline int32 my_atomic_load32(int32 volatile *a)
{
return __sync_fetch_and_or(a, 0);
}
static inline int64 my_atomic_load64(int64 volatile *a)
{
return __sync_fetch_and_or(a, 0);
}
static inline void* my_atomic_loadptr(void * volatile *a)
{
return __sync_fetch_and_or(a, 0);
}
static inline void my_atomic_store32(int32 volatile *a, int32 v)
{
(void) __sync_lock_test_and_set(a, v);
}
static inline void my_atomic_store64(int64 volatile *a, int64 v)
{
(void) __sync_lock_test_and_set(a, v);
}
static inline void my_atomic_storeptr(void * volatile *a, void *v)
{
(void) __sync_lock_test_and_set(a, v);
}
#endif /* GCC_SYNC_INCLUDED */
/* Copyright (c) 2006-2008 MySQL AB, 2009 Sun Microsystems, Inc. #ifndef ATOMIC_MSC_INCLUDED
Use is subject to license terms. #define ATOMIC_MSC_INCLUDED
/* Copyright (c) 2006, 2014, Oracle and/or its affiliates. All rights reserved.
This program is free software; you can redistribute it and/or modify This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by it under the terms of the GNU General Public License as published by
...@@ -14,75 +16,97 @@ ...@@ -14,75 +16,97 @@
along with this program; if not, write to the Free Software along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */
#ifndef _atomic_h_cleanup_
#define _atomic_h_cleanup_ "atomic/generic-msvc.h"
#include <windows.h> #include <windows.h>
/*
x86 compilers (both VS2003 or VS2005) never use instrinsics, but generate static inline int my_atomic_cas32(int32 volatile *a, int32 *cmp, int32 set)
function calls to kernel32 instead, even in the optimized build. {
We force intrinsics as described in MSDN documentation for int32 initial_cmp= *cmp;
_InterlockedCompareExchange. int32 initial_a= InterlockedCompareExchange((volatile LONG*)a,
*/ set, initial_cmp);
#ifdef _M_IX86 int ret= (initial_a == initial_cmp);
if (!ret)
#if (_MSC_VER >= 1500) *cmp= initial_a;
#include <intrin.h> return ret;
#else }
C_MODE_START
/*Visual Studio 2003 and earlier do not have prototypes for atomic intrinsics*/ static inline int my_atomic_cas64(int64 volatile *a, int64 *cmp, int64 set)
LONG _InterlockedCompareExchange (LONG volatile *Target, LONG Value, LONG Comp); {
LONGLONG _InterlockedCompareExchange64 (LONGLONG volatile *Target, int64 initial_cmp= *cmp;
LONGLONG Value, LONGLONG Comp); int64 initial_a= InterlockedCompareExchange64((volatile LONGLONG*)a,
C_MODE_END (LONGLONG)set,
(LONGLONG)initial_cmp);
#pragma intrinsic(_InterlockedCompareExchange) int ret= (initial_a == initial_cmp);
#pragma intrinsic(_InterlockedCompareExchange64) if (!ret)
#endif *cmp= initial_a;
return ret;
#define InterlockedCompareExchange _InterlockedCompareExchange }
#define InterlockedCompareExchange64 _InterlockedCompareExchange64
/* static inline int my_atomic_casptr(void * volatile *a, void **cmp, void *set)
No need to do something special for InterlockedCompareExchangePointer {
as it is a #define to InterlockedCompareExchange. The same applies to void *initial_cmp= *cmp;
InterlockedExchangePointer. void *initial_a= InterlockedCompareExchangePointer(a, set, initial_cmp);
*/ int ret= (initial_a == initial_cmp);
#endif /*_M_IX86*/ if (!ret)
*cmp= initial_a;
#define MY_ATOMIC_MODE "msvc-intrinsics" return ret;
/* Implement using CAS on WIN32 */ }
#define IL_COMP_EXCHG32(X,Y,Z) \
InterlockedCompareExchange((volatile LONG *)(X),(Y),(Z)) static inline int32 my_atomic_add32(int32 volatile *a, int32 v)
#define IL_COMP_EXCHG64(X,Y,Z) \ {
InterlockedCompareExchange64((volatile LONGLONG *)(X), \ return (int32)InterlockedExchangeAdd((volatile LONG*)a, v);
(LONGLONG)(Y),(LONGLONG)(Z)) }
#define IL_COMP_EXCHGptr InterlockedCompareExchangePointer
static inline int64 my_atomic_add64(int64 volatile *a, int64 v)
#define make_atomic_cas_body(S) \ {
int ## S initial_cmp= *cmp; \ return (int64)InterlockedExchangeAdd64((volatile LONGLONG*)a, (LONGLONG)v);
int ## S initial_a= IL_COMP_EXCHG ## S (a, set, initial_cmp); \ }
if (!(ret= (initial_a == initial_cmp))) *cmp= initial_a;
static inline int32 my_atomic_load32(int32 volatile *a)
#ifndef _M_IX86 {
/* Use full set of optimised functions on WIN64 */ return (int32)InterlockedCompareExchange((volatile LONG *)a, 0, 0);
#define IL_EXCHG_ADD32(X,Y) \ }
InterlockedExchangeAdd((volatile LONG *)(X),(Y))
#define IL_EXCHG_ADD64(X,Y) \ static inline int64 my_atomic_load64(int64 volatile *a)
InterlockedExchangeAdd64((volatile LONGLONG *)(X),(LONGLONG)(Y)) {
#define IL_EXCHG32(X,Y) \ return (int64)InterlockedCompareExchange64((volatile LONGLONG *)a, 0, 0);
InterlockedExchange((volatile LONG *)(X),(Y)) }
#define IL_EXCHG64(X,Y) \
InterlockedExchange64((volatile LONGLONG *)(X),(LONGLONG)(Y)) static inline void* my_atomic_loadptr(void * volatile *a)
#define IL_EXCHGptr InterlockedExchangePointer {
return InterlockedCompareExchangePointer(a, 0, 0);
#define make_atomic_add_body(S) \ }
v= IL_EXCHG_ADD ## S (a, v)
#define make_atomic_swap_body(S) \ static inline int32 my_atomic_fas32(int32 volatile *a, int32 v)
v= IL_EXCHG ## S (a, v) {
#define make_atomic_load_body(S) \ return (int32)InterlockedExchange((volatile LONG*)a, v);
ret= 0; /* avoid compiler warning */ \ }
ret= IL_COMP_EXCHG ## S (a, ret, ret);
#endif static inline int64 my_atomic_fas64(int64 volatile *a, int64 v)
{
return (int64)InterlockedExchange64((volatile LONGLONG*)a, v);
}
static inline void * my_atomic_fasptr(void * volatile *a, void * v)
{
return InterlockedExchangePointer(a, v);
}
static inline void my_atomic_store32(int32 volatile *a, int32 v)
{
(void)InterlockedExchange((volatile LONG*)a, v);
}
static inline void my_atomic_store64(int64 volatile *a, int64 v)
{
(void)InterlockedExchange64((volatile LONGLONG*)a, v);
}
static inline void my_atomic_storeptr(void * volatile *a, void *v)
{
(void)InterlockedExchangePointer(a, v);
}
/* /*
my_yield_processor (equivalent of x86 PAUSE instruction) should be used my_yield_processor (equivalent of x86 PAUSE instruction) should be used
to improve performance on hyperthreaded CPUs. Intel recommends to use it in to improve performance on hyperthreaded CPUs. Intel recommends to use it in
...@@ -94,35 +118,18 @@ C_MODE_END ...@@ -94,35 +118,18 @@ C_MODE_END
YieldProcessor in a loop - that is, yielding longer. On Intel boxes setting YieldProcessor in a loop - that is, yielding longer. On Intel boxes setting
loop count in the range 200-300 brought best results. loop count in the range 200-300 brought best results.
*/ */
#ifndef YIELD_LOOPS
#define YIELD_LOOPS 200 #define YIELD_LOOPS 200
#endif
static __inline int my_yield_processor() static inline int my_yield_processor()
{ {
int i; int i;
for(i=0; i<YIELD_LOOPS; i++) for (i=0; i<YIELD_LOOPS; i++)
{ {
#if (_MSC_VER <= 1310)
/* On older compilers YieldProcessor is not available, use inline assembly*/
__asm { rep nop }
#else
YieldProcessor(); YieldProcessor();
#endif
} }
return 1; return 1;
} }
#define LF_BACKOFF my_yield_processor() #define LF_BACKOFF my_yield_processor()
#else /* cleanup */
#endif /* ATOMIC_MSC_INCLUDED */
#undef IL_EXCHG_ADD32
#undef IL_EXCHG_ADD64
#undef IL_COMP_EXCHG32
#undef IL_COMP_EXCHG64
#undef IL_COMP_EXCHGptr
#undef IL_EXCHG32
#undef IL_EXCHG64
#undef IL_EXCHGptr
#endif
/* Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. #ifndef ATOMIC_SOLARIS_INCLUDED
#define ATOMIC_SOLARIS_INCLUDED
/* Copyright (c) 2008, 2014, Oracle and/or its affiliates. All rights reserved.
This program is free software; you can redistribute it and/or modify This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by it under the terms of the GNU General Public License as published by
...@@ -13,51 +16,102 @@ ...@@ -13,51 +16,102 @@
along with this program; if not, write to the Free Software along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */
#ifndef _atomic_h_cleanup_
#define _atomic_h_cleanup_ "atomic/solaris.h"
#include <atomic.h> #include <atomic.h>
#define MY_ATOMIC_MODE "solaris-atomic"
#if defined(__GNUC__) #if defined(__GNUC__)
#define atomic_typeof(T,V) __typeof__(V) #define atomic_typeof(T,V) __typeof__(V)
#else #else
#define atomic_typeof(T,V) T #define atomic_typeof(T,V) T
#endif #endif
#define uintptr_t void * static inline int my_atomic_cas32(int32 volatile *a, int32 *cmp, int32 set)
#define atomic_or_ptr_nv(X,Y) (void *)atomic_or_ulong_nv((volatile ulong_t *)X, Y) {
int ret;
#define make_atomic_cas_body(S) \ atomic_typeof(uint32_t, *cmp) sav;
atomic_typeof(uint ## S ## _t, *cmp) sav; \ sav= atomic_cas_32((volatile uint32_t *)a, (uint32_t)*cmp, (uint32_t)set);
sav = atomic_cas_ ## S( \ ret= (sav == *cmp);
(volatile uint ## S ## _t *)a, \ if (!ret)
(uint ## S ## _t)*cmp, \
(uint ## S ## _t)set); \
if (! (ret= (sav == *cmp))) \
*cmp= sav; *cmp= sav;
return ret;
#define make_atomic_add_body(S) \ }
int ## S nv; /* new value */ \
nv= atomic_add_ ## S ## _nv((volatile uint ## S ## _t *)a, v); \ static inline int my_atomic_cas64(int64 volatile *a, int64 *cmp, int64 set)
v= nv - v {
int ret;
/* ------------------------------------------------------------------------ */ atomic_typeof(uint64_t, *cmp) sav;
sav= atomic_cas_64((volatile uint64_t *)a, (uint64_t)*cmp, (uint64_t)set);
#define make_atomic_load_body(S) \ ret= (sav == *cmp);
ret= atomic_or_ ## S ## _nv((volatile uint ## S ## _t *)a, 0) if (!ret)
*cmp= sav;
#define make_atomic_store_body(S) \ return ret;
(void) atomic_swap_ ## S((volatile uint ## S ## _t *)a, (uint ## S ## _t)v) }
#define make_atomic_fas_body(S) \ static inline int my_atomic_casptr(void * volatile *a, void **cmp, void *set)
v= atomic_swap_ ## S((volatile uint ## S ## _t *)a, (uint ## S ## _t)v) {
int ret;
#else /* cleanup */ atomic_typeof(void *, *cmp) sav;
sav= atomic_cas_ptr((volatile void **)a, (void *)*cmp, (void *)set);
#undef uintptr_t ret= (sav == *cmp);
#undef atomic_or_ptr_nv if (!ret)
*cmp= sav;
#endif return ret;
}
static inline int32 my_atomic_add32(int32 volatile *a, int32 v)
{
int32 nv= atomic_add_32_nv((volatile uint32_t *)a, v);
return nv - v;
}
static inline int64 my_atomic_add64(int64 volatile *a, int64 v)
{
int64 nv= atomic_add_64_nv((volatile uint64_t *)a, v);
return nv - v;
}
static inline int32 my_atomic_fas32(int32 volatile *a, int32 v)
{
return atomic_swap_32((volatile uint32_t *)a, (uint32_t)v);
}
static inline int64 my_atomic_fas64(int64 volatile *a, int64 v)
{
return atomic_swap_64((volatile uint64_t *)a, (uint64_t)v);
}
static inline void * my_atomic_fasptr(void * volatile *a, void * v)
{
return atomic_swap_ptr(a, v);
}
static inline int32 my_atomic_load32(int32 volatile *a)
{
return atomic_or_32_nv((volatile uint32_t *)a, 0);
}
static inline int64 my_atomic_load64(int64 volatile *a)
{
return atomic_or_64_nv((volatile uint64_t *)a, 0);
}
static inline void* my_atomic_loadptr(void * volatile *a)
{
return atomic_add_ptr_nv(a, 0);
}
static inline void my_atomic_store32(int32 volatile *a, int32 v)
{
(void) atomic_swap_32((volatile uint32_t *)a, (uint32_t)v);
}
static inline void my_atomic_store64(int64 volatile *a, int64 v)
{
(void) atomic_swap_64((volatile uint64_t *)a, (uint64_t)v);
}
static inline void my_atomic_storeptr(void * volatile *a, void *v)
{
(void) atomic_swap_ptr((volatile void **)a, (void *)v);
}
#endif /* ATOMIC_SOLARIS_INCLUDED */
...@@ -110,119 +110,13 @@ ...@@ -110,119 +110,13 @@
#include "atomic/generic-msvc.h" #include "atomic/generic-msvc.h"
#elif defined(HAVE_SOLARIS_ATOMIC) #elif defined(HAVE_SOLARIS_ATOMIC)
#include "atomic/solaris.h" #include "atomic/solaris.h"
#elif defined(HAVE_GCC_ATOMIC_BUILTINS) || defined(HAVE_GCC_C11_ATOMICS) #elif defined(HAVE_GCC_C11_ATOMICS)
#include "atomic/gcc_builtins.h" #include "atomic/gcc_builtins.h"
#elif defined(HAVE_GCC_ATOMIC_BUILTINS)
#include "atomic/gcc_sync.h"
#endif #endif
#ifndef HAVE_GCC_C11_ATOMICS
#ifndef make_atomic_cas_body
/* nolock.h was not able to generate even a CAS function, fall back */
#error atomic ops for this platform are not implemented
#endif
#define intptr void *
/* define missing functions by using the already generated ones */
#ifndef make_atomic_add_body
#define make_atomic_add_body(S) \
int ## S tmp=*a; \
while (!my_atomic_cas ## S(a, &tmp, tmp+v)) ; \
v=tmp;
#endif
#ifndef make_atomic_fas_body
#define make_atomic_fas_body(S) \
int ## S tmp=*a; \
while (!my_atomic_cas ## S(a, &tmp, v)) ; \
v=tmp;
#endif
#ifndef make_atomic_load_body
#define make_atomic_load_body(S) \
ret= 0; /* avoid compiler warning */ \
(void)(my_atomic_cas ## S(a, &ret, ret));
#endif
#ifndef make_atomic_store_body
#define make_atomic_store_body(S) \
(void)(my_atomic_fas ## S (a, v));
#endif
#define make_atomic_cas(S) \
static inline int my_atomic_cas ## S(int ## S volatile *a, \
int ## S *cmp, int ## S set) \
{ \
int8 ret; \
make_atomic_cas_body(S); \
return ret; \
}
#define make_atomic_add(S) \
static inline int ## S my_atomic_add ## S( \
int ## S volatile *a, int ## S v) \
{ \
make_atomic_add_body(S); \
return v; \
}
#define make_atomic_fas(S) \
static inline int ## S my_atomic_fas ## S( \
int ## S volatile *a, int ## S v) \
{ \
make_atomic_fas_body(S); \
return v; \
}
#define make_atomic_load(S) \
static inline int ## S my_atomic_load ## S(int ## S volatile *a)\
{ \
int ## S ret; \
make_atomic_load_body(S); \
return ret; \
}
#define make_atomic_store(S) \
static inline void my_atomic_store ## S( \
int ## S volatile *a, int ## S v) \
{ \
make_atomic_store_body(S); \
}
make_atomic_cas(32)
make_atomic_cas(64)
make_atomic_cas(ptr)
make_atomic_add(32)
make_atomic_add(64)
make_atomic_load(32)
make_atomic_load(64)
make_atomic_load(ptr)
make_atomic_fas(32)
make_atomic_fas(64)
make_atomic_fas(ptr)
make_atomic_store(32)
make_atomic_store(64)
make_atomic_store(ptr)
#ifdef _atomic_h_cleanup_
#include _atomic_h_cleanup_
#undef _atomic_h_cleanup_
#endif
#undef make_atomic_add
#undef make_atomic_cas
#undef make_atomic_load
#undef make_atomic_store
#undef make_atomic_fas
#undef make_atomic_add_body
#undef make_atomic_cas_body
#undef make_atomic_load_body
#undef make_atomic_store_body
#undef make_atomic_fas_body
#undef intptr
#endif
/* /*
the macro below defines (as an expression) the code that the macro below defines (as an expression) the code that
will be run in spin-loops. Intel manuals recummend to have PAUSE there. will be run in spin-loops. Intel manuals recummend to have PAUSE there.
......
...@@ -67,7 +67,7 @@ int main(int argc __attribute__((unused)), char **argv) ...@@ -67,7 +67,7 @@ int main(int argc __attribute__((unused)), char **argv)
#define CYCLES 3000 #define CYCLES 3000
#define THREADS 30 #define THREADS 30
diag("N CPUs: %d, atomic ops: %s", my_getncpus(), MY_ATOMIC_MODE); diag("N CPUs: %d", my_getncpus());
do_tests(); do_tests();
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment