stan/jdk-24
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

8155949: Support relaxed semantics in cmpxchg

Browse Source 
Co-authored-by: Hiroshi H Horii <horii@jp.ibm.com>
Reviewed-by: dholmes, kbarrett, goetz, aph
This commit is contained in:
Martin Doerr 2016-06-21 19:25:41 -04:00
parent 708c94be94
commit afc9349c93
14 changed files with 296 additions and 259 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

68
hotspot/src/os_cpu/aix_ppc/vm/atomic_aix_ppc.inline.hpp
View File

@ -291,12 +291,30 @@ inline void* Atomic::xchg_ptr(void* exchange_value, volatile void* dest) {
return (void*)xchg_ptr((intptr_t)exchange_value, (volatile intptr_t*)dest);
}
inline void cmpxchg_pre_membar(cmpxchg_memory_order order) {
if (order != memory_order_relaxed) {
__asm__ __volatile__ (
/* fence */
strasm_sync
);
}
}
inline void cmpxchg_post_membar(cmpxchg_memory_order order) {
if (order != memory_order_relaxed) {
__asm__ __volatile__ (
/* fence */
strasm_sync
);
}
}
#define VM_HAS_SPECIALIZED_CMPXCHG_BYTE
inline jbyte Atomic::cmpxchg(jbyte exchange_value, volatile jbyte* dest, jbyte compare_value) {
inline jbyte Atomic::cmpxchg(jbyte exchange_value, volatile jbyte* dest, jbyte compare_value, cmpxchg_memory_order order) {
// Note that cmpxchg guarantees a two-way memory barrier across
// the cmpxchg, so it's really a a 'fence_cmpxchg_acquire'
// (see atomic.hpp).
// the cmpxchg, so it's really a a 'fence_cmpxchg_fence' if not
// specified otherwise (see atomic.hpp).
// Using 32 bit internally.
volatile int *dest_base = (volatile int*)((uintptr_t)dest & ~3);
@ -312,9 +330,9 @@ inline jbyte Atomic::cmpxchg(jbyte exchange_value, volatile jbyte* dest, jbyte c
unsigned int old_value, value32;
cmpxchg_pre_membar(order);
__asm__ __volatile__ (
/* fence */
strasm_sync
/* simple guard */
" lbz %[old_value], 0(%[dest]) \n"
" cmpw %[masked_compare_val], %[old_value] \n"
@ -331,8 +349,6 @@ inline jbyte Atomic::cmpxchg(jbyte exchange_value, volatile jbyte* dest, jbyte c
" xor %[value32], %[xor_value], %[value32] \n"
" stwcx. %[value32], 0, %[dest_base] \n"
" bne- 1b \n"
/* acquire */
strasm_sync
/* exit */
"2: \n"
/* out */
@ -353,21 +369,23 @@ inline jbyte Atomic::cmpxchg(jbyte exchange_value, volatile jbyte* dest, jbyte c
"memory"
);
cmpxchg_post_membar(order);
return (jbyte)(unsigned char)old_value;
}
inline jint Atomic::cmpxchg(jint exchange_value, volatile jint* dest, jint compare_value) {
inline jint Atomic::cmpxchg(jint exchange_value, volatile jint* dest, jint compare_value, cmpxchg_memory_order order) {
// Note that cmpxchg guarantees a two-way memory barrier across
// the cmpxchg, so it's really a a 'fence_cmpxchg_acquire'
// (see atomic.hpp).
// the cmpxchg, so it's really a a 'fence_cmpxchg_fence' if not
// specified otherwise (see atomic.hpp).
unsigned int old_value;
const uint64_t zero = 0;
cmpxchg_pre_membar(order);
__asm__ __volatile__ (
/* fence */
strasm_sync
/* simple guard */
" lwz %[old_value], 0(%[dest]) \n"
" cmpw %[compare_value], %[old_value] \n"
@ -379,8 +397,6 @@ inline jint Atomic::cmpxchg(jint exchange_value, volatile jint* dest, jint compa
" bne- 2f \n"
" stwcx. %[exchange_value], %[dest], %[zero] \n"
" bne- 1b \n"
/* acquire */
strasm_sync
/* exit */
"2: \n"
/* out */
@ -397,21 +413,23 @@ inline jint Atomic::cmpxchg(jint exchange_value, volatile jint* dest, jint compa
"memory"
);
cmpxchg_post_membar(order);
return (jint) old_value;
}
inline jlong Atomic::cmpxchg(jlong exchange_value, volatile jlong* dest, jlong compare_value) {
inline jlong Atomic::cmpxchg(jlong exchange_value, volatile jlong* dest, jlong compare_value, cmpxchg_memory_order order) {
// Note that cmpxchg guarantees a two-way memory barrier across
// the cmpxchg, so it's really a a 'fence_cmpxchg_acquire'
// (see atomic.hpp).
// the cmpxchg, so it's really a a 'fence_cmpxchg_fence' if not
// specified otherwise (see atomic.hpp).
long old_value;
const uint64_t zero = 0;
cmpxchg_pre_membar(order);
__asm__ __volatile__ (
/* fence */
strasm_sync
/* simple guard */
" ld %[old_value], 0(%[dest]) \n"
" cmpd %[compare_value], %[old_value] \n"
@ -423,8 +441,6 @@ inline jlong Atomic::cmpxchg(jlong exchange_value, volatile jlong* dest, jlong c
" bne- 2f \n"
" stdcx. %[exchange_value], %[dest], %[zero] \n"
" bne- 1b \n"
/* acquire */
strasm_sync
/* exit */
"2: \n"
/* out */
@ -441,15 +457,17 @@ inline jlong Atomic::cmpxchg(jlong exchange_value, volatile jlong* dest, jlong c
"memory"
);
cmpxchg_post_membar(order);
return (jlong) old_value;
}
inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value) {
return (intptr_t)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value);
inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value, cmpxchg_memory_order order) {
return (intptr_t)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value, order);
}
inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value) {
return (void*)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value);
inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value, cmpxchg_memory_order order) {
return (void*)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value, order);
}
#undef strasm_sync

24
hotspot/src/os_cpu/bsd_x86/vm/atomic_bsd_x86.inline.hpp
View File

@ -89,7 +89,7 @@ inline void* Atomic::xchg_ptr(void* exchange_value, volatile void* des
}
#define VM_HAS_SPECIALIZED_CMPXCHG_BYTE
inline jbyte Atomic::cmpxchg (jbyte exchange_value, volatile jbyte* dest, jbyte compare_value) {
inline jbyte Atomic::cmpxchg (jbyte exchange_value, volatile jbyte* dest, jbyte compare_value, cmpxchg_memory_order order) {
int mp = os::is_MP();
__asm__ volatile (LOCK_IF_MP(%4) "cmpxchgb %1,(%3)"
: "=a" (exchange_value)
@ -98,7 +98,7 @@ inline jbyte Atomic::cmpxchg (jbyte exchange_value, volatile jbyte*
return exchange_value;
}
inline jint Atomic::cmpxchg (jint exchange_value, volatile jint* dest, jint compare_value) {
inline jint Atomic::cmpxchg (jint exchange_value, volatile jint* dest, jint compare_value, cmpxchg_memory_order order) {
int mp = os::is_MP();
__asm__ volatile (LOCK_IF_MP(%4) "cmpxchgl %1,(%3)"
: "=a" (exchange_value)
@ -149,7 +149,7 @@ inline intptr_t Atomic::xchg_ptr(intptr_t exchange_value, volatile intptr_t* des
return exchange_value;
}
inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value) {
inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value, cmpxchg_memory_order order) {
bool mp = os::is_MP();
__asm__ __volatile__ (LOCK_IF_MP(%4) "cmpxchgq %1,(%3)"
: "=a" (exchange_value)
@ -158,12 +158,12 @@ inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong*
return exchange_value;
}
inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value) {
return (intptr_t)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value);
inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value, cmpxchg_memory_order order) {
return (intptr_t)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value, order);
}
inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value) {
return (void*)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value);
inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value, cmpxchg_memory_order order) {
return (void*)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value, order);
}
inline jlong Atomic::load(volatile jlong* src) { return *src; }
@ -197,16 +197,16 @@ extern "C" {
void _Atomic_move_long(volatile jlong* src, volatile jlong* dst);
}
inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value) {
inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value, cmpxchg_memory_order order) {
return _Atomic_cmpxchg_long(exchange_value, dest, compare_value, os::is_MP());
}
inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value) {
return (intptr_t)cmpxchg((jint)exchange_value, (volatile jint*)dest, (jint)compare_value);
inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value, cmpxchg_memory_order order) {
return (intptr_t)cmpxchg((jint)exchange_value, (volatile jint*)dest, (jint)compare_value, order);
}
inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value) {
return (void*)cmpxchg((jint)exchange_value, (volatile jint*)dest, (jint)compare_value);
inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value, cmpxchg_memory_order order) {
return (void*)cmpxchg((jint)exchange_value, (volatile jint*)dest, (jint)compare_value, order);
}
inline jlong Atomic::load(volatile jlong* src) {

15
hotspot/src/os_cpu/bsd_zero/vm/atomic_bsd_zero.inline.hpp
View File

@ -270,7 +270,8 @@ inline void* Atomic::xchg_ptr(void* exchange_value, volatile void* dest) {
inline jint Atomic::cmpxchg(jint exchange_value,
volatile jint* dest,
jint compare_value) {
jint compare_value,
cmpxchg_memory_order order) {
#ifdef ARM
return arm_compare_and_swap(dest, compare_value, exchange_value);
#else
@ -284,14 +285,16 @@ inline jint Atomic::cmpxchg(jint exchange_value,
inline jlong Atomic::cmpxchg(jlong exchange_value,
volatile jlong* dest,
jlong compare_value) {
jlong compare_value,
cmpxchg_memory_order order) {
return __sync_val_compare_and_swap(dest, compare_value, exchange_value);
}
inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value,
volatile intptr_t* dest,
intptr_t compare_value) {
intptr_t compare_value,
cmpxchg_memory_order order) {
#ifdef ARM
return arm_compare_and_swap(dest, compare_value, exchange_value);
#else
@ -305,11 +308,13 @@ inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value,
inline void* Atomic::cmpxchg_ptr(void* exchange_value,
volatile void* dest,
void* compare_value) {
void* compare_value,
cmpxchg_memory_order order) {
return (void *) cmpxchg_ptr((intptr_t) exchange_value,
(volatile intptr_t*) dest,
(intptr_t) compare_value);
(intptr_t) compare_value,
order);
}
inline jlong Atomic::load(volatile jlong* src) {

37
hotspot/src/os_cpu/linux_aarch64/vm/atomic_linux_aarch64.inline.hpp
View File

@ -87,10 +87,28 @@ inline void* Atomic::xchg_ptr(void* exchange_value, volatile void* dest)
(volatile intptr_t*) dest);
}
inline jint Atomic::cmpxchg (jint exchange_value, volatile jint* dest, jint compare_value)
template <typename T> T generic_cmpxchg(T exchange_value, volatile T* dest,
T compare_value, cmpxchg_memory_order order)
{
return __sync_val_compare_and_swap(dest, compare_value, exchange_value);
if (order == memory_order_relaxed) {
T value = compare_value;
__atomic_compare_exchange(dest, &value, &exchange_value, /*weak*/false,
__ATOMIC_RELAXED, __ATOMIC_RELAXED);
return value;
} else {
return __sync_val_compare_and_swap(dest, compare_value, exchange_value);
}
}
#define VM_HAS_SPECIALIZED_CMPXCHG_BYTE
inline jbyte Atomic::cmpxchg (jbyte exchange_value, volatile jbyte* dest, jbyte compare_value, cmpxchg_memory_order order)
{
return generic_cmpxchg(exchange_value, dest, compare_value, order);
}
inline jint Atomic::cmpxchg (jint exchange_value, volatile jint* dest, jint compare_value, cmpxchg_memory_order order)
{
return generic_cmpxchg(exchange_value, dest, compare_value, order);
}
inline void Atomic::store (jlong store_value, jlong* dest) { *dest = store_value; }
@ -123,21 +141,22 @@ inline intptr_t Atomic::xchg_ptr(intptr_t exchange_value, volatile intptr_t* des
return res;
}
inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value)
inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value, cmpxchg_memory_order order)
{
return __sync_val_compare_and_swap(dest, compare_value, exchange_value);
return generic_cmpxchg(exchange_value, dest, compare_value, order);
}
inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value)
inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value, cmpxchg_memory_order order)
{
return __sync_val_compare_and_swap(dest, compare_value, exchange_value);
return generic_cmpxchg(exchange_value, dest, compare_value, order);
}
inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value)
inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value, cmpxchg_memory_order order)
{
return (void *) cmpxchg_ptr((intptr_t) exchange_value,
(volatile intptr_t*) dest,
(intptr_t) compare_value);
(intptr_t) compare_value,
order);
}
inline jlong Atomic::load(volatile jlong* src) { return *src; }

68
hotspot/src/os_cpu/linux_ppc/vm/atomic_linux_ppc.inline.hpp
View File

@ -291,12 +291,30 @@ inline void* Atomic::xchg_ptr(void* exchange_value, volatile void* dest) {
return (void*)xchg_ptr((intptr_t)exchange_value, (volatile intptr_t*)dest);
}
inline void cmpxchg_pre_membar(cmpxchg_memory_order order) {
if (order != memory_order_relaxed) {
__asm__ __volatile__ (
/* fence */
strasm_sync
);
}
}
inline void cmpxchg_post_membar(cmpxchg_memory_order order) {
if (order != memory_order_relaxed) {
__asm__ __volatile__ (
/* fence */
strasm_sync
);
}
}
#define VM_HAS_SPECIALIZED_CMPXCHG_BYTE
inline jbyte Atomic::cmpxchg(jbyte exchange_value, volatile jbyte* dest, jbyte compare_value) {
inline jbyte Atomic::cmpxchg(jbyte exchange_value, volatile jbyte* dest, jbyte compare_value, cmpxchg_memory_order order) {
// Note that cmpxchg guarantees a two-way memory barrier across
// the cmpxchg, so it's really a a 'fence_cmpxchg_acquire'
// (see atomic.hpp).
// the cmpxchg, so it's really a a 'fence_cmpxchg_fence' if not
// specified otherwise (see atomic.hpp).
// Using 32 bit internally.
volatile int *dest_base = (volatile int*)((uintptr_t)dest & ~3);
@ -312,9 +330,9 @@ inline jbyte Atomic::cmpxchg(jbyte exchange_value, volatile jbyte* dest, jbyte c
unsigned int old_value, value32;
cmpxchg_pre_membar(order);
__asm__ __volatile__ (
/* fence */
strasm_sync
/* simple guard */
" lbz %[old_value], 0(%[dest]) \n"
" cmpw %[masked_compare_val], %[old_value] \n"
@ -331,8 +349,6 @@ inline jbyte Atomic::cmpxchg(jbyte exchange_value, volatile jbyte* dest, jbyte c
" xor %[value32], %[xor_value], %[value32] \n"
" stwcx. %[value32], 0, %[dest_base] \n"
" bne- 1b \n"
/* acquire */
strasm_sync
/* exit */
"2: \n"
/* out */
@ -353,21 +369,23 @@ inline jbyte Atomic::cmpxchg(jbyte exchange_value, volatile jbyte* dest, jbyte c
"memory"
);
cmpxchg_post_membar(order);
return (jbyte)(unsigned char)old_value;
}
inline jint Atomic::cmpxchg(jint exchange_value, volatile jint* dest, jint compare_value) {
inline jint Atomic::cmpxchg(jint exchange_value, volatile jint* dest, jint compare_value, cmpxchg_memory_order order) {
// Note that cmpxchg guarantees a two-way memory barrier across
// the cmpxchg, so it's really a a 'fence_cmpxchg_acquire'
// (see atomic.hpp).
// the cmpxchg, so it's really a a 'fence_cmpxchg_fence' if not
// specified otherwise (see atomic.hpp).
unsigned int old_value;
const uint64_t zero = 0;
cmpxchg_pre_membar(order);
__asm__ __volatile__ (
/* fence */
strasm_sync
/* simple guard */
" lwz %[old_value], 0(%[dest]) \n"
" cmpw %[compare_value], %[old_value] \n"
@ -379,8 +397,6 @@ inline jint Atomic::cmpxchg(jint exchange_value, volatile jint* dest, jint compa
" bne- 2f \n"
" stwcx. %[exchange_value], %[dest], %[zero] \n"
" bne- 1b \n"
/* acquire */
strasm_sync
/* exit */
"2: \n"
/* out */
@ -397,21 +413,23 @@ inline jint Atomic::cmpxchg(jint exchange_value, volatile jint* dest, jint compa
"memory"
);
cmpxchg_post_membar(order);
return (jint) old_value;
}
inline jlong Atomic::cmpxchg(jlong exchange_value, volatile jlong* dest, jlong compare_value) {
inline jlong Atomic::cmpxchg(jlong exchange_value, volatile jlong* dest, jlong compare_value, cmpxchg_memory_order order) {
// Note that cmpxchg guarantees a two-way memory barrier across
// the cmpxchg, so it's really a a 'fence_cmpxchg_acquire'
// (see atomic.hpp).
// the cmpxchg, so it's really a a 'fence_cmpxchg_fence' if not
// specified otherwise (see atomic.hpp).
long old_value;
const uint64_t zero = 0;
cmpxchg_pre_membar(order);
__asm__ __volatile__ (
/* fence */
strasm_sync
/* simple guard */
" ld %[old_value], 0(%[dest]) \n"
" cmpd %[compare_value], %[old_value] \n"
@ -423,8 +441,6 @@ inline jlong Atomic::cmpxchg(jlong exchange_value, volatile jlong* dest, jlong c
" bne- 2f \n"
" stdcx. %[exchange_value], %[dest], %[zero] \n"
" bne- 1b \n"
/* acquire */
strasm_sync
/* exit */
"2: \n"
/* out */
@ -441,15 +457,17 @@ inline jlong Atomic::cmpxchg(jlong exchange_value, volatile jlong* dest, jlong c
"memory"
);
cmpxchg_post_membar(order);
return (jlong) old_value;
}
inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value) {
return (intptr_t)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value);
inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value, cmpxchg_memory_order order) {
return (intptr_t)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value, order);
}
inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value) {
return (void*)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value);
inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value, cmpxchg_memory_order order) {
return (void*)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value, order);
}
#undef strasm_sync

10
hotspot/src/os_cpu/linux_sparc/vm/atomic_linux_sparc.inline.hpp
View File

@ -148,7 +148,7 @@ inline void* Atomic::xchg_ptr(void* exchange_value, volatile void* des
}
inline jint Atomic::cmpxchg (jint exchange_value, volatile jint* dest, jint compare_value) {
inline jint Atomic::cmpxchg (jint exchange_value, volatile jint* dest, jint compare_value, cmpxchg_memory_order order) {
jint rv;
__asm__ volatile(
" cas [%2], %3, %0"
@ -158,7 +158,7 @@ inline jint Atomic::cmpxchg (jint exchange_value, volatile jint*
return rv;
}
inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value) {
inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value, cmpxchg_memory_order order) {
#ifdef _LP64
jlong rv;
__asm__ volatile(
@ -190,7 +190,7 @@ inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong*
#endif
}
inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value) {
inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value, cmpxchg_memory_order order) {
intptr_t rv;
#ifdef _LP64
__asm__ volatile(
@ -208,8 +208,8 @@ inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t*
return rv;
}
inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value) {
return (void*)cmpxchg_ptr((intptr_t)exchange_value, (volatile intptr_t*)dest, (intptr_t)compare_value);
inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value, cmpxchg_memory_order order) {
return (void*)cmpxchg_ptr((intptr_t)exchange_value, (volatile intptr_t*)dest, (intptr_t)compare_value, order);
}
#endif // OS_CPU_LINUX_SPARC_VM_ATOMIC_LINUX_SPARC_INLINE_HPP

24
hotspot/src/os_cpu/linux_x86/vm/atomic_linux_x86.inline.hpp
View File

@ -89,7 +89,7 @@ inline void* Atomic::xchg_ptr(void* exchange_value, volatile void* des
}
#define VM_HAS_SPECIALIZED_CMPXCHG_BYTE
inline jbyte Atomic::cmpxchg (jbyte exchange_value, volatile jbyte* dest, jbyte compare_value) {
inline jbyte Atomic::cmpxchg (jbyte exchange_value, volatile jbyte* dest, jbyte compare_value, cmpxchg_memory_order order) {
int mp = os::is_MP();
__asm__ volatile (LOCK_IF_MP(%4) "cmpxchgb %1,(%3)"
: "=a" (exchange_value)
@ -98,7 +98,7 @@ inline jbyte Atomic::cmpxchg (jbyte exchange_value, volatile jbyte*
return exchange_value;
}
inline jint Atomic::cmpxchg (jint exchange_value, volatile jint* dest, jint compare_value) {
inline jint Atomic::cmpxchg (jint exchange_value, volatile jint* dest, jint compare_value, cmpxchg_memory_order order) {
int mp = os::is_MP();
__asm__ volatile (LOCK_IF_MP(%4) "cmpxchgl %1,(%3)"
: "=a" (exchange_value)
@ -149,7 +149,7 @@ inline intptr_t Atomic::xchg_ptr(intptr_t exchange_value, volatile intptr_t* des
return exchange_value;
}
inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value) {
inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value, cmpxchg_memory_order order) {
bool mp = os::is_MP();
__asm__ __volatile__ (LOCK_IF_MP(%4) "cmpxchgq %1,(%3)"
: "=a" (exchange_value)
@ -158,12 +158,12 @@ inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong*
return exchange_value;
}
inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value) {
return (intptr_t)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value);
inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value, cmpxchg_memory_order order) {
return (intptr_t)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value, order);
}
inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value) {
return (void*)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value);
inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value, cmpxchg_memory_order order) {
return (void*)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value, order);
}
inline jlong Atomic::load(volatile jlong* src) { return *src; }
@ -197,16 +197,16 @@ extern "C" {
void _Atomic_move_long(volatile jlong* src, volatile jlong* dst);
}
inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value) {
inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value, cmpxchg_memory_order order) {
return _Atomic_cmpxchg_long(exchange_value, dest, compare_value, os::is_MP());
}
inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value) {
return (intptr_t)cmpxchg((jint)exchange_value, (volatile jint*)dest, (jint)compare_value);
inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value, cmpxchg_memory_order order) {
return (intptr_t)cmpxchg((jint)exchange_value, (volatile jint*)dest, (jint)compare_value, order);
}
inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value) {
return (void*)cmpxchg((jint)exchange_value, (volatile jint*)dest, (jint)compare_value);
inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value, cmpxchg_memory_order order) {
return (void*)cmpxchg((jint)exchange_value, (volatile jint*)dest, (jint)compare_value, order);
}
inline jlong Atomic::load(volatile jlong* src) {

15
hotspot/src/os_cpu/linux_zero/vm/atomic_linux_zero.inline.hpp
View File

@ -264,7 +264,8 @@ inline void* Atomic::xchg_ptr(void* exchange_value, volatile void* dest) {
inline jint Atomic::cmpxchg(jint exchange_value,
volatile jint* dest,
jint compare_value) {
jint compare_value,
cmpxchg_memory_order order) {
#ifdef ARM
return arm_compare_and_swap(dest, compare_value, exchange_value);
#else
@ -278,14 +279,16 @@ inline jint Atomic::cmpxchg(jint exchange_value,
inline jlong Atomic::cmpxchg(jlong exchange_value,
volatile jlong* dest,
jlong compare_value) {
jlong compare_value,
cmpxchg_memory_order order) {
return __sync_val_compare_and_swap(dest, compare_value, exchange_value);
}
inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value,
volatile intptr_t* dest,
intptr_t compare_value) {
intptr_t compare_value,
cmpxchg_memory_order order) {
#ifdef ARM
return arm_compare_and_swap(dest, compare_value, exchange_value);
#else
@ -299,11 +302,13 @@ inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value,
inline void* Atomic::cmpxchg_ptr(void* exchange_value,
volatile void* dest,
void* compare_value) {
void* compare_value,
cmpxchg_memory_order order) {
return (void *) cmpxchg_ptr((intptr_t) exchange_value,
(volatile intptr_t*) dest,
(intptr_t) compare_value);
(intptr_t) compare_value,
order);
}
inline jlong Atomic::load(volatile jlong* src) {

32
hotspot/src/os_cpu/solaris_sparc/vm/atomic_solaris_sparc.inline.hpp
View File

@ -177,7 +177,7 @@ inline void* Atomic::xchg_ptr(void* exchange_value, volatile void* des
}
inline jint Atomic::cmpxchg (jint exchange_value, volatile jint* dest, jint compare_value) {
inline jint Atomic::cmpxchg (jint exchange_value, volatile jint* dest, jint compare_value, cmpxchg_memory_order order) {
jint rv;
__asm__ volatile(
" cas [%2], %3, %0"
@ -187,7 +187,7 @@ inline jint Atomic::cmpxchg (jint exchange_value, volatile jint*
return rv;
}
inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value) {
inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value, cmpxchg_memory_order order) {
#ifdef _LP64
jlong rv;
__asm__ volatile(
@ -219,7 +219,7 @@ inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong*
#endif //_LP64
}
inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value) {
inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value, cmpxchg_memory_order order) {
intptr_t rv;
#ifdef _LP64
__asm__ volatile(
@ -237,8 +237,8 @@ inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t*
return rv;
}
inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value) {
return (void*)cmpxchg_ptr((intptr_t)exchange_value, (volatile intptr_t*)dest, (intptr_t)compare_value);
inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value, cmpxchg_memory_order order) {
return (void*)cmpxchg_ptr((intptr_t)exchange_value, (volatile intptr_t*)dest, (intptr_t)compare_value, order);
}
#else // _GNU_SOURCE
@ -296,11 +296,11 @@ inline void* Atomic::xchg_ptr(void* exchange_value, volatile void* des
}
inline jint Atomic::cmpxchg (jint exchange_value, volatile jint* dest, jint compare_value) {
inline jint Atomic::cmpxchg (jint exchange_value, volatile jint* dest, jint compare_value, cmpxchg_memory_order order) {
return _Atomic_cas32(exchange_value, dest, compare_value);
}
inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value) {
inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value, cmpxchg_memory_order order) {
#ifdef _LP64
// Return 64 bit value in %o0
return _Atomic_cas64((intptr_t)exchange_value, (intptr_t *)dest, (intptr_t)compare_value);
@ -310,7 +310,7 @@ inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong*
#endif // _LP64
}
inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value) {
inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value, cmpxchg_memory_order order) {
#ifdef _LP64
return _Atomic_cas64(exchange_value, dest, compare_value);
#else // _LP64
@ -318,8 +318,8 @@ inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t*
#endif // _LP64
}
inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value) {
return (void*)cmpxchg_ptr((intptr_t)exchange_value, (volatile intptr_t*)dest, (intptr_t)compare_value);
inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value, cmpxchg_memory_order order) {
return (void*)cmpxchg_ptr((intptr_t)exchange_value, (volatile intptr_t*)dest, (intptr_t)compare_value, order);
}
@ -354,20 +354,20 @@ inline void* Atomic::xchg_ptr(void* exchange_value, volatile void* des
}
inline jint Atomic::cmpxchg (jint exchange_value, volatile jint* dest, jint compare_value) {
inline jint Atomic::cmpxchg (jint exchange_value, volatile jint* dest, jint compare_value, cmpxchg_memory_order order) {
return (*os::atomic_cmpxchg_func)(exchange_value, dest, compare_value);
}
inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value) {
inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value, cmpxchg_memory_order order) {
return (*os::atomic_cmpxchg_long_func)(exchange_value, dest, compare_value);
}
inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value) {
return (intptr_t)cmpxchg((jint)exchange_value, (volatile jint*)dest, (jint)compare_value);
inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value, cmpxchg_memory_order order) {
return (intptr_t)cmpxchg((jint)exchange_value, (volatile jint*)dest, (jint)compare_value, order);
}
inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value) {
return (void*)cmpxchg((jint)exchange_value, (volatile jint*)dest, (jint)compare_value);
inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value, cmpxchg_memory_order order) {
return (void*)cmpxchg((jint)exchange_value, (volatile jint*)dest, (jint)compare_value, order);
}
#endif // _LP64 || COMPILER2

18
hotspot/src/os_cpu/solaris_x86/vm/atomic_solaris_x86.inline.hpp
View File

@ -85,15 +85,15 @@ inline jint Atomic::xchg (jint exchange_value, volatile jint*
}
#define VM_HAS_SPECIALIZED_CMPXCHG_BYTE
inline jbyte Atomic::cmpxchg (jbyte exchange_value, volatile jbyte* dest, jbyte compare_value) {
inline jbyte Atomic::cmpxchg (jbyte exchange_value, volatile jbyte* dest, jbyte compare_value, cmpxchg_memory_order order) {
return _Atomic_cmpxchg_byte(exchange_value, dest, compare_value IS_MP_ARG());
}
inline jint Atomic::cmpxchg (jint exchange_value, volatile jint* dest, jint compare_value) {
inline jint Atomic::cmpxchg (jint exchange_value, volatile jint* dest, jint compare_value, cmpxchg_memory_order order) {
return _Atomic_cmpxchg(exchange_value, dest, compare_value IS_MP_ARG());
}
inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value) {
inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value, cmpxchg_memory_order order) {
return _Atomic_cmpxchg_long(exchange_value, dest, compare_value IS_MP_ARG());
}
@ -120,11 +120,11 @@ inline void* Atomic::xchg_ptr(void* exchange_value, volatile void* des
return (void*)_Atomic_xchg_long((jlong)exchange_value, (volatile jlong*)dest);
}
inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value) {
inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value, cmpxchg_memory_order order) {
return (intptr_t)_Atomic_cmpxchg_long((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value);
}
inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value) {
inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value, cmpxchg_memory_order order) {
return (void*)_Atomic_cmpxchg_long((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value);
}
@ -148,12 +148,12 @@ inline void* Atomic::xchg_ptr(void* exchange_value, volatile void* des
return (void*)xchg((jint)exchange_value, (volatile jint*)dest);
}
inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value) {
return (intptr_t)cmpxchg((jint)exchange_value, (volatile jint*)dest, (jint)compare_value);
inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value, cmpxchg_memory_order order) {
return (intptr_t)cmpxchg((jint)exchange_value, (volatile jint*)dest, (jint)compare_value, order);
}
inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value) {
return (void*)cmpxchg((jint)exchange_value, (volatile jint*)dest, (jint)compare_value);
inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value, cmpxchg_memory_order order) {
return (void*)cmpxchg((jint)exchange_value, (volatile jint*)dest, (jint)compare_value, order);
}
extern "C" void _Atomic_move_long(volatile jlong* src, volatile jlong* dst);

28
hotspot/src/os_cpu/windows_x86/vm/atomic_windows_x86.inline.hpp
View File

@ -119,25 +119,25 @@ inline void* Atomic::xchg_ptr(void* exchange_value, volatile void* des
return (void *)(os::atomic_xchg_ptr_func)((intptr_t)exchange_value, (volatile intptr_t*)dest);
}
inline jint Atomic::cmpxchg (jint exchange_value, volatile jint* dest, jint compare_value) {
inline jint Atomic::cmpxchg (jint exchange_value, volatile jint* dest, jint compare_value, cmpxchg_memory_order order) {
return (*os::atomic_cmpxchg_func)(exchange_value, dest, compare_value);
}
#define VM_HAS_SPECIALIZED_CMPXCHG_BYTE
inline jbyte Atomic::cmpxchg (jbyte exchange_value, volatile jbyte* dest, jbyte compare_value) {
inline jbyte Atomic::cmpxchg (jbyte exchange_value, volatile jbyte* dest, jbyte compare_value, cmpxchg_memory_order order) {
return (*os::atomic_cmpxchg_byte_func)(exchange_value, dest, compare_value);
}
inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value) {
inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value, cmpxchg_memory_order order) {
return (*os::atomic_cmpxchg_long_func)(exchange_value, dest, compare_value);
}
inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value) {
return (intptr_t)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value);
inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value, cmpxchg_memory_order order) {
return (intptr_t)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value, order);
}
inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value) {
return (void*)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value);
inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value, cmpxchg_memory_order order) {
return (void*)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value, order);
}
inline jlong Atomic::load(volatile jlong* src) { return *src; }
@ -218,7 +218,7 @@ inline void* Atomic::xchg_ptr(void* exchange_value, volatile void* des
}
#define VM_HAS_SPECIALIZED_CMPXCHG_BYTE
inline jbyte Atomic::cmpxchg (jbyte exchange_value, volatile jbyte* dest, jbyte compare_value) {
inline jbyte Atomic::cmpxchg (jbyte exchange_value, volatile jbyte* dest, jbyte compare_value, cmpxchg_memory_order order) {
// alternative for InterlockedCompareExchange
int mp = os::is_MP();
__asm {
@ -230,7 +230,7 @@ inline jbyte Atomic::cmpxchg (jbyte exchange_value, volatile jbyte*
}
}
inline jint Atomic::cmpxchg (jint exchange_value, volatile jint* dest, jint compare_value) {
inline jint Atomic::cmpxchg (jint exchange_value, volatile jint* dest, jint compare_value, cmpxchg_memory_order order) {
// alternative for InterlockedCompareExchange
int mp = os::is_MP();
__asm {
@ -242,7 +242,7 @@ inline jint Atomic::cmpxchg (jint exchange_value, volatile jint*
}
}
inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value) {
inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value, cmpxchg_memory_order order) {
int mp = os::is_MP();
jint ex_lo = (jint)exchange_value;
jint ex_hi = *( ((jint*)&exchange_value) + 1 );
@ -263,12 +263,12 @@ inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong*
}
}
inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value) {
return (intptr_t)cmpxchg((jint)exchange_value, (volatile jint*)dest, (jint)compare_value);
inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value, cmpxchg_memory_order order) {
return (intptr_t)cmpxchg((jint)exchange_value, (volatile jint*)dest, (jint)compare_value, order);
}
inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value) {
return (void*)cmpxchg((jint)exchange_value, (volatile jint*)dest, (jint)compare_value);
inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value, cmpxchg_memory_order order) {
return (void*)cmpxchg((jint)exchange_value, (volatile jint*)dest, (jint)compare_value, order);
}
inline jlong Atomic::load(volatile jlong* src) {

103
hotspot/src/share/vm/runtime/atomic.cpp
View File

@ -1,103 +0,0 @@
/*
* Copyright (c) 2001, 2014, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*
*/
#include "precompiled.hpp"
#include "runtime/atomic.inline.hpp"
/*
* This is the default implementation of byte-sized cmpxchg. It emulates jbyte-sized cmpxchg
* in terms of jint-sized cmpxchg. Platforms may override this by defining their own inline definition
* as well as defining VM_HAS_SPECIALIZED_CMPXCHG_BYTE. This will cause the platform specific
* implementation to be used instead.
*/
jbyte Atomic::cmpxchg_general(jbyte exchange_value, volatile jbyte* dest, jbyte compare_value) {
assert(sizeof(jbyte) == 1, "assumption.");
uintptr_t dest_addr = (uintptr_t)dest;
uintptr_t offset = dest_addr % sizeof(jint);
volatile jint* dest_int = (volatile jint*)(dest_addr - offset);
jint cur = *dest_int;
jbyte* cur_as_bytes = (jbyte*)(&cur);
jint new_val = cur;
jbyte* new_val_as_bytes = (jbyte*)(&new_val);
new_val_as_bytes[offset] = exchange_value;
while (cur_as_bytes[offset] == compare_value) {
jint res = cmpxchg(new_val, dest_int, cur);
if (res == cur) break;
cur = res;
new_val = cur;
new_val_as_bytes[offset] = exchange_value;
}
return cur_as_bytes[offset];
}
unsigned Atomic::xchg(unsigned int exchange_value, volatile unsigned int* dest) {
assert(sizeof(unsigned int) == sizeof(jint), "more work to do");
return (unsigned int)Atomic::xchg((jint)exchange_value, (volatile jint*)dest);
}
unsigned Atomic::cmpxchg(unsigned int exchange_value,
volatile unsigned int* dest, unsigned int compare_value) {
assert(sizeof(unsigned int) == sizeof(jint), "more work to do");
return (unsigned int)Atomic::cmpxchg((jint)exchange_value, (volatile jint*)dest,
(jint)compare_value);
}
jlong Atomic::add(jlong add_value, volatile jlong* dest) {
jlong old = load(dest);
jlong new_value = old + add_value;
while (old != cmpxchg(new_value, dest, old)) {
old = load(dest);
new_value = old + add_value;
}
return old;
}
void Atomic::inc(volatile short* dest) {
// Most platforms do not support atomic increment on a 2-byte value. However,
// if the value occupies the most significant 16 bits of an aligned 32-bit
// word, then we can do this with an atomic add of 0x10000 to the 32-bit word.
//
// The least significant parts of this 32-bit word will never be affected, even
// in case of overflow/underflow.
//
// Use the ATOMIC_SHORT_PAIR macro to get the desired alignment.
#ifdef VM_LITTLE_ENDIAN
assert((intx(dest) & 0x03) == 0x02, "wrong alignment");
(void)Atomic::add(0x10000, (volatile int*)(dest-1));
#else
assert((intx(dest) & 0x03) == 0x00, "wrong alignment");
(void)Atomic::add(0x10000, (volatile int*)(dest));
#endif
}
void Atomic::dec(volatile short* dest) {
#ifdef VM_LITTLE_ENDIAN
assert((intx(dest) & 0x03) == 0x02, "wrong alignment");
(void)Atomic::add(-0x10000, (volatile int*)(dest-1));
#else
assert((intx(dest) & 0x03) == 0x00, "wrong alignment");
(void)Atomic::add(-0x10000, (volatile int*)(dest));
#endif
}

35
hotspot/src/share/vm/runtime/atomic.hpp
View File

@ -27,10 +27,13 @@
#include "memory/allocation.hpp"
class Atomic : AllStatic {
private:
static jbyte cmpxchg_general(jbyte exchange_value, volatile jbyte* dest, jbyte compare_value);
enum cmpxchg_memory_order {
memory_order_relaxed,
// Use value which doesn't interfere with C++2011. We need to be more conservative.
memory_order_conservative = 8
};
class Atomic : AllStatic {
public:
// Atomic operations on jlong types are not available on all 32-bit
// platforms. If atomic ops on jlongs are defined here they must only
@ -77,12 +80,12 @@ class Atomic : AllStatic {
inline static intptr_t add_ptr(intptr_t add_value, volatile intptr_t* dest);
inline static void* add_ptr(intptr_t add_value, volatile void* dest);
// See comment above about using jlong atomics on 32-bit platforms
static jlong add (jlong add_value, volatile jlong* dest);
inline static jlong add (jlong add_value, volatile jlong* dest);
// Atomically increment location. inc*() provide:
// <fence> increment-dest <membar StoreLoad|StoreStore>
inline static void inc (volatile jint* dest);
static void inc (volatile jshort* dest);
inline static void inc (volatile jshort* dest);
inline static void inc (volatile size_t* dest);
inline static void inc_ptr(volatile intptr_t* dest);
inline static void inc_ptr(volatile void* dest);
@ -90,7 +93,7 @@ class Atomic : AllStatic {
// Atomically decrement a location. dec*() provide:
// <fence> decrement-dest <membar StoreLoad|StoreStore>
inline static void dec (volatile jint* dest);
static void dec (volatile jshort* dest);
inline static void dec (volatile jshort* dest);
inline static void dec (volatile size_t* dest);
inline static void dec_ptr(volatile intptr_t* dest);
inline static void dec_ptr(volatile void* dest);
@ -98,22 +101,22 @@ class Atomic : AllStatic {
// Performs atomic exchange of *dest with exchange_value. Returns old
// prior value of *dest. xchg*() provide:
// <fence> exchange-value-with-dest <membar StoreLoad|StoreStore>
inline static jint xchg (jint exchange_value, volatile jint* dest);
static unsigned int xchg (unsigned int exchange_value, volatile unsigned int* dest);
inline static intptr_t xchg_ptr(intptr_t exchange_value, volatile intptr_t* dest);
inline static void* xchg_ptr(void* exchange_value, volatile void* dest);
inline static jint xchg (jint exchange_value, volatile jint* dest);
inline static unsigned int xchg (unsigned int exchange_value, volatile unsigned int* dest);
inline static intptr_t xchg_ptr(intptr_t exchange_value, volatile intptr_t* dest);
inline static void* xchg_ptr(void* exchange_value, volatile void* dest);
// Performs atomic compare of *dest and compare_value, and exchanges
// *dest with exchange_value if the comparison succeeded. Returns prior
// value of *dest. cmpxchg*() provide:
// <fence> compare-and-exchange <membar StoreLoad|StoreStore>
inline static jbyte cmpxchg (jbyte exchange_value, volatile jbyte* dest, jbyte compare_value);
inline static jint cmpxchg (jint exchange_value, volatile jint* dest, jint compare_value);
inline static jbyte cmpxchg (jbyte exchange_value, volatile jbyte* dest, jbyte compare_value, cmpxchg_memory_order order = memory_order_conservative);
inline static jint cmpxchg (jint exchange_value, volatile jint* dest, jint compare_value, cmpxchg_memory_order order = memory_order_conservative);
// See comment above about using jlong atomics on 32-bit platforms
inline static jlong cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value);
static unsigned int cmpxchg (unsigned int exchange_value, volatile unsigned int* dest, unsigned int compare_value);
inline static intptr_t cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value);
inline static void* cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value);
inline static jlong cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value, cmpxchg_memory_order order = memory_order_conservative);
inline static unsigned int cmpxchg (unsigned int exchange_value, volatile unsigned int* dest, unsigned int compare_value, cmpxchg_memory_order order = memory_order_conservative);
inline static intptr_t cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value, cmpxchg_memory_order order = memory_order_conservative);
inline static void* cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value, cmpxchg_memory_order order = memory_order_conservative);
};
// To use Atomic::inc(jshort* dest) and Atomic::dec(jshort* dest), the address must be specially

78
hotspot/src/share/vm/runtime/atomic.inline.hpp
View File

@ -91,11 +91,83 @@ inline void Atomic::dec(volatile size_t* dest) {
}
#ifndef VM_HAS_SPECIALIZED_CMPXCHG_BYTE
// See comment in atomic.cpp how to override.
inline jbyte Atomic::cmpxchg(jbyte exchange_value, volatile jbyte *dest, jbyte comparand)
/*
* This is the default implementation of byte-sized cmpxchg. It emulates jbyte-sized cmpxchg
* in terms of jint-sized cmpxchg. Platforms may override this by defining their own inline definition
* as well as defining VM_HAS_SPECIALIZED_CMPXCHG_BYTE. This will cause the platform specific
* implementation to be used instead.
*/
inline jbyte Atomic::cmpxchg(jbyte exchange_value, volatile jbyte *dest, jbyte comparand, cmpxchg_memory_order order)
{
return cmpxchg_general(exchange_value, dest, comparand);
assert(sizeof(jbyte) == 1, "assumption.");
uintptr_t dest_addr = (uintptr_t)dest;
uintptr_t offset = dest_addr % sizeof(jint);
volatile jint* dest_int = (volatile jint*)(dest_addr - offset);
jint cur = *dest_int;
jbyte* cur_as_bytes = (jbyte*)(&cur);
jint new_val = cur;
jbyte* new_val_as_bytes = (jbyte*)(&new_val);
new_val_as_bytes[offset] = exchange_value;
while (cur_as_bytes[offset] == comparand) {
jint res = cmpxchg(new_val, dest_int, cur, order);
if (res == cur) break;
cur = res;
new_val = cur;
new_val_as_bytes[offset] = exchange_value;
}
return cur_as_bytes[offset];
}
#endif // VM_HAS_SPECIALIZED_CMPXCHG_BYTE
inline unsigned Atomic::xchg(unsigned int exchange_value, volatile unsigned int* dest) {
assert(sizeof(unsigned int) == sizeof(jint), "more work to do");
return (unsigned int)Atomic::xchg((jint)exchange_value, (volatile jint*)dest);
}
inline unsigned Atomic::cmpxchg(unsigned int exchange_value,
volatile unsigned int* dest, unsigned int compare_value,
cmpxchg_memory_order order) {
assert(sizeof(unsigned int) == sizeof(jint), "more work to do");
return (unsigned int)Atomic::cmpxchg((jint)exchange_value, (volatile jint*)dest,
(jint)compare_value, order);
}
inline jlong Atomic::add(jlong add_value, volatile jlong* dest) {
jlong old = load(dest);
jlong new_value = old + add_value;
while (old != cmpxchg(new_value, dest, old)) {
old = load(dest);
new_value = old + add_value;
}
return old;
}
inline void Atomic::inc(volatile short* dest) {
// Most platforms do not support atomic increment on a 2-byte value. However,
// if the value occupies the most significant 16 bits of an aligned 32-bit
// word, then we can do this with an atomic add of 0x10000 to the 32-bit word.
//
// The least significant parts of this 32-bit word will never be affected, even
// in case of overflow/underflow.
//
// Use the ATOMIC_SHORT_PAIR macro to get the desired alignment.
#ifdef VM_LITTLE_ENDIAN
assert((intx(dest) & 0x03) == 0x02, "wrong alignment");
(void)Atomic::add(0x10000, (volatile int*)(dest-1));
#else
assert((intx(dest) & 0x03) == 0x00, "wrong alignment");
(void)Atomic::add(0x10000, (volatile int*)(dest));
#endif
}
inline void Atomic::dec(volatile short* dest) {
#ifdef VM_LITTLE_ENDIAN
assert((intx(dest) & 0x03) == 0x02, "wrong alignment");
(void)Atomic::add(-0x10000, (volatile int*)(dest-1));
#else
assert((intx(dest) & 0x03) == 0x00, "wrong alignment");
(void)Atomic::add(-0x10000, (volatile int*)(dest));
#endif
}
#endif // SHARE_VM_RUNTIME_ATOMIC_INLINE_HPP