8310644: Make panama memory segment close use async handshakes

Reviewed-by: jvernee, mcimadamore, pchilanomate
This commit is contained in:
Erik Österlund 2023-11-29 12:40:21 +00:00
parent 65dfcae6d6
commit 159465324f
8 changed files with 162 additions and 149 deletions

@ -35,54 +35,72 @@
#include "runtime/sharedRuntime.hpp"
#include "runtime/vframe.inline.hpp"
class CloseScopedMemoryFindOopClosure : public OopClosure {
oop _deopt;
bool _found;
static bool is_in_scoped_access(JavaThread* jt, oop session) {
const int max_critical_stack_depth = 10;
int depth = 0;
for (vframeStream stream(jt); !stream.at_end(); stream.next()) {
Method* m = stream.method();
if (m->is_scoped()) {
StackValueCollection* locals = stream.asJavaVFrame()->locals();
for (int i = 0; i < locals->size(); i++) {
StackValue* var = locals->at(i);
if (var->type() == T_OBJECT) {
if (var->get_obj() == session) {
assert(depth < max_critical_stack_depth, "can't have more than %d critical frames", max_critical_stack_depth);
return true;
}
}
}
break;
}
depth++;
#ifndef ASSERT
if (depth >= max_critical_stack_depth) {
break;
}
#endif
}
return false;
}
class ScopedAsyncExceptionHandshake : public AsyncExceptionHandshake {
OopHandle _session;
public:
CloseScopedMemoryFindOopClosure(jobject deopt) :
_deopt(JNIHandles::resolve(deopt)),
_found(false) {}
ScopedAsyncExceptionHandshake(OopHandle& session, OopHandle& error)
: AsyncExceptionHandshake(error),
_session(session) {}
template <typename T>
void do_oop_work(T* p) {
if (_found) {
return;
~ScopedAsyncExceptionHandshake() {
_session.release(Universe::vm_global());
}
virtual void do_thread(Thread* thread) {
JavaThread* jt = JavaThread::cast(thread);
ResourceMark rm;
if (is_in_scoped_access(jt, _session.resolve())) {
// Throw exception to unwind out from the scoped access
AsyncExceptionHandshake::do_thread(thread);
}
if (RawAccess<>::oop_load(p) == _deopt) {
_found = true;
}
}
virtual void do_oop(oop* p) {
do_oop_work(p);
}
virtual void do_oop(narrowOop* p) {
do_oop_work(p);
}
bool found() {
return _found;
}
};
class CloseScopedMemoryClosure : public HandshakeClosure {
jobject _deopt;
jobject _session;
jobject _error;
public:
jboolean _found;
CloseScopedMemoryClosure(jobject deopt, jobject exception)
CloseScopedMemoryClosure(jobject session, jobject error)
: HandshakeClosure("CloseScopedMemory")
, _deopt(deopt)
, _found(false) {}
, _session(session)
, _error(error) {}
void do_thread(Thread* thread) {
JavaThread* jt = JavaThread::cast(thread);
if (!jt->has_last_Java_frame()) {
// No frames; not in a scoped memory access
return;
}
@ -97,44 +115,27 @@ public:
}
ResourceMark rm;
if (_deopt != nullptr && last_frame.is_compiled_frame() && last_frame.can_be_deoptimized()) {
CloseScopedMemoryFindOopClosure cl(_deopt);
CompiledMethod* cm = last_frame.cb()->as_compiled_method();
/* FIXME: this doesn't work if reachability fences are violated by C2
last_frame.oops_do(&cl, nullptr, &register_map);
if (cl.found()) {
//Found the deopt oop in a compiled method; deoptimize.
Deoptimization::deoptimize(jt, last_frame);
}
so... we unconditionally deoptimize, for now: */
if (last_frame.is_compiled_frame() && last_frame.can_be_deoptimized()) {
// FIXME: we would like to conditionally deoptimize only if the corresponding
// _session is reachable from the frame, but reachabilityFence doesn't currently
// work the way it should. Therefore we deopt unconditionally for now.
Deoptimization::deoptimize(jt, last_frame);
}
const int max_critical_stack_depth = 10;
int depth = 0;
for (vframeStream stream(jt); !stream.at_end(); stream.next()) {
Method* m = stream.method();
if (m->is_scoped()) {
StackValueCollection* locals = stream.asJavaVFrame()->locals();
for (int i = 0; i < locals->size(); i++) {
StackValue* var = locals->at(i);
if (var->type() == T_OBJECT) {
if (var->get_obj() == JNIHandles::resolve(_deopt)) {
assert(depth < max_critical_stack_depth, "can't have more than %d critical frames", max_critical_stack_depth);
_found = true;
return;
}
}
}
break;
}
depth++;
#ifndef ASSERT
if (depth >= max_critical_stack_depth) {
break;
}
#endif
if (jt->has_async_exception_condition()) {
// Target thread just about to throw an async exception using async handshakes,
// we will then unwind out from the scoped memory access.
return;
}
if (is_in_scoped_access(jt, JNIHandles::resolve(_session))) {
// We have found that the target thread is inside of a scoped access.
// An asynchronous handshake is sent to the target thread, telling it
// to throw an exception, which will unwind the target thread out from
// the scoped access.
OopHandle session(Universe::vm_global(), JNIHandles::resolve(_session));
OopHandle error(Universe::vm_global(), JNIHandles::resolve(_error));
jt->install_async_exception(new ScopedAsyncExceptionHandshake(session, error));
}
}
};
@ -146,10 +147,9 @@ public:
* class annotated with the '@Scoped' annotation), and whose local variables mention the session being
* closed (deopt), this method returns false, signalling that the session cannot be closed safely.
*/
JVM_ENTRY(jboolean, ScopedMemoryAccess_closeScope(JNIEnv *env, jobject receiver, jobject deopt, jobject exception))
CloseScopedMemoryClosure cl(deopt, exception);
JVM_ENTRY(void, ScopedMemoryAccess_closeScope(JNIEnv *env, jobject receiver, jobject session, jobject error))
CloseScopedMemoryClosure cl(session, error);
Handshake::execute(&cl);
return !cl._found;
JVM_END
/// JVM_RegisterUnsafeMethods
@ -157,14 +157,14 @@ JVM_END
#define PKG_MISC "Ljdk/internal/misc/"
#define PKG_FOREIGN "Ljdk/internal/foreign/"
#define MEMACCESS "ScopedMemoryAccess"
#define SCOPE PKG_FOREIGN "MemorySessionImpl;"
#define SCOPED_SESSION PKG_FOREIGN "MemorySessionImpl;"
#define SCOPED_ERROR PKG_MISC "ScopedMemoryAccess$ScopedAccessError;"
#define CC (char*) /*cast a literal from (const char*)*/
#define FN_PTR(f) CAST_FROM_FN_PTR(void*, &f)
static JNINativeMethod jdk_internal_misc_ScopedMemoryAccess_methods[] = {
{CC "closeScope0", CC "(" SCOPE ")Z", FN_PTR(ScopedMemoryAccess_closeScope)},
{CC "closeScope0", CC "(" SCOPED_SESSION SCOPED_ERROR ")V", FN_PTR(ScopedMemoryAccess_closeScope)},
};
#undef CC
@ -172,8 +172,8 @@ static JNINativeMethod jdk_internal_misc_ScopedMemoryAccess_methods[] = {
#undef PKG_MISC
#undef PKG_FOREIGN
#undef MEMACCESS
#undef SCOPE
#undef SCOPED_SESSION
#undef SCOPED_ERROR
// This function is exported, used by NativeLookup.

@ -70,13 +70,36 @@
( arrayOopDesc::header_size(T_DOUBLE) * HeapWordSize \
+ ((julong)max_jint * sizeof(double)) )
#define UNSAFE_ENTRY(result_type, header) \
JVM_ENTRY(static result_type, header)
#define UNSAFE_LEAF(result_type, header) \
JVM_LEAF(static result_type, header)
// Note that scoped accesses (cf. scopedMemoryAccess.cpp) can install
// an async handshake on the entry to an Unsafe method. When that happens,
// it is expected that we are not allowed to touch the underlying memory
// that might have gotten unmapped. Therefore, we check at the entry
// to unsafe functions, if we have such async exception conditions,
// and return immediately if that is the case.
//
// We also use NoSafepointVerifier to block potential safepoints.
// It would be problematic if an async exception handshake were installed later on
// during another safepoint in the function, but before the memory access happens,
// as the memory will be freed after the handshake is installed. We must notice
// the installed handshake and return early before doing the memory access to prevent
// accesses to freed memory.
//
// Note also that we MUST do a scoped memory access in the VM (or Java) thread
// state. Since we rely on a handshake to check for threads that are accessing
// scoped memory, and we need the handshaking thread to wait until we get to a
// safepoint, in order to make sure we are not in the middle of accessing memory
// that is about to be freed. (i.e. there can be no UNSAFE_LEAF_SCOPED)
#define UNSAFE_ENTRY_SCOPED(result_type, header) \
JVM_ENTRY(static result_type, header) \
if (thread->has_async_exception_condition()) {return (result_type)0;} \
NoSafepointVerifier nsv;
#define UNSAFE_END JVM_END
@ -279,11 +302,11 @@ UNSAFE_ENTRY(jobject, Unsafe_GetUncompressedObject(JNIEnv *env, jobject unsafe,
#define DEFINE_GETSETOOP(java_type, Type) \
\
UNSAFE_ENTRY(java_type, Unsafe_Get##Type(JNIEnv *env, jobject unsafe, jobject obj, jlong offset)) { \
UNSAFE_ENTRY_SCOPED(java_type, Unsafe_Get##Type(JNIEnv *env, jobject unsafe, jobject obj, jlong offset)) { \
return MemoryAccess<java_type>(thread, obj, offset).get(); \
} UNSAFE_END \
\
UNSAFE_ENTRY(void, Unsafe_Put##Type(JNIEnv *env, jobject unsafe, jobject obj, jlong offset, java_type x)) { \
UNSAFE_ENTRY_SCOPED(void, Unsafe_Put##Type(JNIEnv *env, jobject unsafe, jobject obj, jlong offset, java_type x)) { \
MemoryAccess<java_type>(thread, obj, offset).put(x); \
} UNSAFE_END \
\
@ -302,11 +325,11 @@ DEFINE_GETSETOOP(jdouble, Double);
#define DEFINE_GETSETOOP_VOLATILE(java_type, Type) \
\
UNSAFE_ENTRY(java_type, Unsafe_Get##Type##Volatile(JNIEnv *env, jobject unsafe, jobject obj, jlong offset)) { \
UNSAFE_ENTRY_SCOPED(java_type, Unsafe_Get##Type##Volatile(JNIEnv *env, jobject unsafe, jobject obj, jlong offset)) { \
return MemoryAccess<java_type>(thread, obj, offset).get_volatile(); \
} UNSAFE_END \
\
UNSAFE_ENTRY(void, Unsafe_Put##Type##Volatile(JNIEnv *env, jobject unsafe, jobject obj, jlong offset, java_type x)) { \
UNSAFE_ENTRY_SCOPED(void, Unsafe_Put##Type##Volatile(JNIEnv *env, jobject unsafe, jobject obj, jlong offset, java_type x)) { \
MemoryAccess<java_type>(thread, obj, offset).put_volatile(x); \
} UNSAFE_END \
\
@ -362,7 +385,7 @@ UNSAFE_LEAF(void, Unsafe_FreeMemory0(JNIEnv *env, jobject unsafe, jlong addr)) {
os::free(p);
} UNSAFE_END
UNSAFE_ENTRY(void, Unsafe_SetMemory0(JNIEnv *env, jobject unsafe, jobject obj, jlong offset, jlong size, jbyte value)) {
UNSAFE_ENTRY_SCOPED(void, Unsafe_SetMemory0(JNIEnv *env, jobject unsafe, jobject obj, jlong offset, jlong size, jbyte value)) {
size_t sz = (size_t)size;
oop base = JNIHandles::resolve(obj);
@ -371,7 +394,7 @@ UNSAFE_ENTRY(void, Unsafe_SetMemory0(JNIEnv *env, jobject unsafe, jobject obj, j
Copy::fill_to_memory_atomic(p, sz, value);
} UNSAFE_END
UNSAFE_ENTRY(void, Unsafe_CopyMemory0(JNIEnv *env, jobject unsafe, jobject srcObj, jlong srcOffset, jobject dstObj, jlong dstOffset, jlong size)) {
UNSAFE_ENTRY_SCOPED(void, Unsafe_CopyMemory0(JNIEnv *env, jobject unsafe, jobject srcObj, jlong srcOffset, jobject dstObj, jlong dstOffset, jlong size)) {
size_t sz = (size_t)size;
oop srcp = JNIHandles::resolve(srcObj);
@ -390,39 +413,19 @@ UNSAFE_ENTRY(void, Unsafe_CopyMemory0(JNIEnv *env, jobject unsafe, jobject srcOb
}
} UNSAFE_END
// This function is a leaf since if the source and destination are both in native memory
// the copy may potentially be very large, and we don't want to disable GC if we can avoid it.
// If either source or destination (or both) are on the heap, the function will enter VM using
// JVM_ENTRY_FROM_LEAF
UNSAFE_LEAF(void, Unsafe_CopySwapMemory0(JNIEnv *env, jobject unsafe, jobject srcObj, jlong srcOffset, jobject dstObj, jlong dstOffset, jlong size, jlong elemSize)) {
UNSAFE_ENTRY_SCOPED(void, Unsafe_CopySwapMemory0(JNIEnv *env, jobject unsafe, jobject srcObj, jlong srcOffset, jobject dstObj, jlong dstOffset, jlong size, jlong elemSize)) {
size_t sz = (size_t)size;
size_t esz = (size_t)elemSize;
if (srcObj == nullptr && dstObj == nullptr) {
// Both src & dst are in native memory
address src = (address)srcOffset;
address dst = (address)dstOffset;
oop srcp = JNIHandles::resolve(srcObj);
oop dstp = JNIHandles::resolve(dstObj);
{
JavaThread* thread = JavaThread::thread_from_jni_environment(env);
GuardUnsafeAccess guard(thread);
Copy::conjoint_swap(src, dst, sz, esz);
}
} else {
// At least one of src/dst are on heap, transition to VM to access raw pointers
address src = (address)index_oop_from_field_offset_long(srcp, srcOffset);
address dst = (address)index_oop_from_field_offset_long(dstp, dstOffset);
JVM_ENTRY_FROM_LEAF(env, void, Unsafe_CopySwapMemory0) {
oop srcp = JNIHandles::resolve(srcObj);
oop dstp = JNIHandles::resolve(dstObj);
address src = (address)index_oop_from_field_offset_long(srcp, srcOffset);
address dst = (address)index_oop_from_field_offset_long(dstp, dstOffset);
{
GuardUnsafeAccess guard(thread);
Copy::conjoint_swap(src, dst, sz, esz);
}
} JVM_END
{
GuardUnsafeAccess guard(thread);
Copy::conjoint_swap(src, dst, sz, esz);
}
} UNSAFE_END
@ -718,13 +721,13 @@ UNSAFE_ENTRY(jobject, Unsafe_CompareAndExchangeReference(JNIEnv *env, jobject un
return JNIHandles::make_local(THREAD, res);
} UNSAFE_END
UNSAFE_ENTRY(jint, Unsafe_CompareAndExchangeInt(JNIEnv *env, jobject unsafe, jobject obj, jlong offset, jint e, jint x)) {
UNSAFE_ENTRY_SCOPED(jint, Unsafe_CompareAndExchangeInt(JNIEnv *env, jobject unsafe, jobject obj, jlong offset, jint e, jint x)) {
oop p = JNIHandles::resolve(obj);
volatile jint* addr = (volatile jint*)index_oop_from_field_offset_long(p, offset);
return Atomic::cmpxchg(addr, e, x);
} UNSAFE_END
UNSAFE_ENTRY(jlong, Unsafe_CompareAndExchangeLong(JNIEnv *env, jobject unsafe, jobject obj, jlong offset, jlong e, jlong x)) {
UNSAFE_ENTRY_SCOPED(jlong, Unsafe_CompareAndExchangeLong(JNIEnv *env, jobject unsafe, jobject obj, jlong offset, jlong e, jlong x)) {
oop p = JNIHandles::resolve(obj);
volatile jlong* addr = (volatile jlong*)index_oop_from_field_offset_long(p, offset);
return Atomic::cmpxchg(addr, e, x);
@ -739,13 +742,13 @@ UNSAFE_ENTRY(jboolean, Unsafe_CompareAndSetReference(JNIEnv *env, jobject unsafe
return ret == e;
} UNSAFE_END
UNSAFE_ENTRY(jboolean, Unsafe_CompareAndSetInt(JNIEnv *env, jobject unsafe, jobject obj, jlong offset, jint e, jint x)) {
UNSAFE_ENTRY_SCOPED(jboolean, Unsafe_CompareAndSetInt(JNIEnv *env, jobject unsafe, jobject obj, jlong offset, jint e, jint x)) {
oop p = JNIHandles::resolve(obj);
volatile jint* addr = (volatile jint*)index_oop_from_field_offset_long(p, offset);
return Atomic::cmpxchg(addr, e, x) == e;
} UNSAFE_END
UNSAFE_ENTRY(jboolean, Unsafe_CompareAndSetLong(JNIEnv *env, jobject unsafe, jobject obj, jlong offset, jlong e, jlong x)) {
UNSAFE_ENTRY_SCOPED(jboolean, Unsafe_CompareAndSetLong(JNIEnv *env, jobject unsafe, jobject obj, jlong offset, jlong e, jlong x)) {
oop p = JNIHandles::resolve(obj);
volatile jlong* addr = (volatile jlong*)index_oop_from_field_offset_long(p, offset);
return Atomic::cmpxchg(addr, e, x) == e;

@ -406,14 +406,6 @@ extern "C" { \
VM_LEAF_BASE(result_type, header)
#define JVM_ENTRY_FROM_LEAF(env, result_type, header) \
{ { \
JavaThread* thread=JavaThread::thread_from_jni_environment(env); \
ThreadInVMfromNative __tiv(thread); \
debug_only(VMNativeEntryWrapper __vew;) \
VM_ENTRY_BASE_FROM_LEAF(result_type, header, thread)
#define JVM_END } }
#endif // SHARE_RUNTIME_INTERFACESUPPORT_INLINE_HPP

@ -225,9 +225,9 @@ class JavaThread: public Thread {
friend class AsyncExceptionHandshake;
friend class HandshakeState;
void install_async_exception(AsyncExceptionHandshake* aec = nullptr);
void handle_async_exception(oop java_throwable);
public:
void install_async_exception(AsyncExceptionHandshake* aec = nullptr);
bool has_async_exception_condition();
inline void set_pending_unsafe_access_error();
static void send_async_exception(JavaThread* jt, oop java_throwable);

@ -55,8 +55,7 @@ public abstract sealed class MemorySessionImpl
implements Scope
permits ConfinedSession, GlobalSession, SharedSession {
static final int OPEN = 0;
static final int CLOSING = -1;
static final int CLOSED = -2;
static final int CLOSED = -1;
static final VarHandle STATE;
static final int MAX_FORKS = Integer.MAX_VALUE;

@ -77,17 +77,13 @@ sealed class SharedSession extends MemorySessionImpl permits ImplicitSession {
}
void justClose() {
int prevState = (int) STATE.compareAndExchange(this, OPEN, CLOSING);
int prevState = (int) STATE.compareAndExchange(this, OPEN, CLOSED);
if (prevState < 0) {
throw alreadyClosed();
} else if (prevState != OPEN) {
throw alreadyAcquired(prevState);
}
boolean success = SCOPED_MEMORY_ACCESS.closeScope(this);
STATE.setVolatile(this, success ? CLOSED : OPEN);
if (!success) {
throw alreadyAcquired(1);
}
SCOPED_MEMORY_ACCESS.closeScope(this, ALREADY_CLOSED);
}
/**

@ -83,11 +83,11 @@ public class ScopedMemoryAccess {
registerNatives();
}
public boolean closeScope(MemorySessionImpl session) {
return closeScope0(session);
public void closeScope(MemorySessionImpl session, ScopedAccessError error) {
closeScope0(session, error);
}
native boolean closeScope0(MemorySessionImpl session);
native void closeScope0(MemorySessionImpl session, ScopedAccessError error);
private ScopedMemoryAccess() {}

@ -33,6 +33,7 @@
import java.lang.foreign.Arena;
import java.lang.foreign.MemorySegment;
import java.lang.foreign.ValueLayout;
import java.lang.invoke.MethodHandles;
import java.lang.invoke.VarHandle;
import java.nio.ByteBuffer;
@ -48,6 +49,7 @@ import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
import static java.lang.foreign.ValueLayout.JAVA_BYTE;
import static java.lang.foreign.ValueLayout.JAVA_INT;
import static org.testng.Assert.*;
public class TestHandshake {
@ -98,16 +100,19 @@ public class TestHandshake {
@Override
public final void run() {
start("\"Accessor #\" + id");
outer: while (segment.scope().isAlive()) {
while (segment.scope().isAlive()) {
try {
doAccess();
} catch (IllegalStateException ex) {
long delay = System.currentTimeMillis() - start.get();
System.out.println("Accessor #" + id + " suspending - elapsed (ms): " + delay);
backoff();
delay = System.currentTimeMillis() - start.get();
System.out.println("Accessor #" + id + " resuming - elapsed (ms): " + delay);
continue outer;
if (!failed.get()) {
// ignore - this means segment was alive, but was closed while we were accessing it
// next isAlive test should fail
assertFalse(segment.scope().isAlive());
failed.set(true);
} else {
// rethrow!
throw ex;
}
}
}
long delay = System.currentTimeMillis() - start.get();
@ -174,6 +179,30 @@ public class TestHandshake {
}
}
static class SegmentSwappyCopyAccessor extends AbstractSegmentAccessor {
MemorySegment first, second;
ValueLayout sourceLayout, destLayout;
long count;
SegmentSwappyCopyAccessor(int id, MemorySegment segment, Arena _unused) {
super(id, segment);
long split = segment.byteSize() / 2;
first = segment.asSlice(0, split);
sourceLayout = JAVA_INT.withOrder(ByteOrder.LITTLE_ENDIAN);
second = segment.asSlice(split);
destLayout = JAVA_INT.withOrder(ByteOrder.BIG_ENDIAN);
count = Math.min(first.byteSize() / sourceLayout.byteSize(),
second.byteSize() / destLayout.byteSize());
}
@Override
public void doAccess() {
MemorySegment.copy(first, sourceLayout, 0L, second, destLayout, 0L, count);
}
}
static class SegmentFillAccessor extends AbstractSegmentAccessor {
SegmentFillAccessor(int id, MemorySegment segment, Arena _unused) {
@ -246,14 +275,7 @@ public class TestHandshake {
@Override
public void run() {
start("Handshaker");
while (true) {
try {
arena.close();
break;
} catch (IllegalStateException ex) {
Thread.onSpinWait();
}
}
arena.close(); // This should NOT throw
long delay = System.currentTimeMillis() - start.get();
System.out.println("Segment closed - elapsed (ms): " + delay);
}
@ -268,6 +290,7 @@ public class TestHandshake {
return new Object[][] {
{ "SegmentAccessor", (AccessorFactory)SegmentAccessor::new },
{ "SegmentCopyAccessor", (AccessorFactory)SegmentCopyAccessor::new },
{ "SegmentSwappyCopyAccessor", (AccessorFactory)SegmentSwappyCopyAccessor::new },
{ "SegmentMismatchAccessor", (AccessorFactory)SegmentMismatchAccessor::new },
{ "SegmentFillAccessor", (AccessorFactory)SegmentFillAccessor::new },
{ "BufferAccessor", (AccessorFactory)BufferAccessor::new },