8338677: Improve startup of memory access var handles by simplifying combinator chains
Reviewed-by: redestad
This commit is contained in:
Maurizio Cimadamore
parent
3aeb6733f9
commit
0e8fe3550b
@ -25,9 +25,11 @@
|
||||
package java.lang.invoke;
|
||||
|
||||
import jdk.internal.foreign.AbstractMemorySegmentImpl;
|
||||
import jdk.internal.foreign.Utils;
|
||||
import jdk.internal.misc.ScopedMemoryAccess;
|
||||
import jdk.internal.vm.annotation.ForceInline;
|
||||
|
||||
import java.lang.foreign.MemoryLayout;
|
||||
import java.lang.foreign.MemorySegment;
|
||||
import java.lang.ref.Reference;
|
||||
|
||||
@ -45,7 +47,7 @@ final class VarHandleSegmentAs$Type$s extends VarHandleSegmentViewBase {
|
||||
|
||||
static final int NON_PLAIN_ACCESS_MIN_ALIGN_MASK = $BoxType$.BYTES - 1;
|
||||
|
||||
static final VarForm FORM = new VarForm(VarHandleSegmentAs$Type$s.class, MemorySegment.class, $type$.class, long.class);
|
||||
static final VarForm FORM = new VarForm(VarHandleSegmentAs$Type$s.class, MemorySegment.class, $type$.class, MemoryLayout.class, long.class, long.class);
|
||||
|
||||
VarHandleSegmentAs$Type$s(boolean be, long alignmentMask, boolean exact) {
|
||||
super(FORM, be, alignmentMask, exact);
|
||||
@ -53,7 +55,7 @@ final class VarHandleSegmentAs$Type$s extends VarHandleSegmentViewBase {
|
||||
|
||||
@Override
|
||||
final MethodType accessModeTypeUncached(VarHandle.AccessType accessType) {
|
||||
return accessType.accessModeType(MemorySegment.class, $type$.class, long.class);
|
||||
return accessType.accessModeType(MemorySegment.class, $type$.class, MemoryLayout.class, long.class, long.class);
|
||||
}
|
||||
|
||||
@Override
|
||||
@ -97,70 +99,70 @@ final class VarHandleSegmentAs$Type$s extends VarHandleSegmentViewBase {
|
||||
#end[floatingPoint]
|
||||
|
||||
@ForceInline
|
||||
static AbstractMemorySegmentImpl checkReadOnly(Object obb, boolean ro) {
|
||||
static AbstractMemorySegmentImpl checkSegment(Object obb, Object encl, long base, boolean ro) {
|
||||
AbstractMemorySegmentImpl oo = (AbstractMemorySegmentImpl)Objects.requireNonNull(obb);
|
||||
oo.checkReadOnly(ro);
|
||||
oo.checkEnclosingLayout(base, (MemoryLayout)encl, ro);
|
||||
return oo;
|
||||
}
|
||||
|
||||
@ForceInline
|
||||
static long offsetNonPlain(AbstractMemorySegmentImpl bb, long offset, long alignmentMask) {
|
||||
static long offsetNonPlain(AbstractMemorySegmentImpl bb, long base, long offset, long alignmentMask) {
|
||||
if ((alignmentMask & NON_PLAIN_ACCESS_MIN_ALIGN_MASK) != NON_PLAIN_ACCESS_MIN_ALIGN_MASK) {
|
||||
throw VarHandleSegmentViewBase.newUnsupportedAccessModeForAlignment(alignmentMask + 1);
|
||||
}
|
||||
return offsetPlain(bb, offset);
|
||||
return offsetPlain(bb, base, offset);
|
||||
}
|
||||
|
||||
@ForceInline
|
||||
static long offsetPlain(AbstractMemorySegmentImpl bb, long offset) {
|
||||
long base = bb.unsafeGetOffset();
|
||||
return base + offset;
|
||||
static long offsetPlain(AbstractMemorySegmentImpl bb, long base, long offset) {
|
||||
long segment_base = bb.unsafeGetOffset();
|
||||
return segment_base + base + offset;
|
||||
}
|
||||
|
||||
@ForceInline
|
||||
static $type$ get(VarHandle ob, Object obb, long base) {
|
||||
static $type$ get(VarHandle ob, Object obb, Object encl, long base, long offset) {
|
||||
VarHandleSegmentViewBase handle = (VarHandleSegmentViewBase)ob;
|
||||
AbstractMemorySegmentImpl bb = checkReadOnly(obb, true);
|
||||
AbstractMemorySegmentImpl bb = checkSegment(obb, encl, base, true);
|
||||
#if[floatingPoint]
|
||||
$rawType$ rawValue = SCOPED_MEMORY_ACCESS.get$RawType$Unaligned(bb.sessionImpl(),
|
||||
bb.unsafeGetBase(),
|
||||
offsetPlain(bb, base),
|
||||
offsetPlain(bb, base, offset),
|
||||
handle.be);
|
||||
return $Type$.$rawType$BitsTo$Type$(rawValue);
|
||||
#else[floatingPoint]
|
||||
#if[byte]
|
||||
return SCOPED_MEMORY_ACCESS.get$Type$(bb.sessionImpl(),
|
||||
bb.unsafeGetBase(),
|
||||
offsetPlain(bb, base));
|
||||
offsetPlain(bb, base, offset));
|
||||
#else[byte]
|
||||
return SCOPED_MEMORY_ACCESS.get$Type$Unaligned(bb.sessionImpl(),
|
||||
bb.unsafeGetBase(),
|
||||
offsetPlain(bb, base),
|
||||
offsetPlain(bb, base, offset),
|
||||
handle.be);
|
||||
#end[byte]
|
||||
#end[floatingPoint]
|
||||
}
|
||||
|
||||
@ForceInline
|
||||
static void set(VarHandle ob, Object obb, long base, $type$ value) {
|
||||
static void set(VarHandle ob, Object obb, Object encl, long base, long offset, $type$ value) {
|
||||
VarHandleSegmentViewBase handle = (VarHandleSegmentViewBase)ob;
|
||||
AbstractMemorySegmentImpl bb = checkReadOnly(obb, false);
|
||||
AbstractMemorySegmentImpl bb = checkSegment(obb, encl, base, false);
|
||||
#if[floatingPoint]
|
||||
SCOPED_MEMORY_ACCESS.put$RawType$Unaligned(bb.sessionImpl(),
|
||||
bb.unsafeGetBase(),
|
||||
offsetPlain(bb, base),
|
||||
offsetPlain(bb, base, offset),
|
||||
$Type$.$type$ToRaw$RawType$Bits(value),
|
||||
handle.be);
|
||||
#else[floatingPoint]
|
||||
#if[byte]
|
||||
SCOPED_MEMORY_ACCESS.put$Type$(bb.sessionImpl(),
|
||||
bb.unsafeGetBase(),
|
||||
offsetPlain(bb, base),
|
||||
offsetPlain(bb, base, offset),
|
||||
value);
|
||||
#else[byte]
|
||||
SCOPED_MEMORY_ACCESS.put$Type$Unaligned(bb.sessionImpl(),
|
||||
bb.unsafeGetBase(),
|
||||
offsetPlain(bb, base),
|
||||
offsetPlain(bb, base, offset),
|
||||
value,
|
||||
handle.be);
|
||||
#end[byte]
|
||||
@ -168,223 +170,223 @@ final class VarHandleSegmentAs$Type$s extends VarHandleSegmentViewBase {
|
||||
}
|
||||
|
||||
@ForceInline
|
||||
static $type$ getVolatile(VarHandle ob, Object obb, long base) {
|
||||
static $type$ getVolatile(VarHandle ob, Object obb, Object encl, long base, long offset) {
|
||||
VarHandleSegmentViewBase handle = (VarHandleSegmentViewBase)ob;
|
||||
AbstractMemorySegmentImpl bb = checkReadOnly(obb, true);
|
||||
AbstractMemorySegmentImpl bb = checkSegment(obb, encl, base, true);
|
||||
return convEndian(handle.be,
|
||||
SCOPED_MEMORY_ACCESS.get$RawType$Volatile(bb.sessionImpl(),
|
||||
bb.unsafeGetBase(),
|
||||
offsetNonPlain(bb, base, handle.alignmentMask)));
|
||||
offsetNonPlain(bb, base, offset, handle.alignmentMask)));
|
||||
}
|
||||
|
||||
@ForceInline
|
||||
static void setVolatile(VarHandle ob, Object obb, long base, $type$ value) {
|
||||
static void setVolatile(VarHandle ob, Object obb, Object encl, long base, long offset, $type$ value) {
|
||||
VarHandleSegmentViewBase handle = (VarHandleSegmentViewBase)ob;
|
||||
AbstractMemorySegmentImpl bb = checkReadOnly(obb, false);
|
||||
AbstractMemorySegmentImpl bb = checkSegment(obb, encl, base, false);
|
||||
SCOPED_MEMORY_ACCESS.put$RawType$Volatile(bb.sessionImpl(),
|
||||
bb.unsafeGetBase(),
|
||||
offsetNonPlain(bb, base, handle.alignmentMask),
|
||||
offsetNonPlain(bb, base, offset, handle.alignmentMask),
|
||||
convEndian(handle.be, value));
|
||||
}
|
||||
|
||||
@ForceInline
|
||||
static $type$ getAcquire(VarHandle ob, Object obb, long base) {
|
||||
static $type$ getAcquire(VarHandle ob, Object obb, Object encl, long base, long offset) {
|
||||
VarHandleSegmentViewBase handle = (VarHandleSegmentViewBase)ob;
|
||||
AbstractMemorySegmentImpl bb = checkReadOnly(obb, true);
|
||||
AbstractMemorySegmentImpl bb = checkSegment(obb, encl, base, true);
|
||||
return convEndian(handle.be,
|
||||
SCOPED_MEMORY_ACCESS.get$RawType$Acquire(bb.sessionImpl(),
|
||||
bb.unsafeGetBase(),
|
||||
offsetNonPlain(bb, base, handle.alignmentMask)));
|
||||
offsetNonPlain(bb, base, offset, handle.alignmentMask)));
|
||||
}
|
||||
|
||||
@ForceInline
|
||||
static void setRelease(VarHandle ob, Object obb, long base, $type$ value) {
|
||||
static void setRelease(VarHandle ob, Object obb, Object encl, long base, long offset, $type$ value) {
|
||||
VarHandleSegmentViewBase handle = (VarHandleSegmentViewBase)ob;
|
||||
AbstractMemorySegmentImpl bb = checkReadOnly(obb, false);
|
||||
AbstractMemorySegmentImpl bb = checkSegment(obb, encl, base, false);
|
||||
SCOPED_MEMORY_ACCESS.put$RawType$Release(bb.sessionImpl(),
|
||||
bb.unsafeGetBase(),
|
||||
offsetNonPlain(bb, base, handle.alignmentMask),
|
||||
offsetNonPlain(bb, base, offset, handle.alignmentMask),
|
||||
convEndian(handle.be, value));
|
||||
}
|
||||
|
||||
@ForceInline
|
||||
static $type$ getOpaque(VarHandle ob, Object obb, long base) {
|
||||
static $type$ getOpaque(VarHandle ob, Object obb, Object encl, long base, long offset) {
|
||||
VarHandleSegmentViewBase handle = (VarHandleSegmentViewBase)ob;
|
||||
AbstractMemorySegmentImpl bb = checkReadOnly(obb, true);
|
||||
AbstractMemorySegmentImpl bb = checkSegment(obb, encl, base, true);
|
||||
return convEndian(handle.be,
|
||||
SCOPED_MEMORY_ACCESS.get$RawType$Opaque(bb.sessionImpl(),
|
||||
bb.unsafeGetBase(),
|
||||
offsetNonPlain(bb, base, handle.alignmentMask)));
|
||||
offsetNonPlain(bb, base, offset, handle.alignmentMask)));
|
||||
}
|
||||
|
||||
@ForceInline
|
||||
static void setOpaque(VarHandle ob, Object obb, long base, $type$ value) {
|
||||
static void setOpaque(VarHandle ob, Object obb, Object encl, long base, long offset, $type$ value) {
|
||||
VarHandleSegmentViewBase handle = (VarHandleSegmentViewBase)ob;
|
||||
AbstractMemorySegmentImpl bb = checkReadOnly(obb, false);
|
||||
AbstractMemorySegmentImpl bb = checkSegment(obb, encl, base, false);
|
||||
SCOPED_MEMORY_ACCESS.put$RawType$Opaque(bb.sessionImpl(),
|
||||
bb.unsafeGetBase(),
|
||||
offsetNonPlain(bb, base, handle.alignmentMask),
|
||||
offsetNonPlain(bb, base, offset, handle.alignmentMask),
|
||||
convEndian(handle.be, value));
|
||||
}
|
||||
#if[CAS]
|
||||
|
||||
@ForceInline
|
||||
static boolean compareAndSet(VarHandle ob, Object obb, long base, $type$ expected, $type$ value) {
|
||||
static boolean compareAndSet(VarHandle ob, Object obb, Object encl, long base, long offset, $type$ expected, $type$ value) {
|
||||
VarHandleSegmentViewBase handle = (VarHandleSegmentViewBase)ob;
|
||||
AbstractMemorySegmentImpl bb = checkReadOnly(obb, false);
|
||||
AbstractMemorySegmentImpl bb = checkSegment(obb, encl, base, false);
|
||||
return SCOPED_MEMORY_ACCESS.compareAndSet$RawType$(bb.sessionImpl(),
|
||||
bb.unsafeGetBase(),
|
||||
offsetNonPlain(bb, base, handle.alignmentMask),
|
||||
offsetNonPlain(bb, base, offset, handle.alignmentMask),
|
||||
convEndian(handle.be, expected), convEndian(handle.be, value));
|
||||
}
|
||||
|
||||
@ForceInline
|
||||
static $type$ compareAndExchange(VarHandle ob, Object obb, long base, $type$ expected, $type$ value) {
|
||||
static $type$ compareAndExchange(VarHandle ob, Object obb, Object encl, long base, long offset, $type$ expected, $type$ value) {
|
||||
VarHandleSegmentViewBase handle = (VarHandleSegmentViewBase)ob;
|
||||
AbstractMemorySegmentImpl bb = checkReadOnly(obb, false);
|
||||
AbstractMemorySegmentImpl bb = checkSegment(obb, encl, base, false);
|
||||
return convEndian(handle.be,
|
||||
SCOPED_MEMORY_ACCESS.compareAndExchange$RawType$(bb.sessionImpl(),
|
||||
bb.unsafeGetBase(),
|
||||
offsetNonPlain(bb, base, handle.alignmentMask),
|
||||
offsetNonPlain(bb, base, offset, handle.alignmentMask),
|
||||
convEndian(handle.be, expected), convEndian(handle.be, value)));
|
||||
}
|
||||
|
||||
@ForceInline
|
||||
static $type$ compareAndExchangeAcquire(VarHandle ob, Object obb, long base, $type$ expected, $type$ value) {
|
||||
static $type$ compareAndExchangeAcquire(VarHandle ob, Object obb, Object encl, long base, long offset, $type$ expected, $type$ value) {
|
||||
VarHandleSegmentViewBase handle = (VarHandleSegmentViewBase)ob;
|
||||
AbstractMemorySegmentImpl bb = checkReadOnly(obb, false);
|
||||
AbstractMemorySegmentImpl bb = checkSegment(obb, encl, base, false);
|
||||
return convEndian(handle.be,
|
||||
SCOPED_MEMORY_ACCESS.compareAndExchange$RawType$Acquire(bb.sessionImpl(),
|
||||
bb.unsafeGetBase(),
|
||||
offsetNonPlain(bb, base, handle.alignmentMask),
|
||||
offsetNonPlain(bb, base, offset, handle.alignmentMask),
|
||||
convEndian(handle.be, expected), convEndian(handle.be, value)));
|
||||
}
|
||||
|
||||
@ForceInline
|
||||
static $type$ compareAndExchangeRelease(VarHandle ob, Object obb, long base, $type$ expected, $type$ value) {
|
||||
static $type$ compareAndExchangeRelease(VarHandle ob, Object obb, Object encl, long base, long offset, $type$ expected, $type$ value) {
|
||||
VarHandleSegmentViewBase handle = (VarHandleSegmentViewBase)ob;
|
||||
AbstractMemorySegmentImpl bb = checkReadOnly(obb, false);
|
||||
AbstractMemorySegmentImpl bb = checkSegment(obb, encl, base, false);
|
||||
return convEndian(handle.be,
|
||||
SCOPED_MEMORY_ACCESS.compareAndExchange$RawType$Release(bb.sessionImpl(),
|
||||
bb.unsafeGetBase(),
|
||||
offsetNonPlain(bb, base, handle.alignmentMask),
|
||||
offsetNonPlain(bb, base, offset, handle.alignmentMask),
|
||||
convEndian(handle.be, expected), convEndian(handle.be, value)));
|
||||
}
|
||||
|
||||
@ForceInline
|
||||
static boolean weakCompareAndSetPlain(VarHandle ob, Object obb, long base, $type$ expected, $type$ value) {
|
||||
static boolean weakCompareAndSetPlain(VarHandle ob, Object obb, Object encl, long base, long offset, $type$ expected, $type$ value) {
|
||||
VarHandleSegmentViewBase handle = (VarHandleSegmentViewBase)ob;
|
||||
AbstractMemorySegmentImpl bb = checkReadOnly(obb, false);
|
||||
AbstractMemorySegmentImpl bb = checkSegment(obb, encl, base, false);
|
||||
return SCOPED_MEMORY_ACCESS.weakCompareAndSet$RawType$Plain(bb.sessionImpl(),
|
||||
bb.unsafeGetBase(),
|
||||
offsetNonPlain(bb, base, handle.alignmentMask),
|
||||
offsetNonPlain(bb, base, offset, handle.alignmentMask),
|
||||
convEndian(handle.be, expected), convEndian(handle.be, value));
|
||||
}
|
||||
|
||||
@ForceInline
|
||||
static boolean weakCompareAndSet(VarHandle ob, Object obb, long base, $type$ expected, $type$ value) {
|
||||
static boolean weakCompareAndSet(VarHandle ob, Object obb, Object encl, long base, long offset, $type$ expected, $type$ value) {
|
||||
VarHandleSegmentViewBase handle = (VarHandleSegmentViewBase)ob;
|
||||
AbstractMemorySegmentImpl bb = checkReadOnly(obb, false);
|
||||
AbstractMemorySegmentImpl bb = checkSegment(obb, encl, base, false);
|
||||
return SCOPED_MEMORY_ACCESS.weakCompareAndSet$RawType$(bb.sessionImpl(),
|
||||
bb.unsafeGetBase(),
|
||||
offsetNonPlain(bb, base, handle.alignmentMask),
|
||||
offsetNonPlain(bb, base, offset, handle.alignmentMask),
|
||||
convEndian(handle.be, expected), convEndian(handle.be, value));
|
||||
}
|
||||
|
||||
@ForceInline
|
||||
static boolean weakCompareAndSetAcquire(VarHandle ob, Object obb, long base, $type$ expected, $type$ value) {
|
||||
static boolean weakCompareAndSetAcquire(VarHandle ob, Object obb, Object encl, long base, long offset, $type$ expected, $type$ value) {
|
||||
VarHandleSegmentViewBase handle = (VarHandleSegmentViewBase)ob;
|
||||
AbstractMemorySegmentImpl bb = checkReadOnly(obb, false);
|
||||
AbstractMemorySegmentImpl bb = checkSegment(obb, encl, base, false);
|
||||
return SCOPED_MEMORY_ACCESS.weakCompareAndSet$RawType$Acquire(bb.sessionImpl(),
|
||||
bb.unsafeGetBase(),
|
||||
offsetNonPlain(bb, base, handle.alignmentMask),
|
||||
offsetNonPlain(bb, base, offset, handle.alignmentMask),
|
||||
convEndian(handle.be, expected), convEndian(handle.be, value));
|
||||
}
|
||||
|
||||
@ForceInline
|
||||
static boolean weakCompareAndSetRelease(VarHandle ob, Object obb, long base, $type$ expected, $type$ value) {
|
||||
static boolean weakCompareAndSetRelease(VarHandle ob, Object obb, Object encl, long base, long offset, $type$ expected, $type$ value) {
|
||||
VarHandleSegmentViewBase handle = (VarHandleSegmentViewBase)ob;
|
||||
AbstractMemorySegmentImpl bb = checkReadOnly(obb, false);
|
||||
AbstractMemorySegmentImpl bb = checkSegment(obb, encl, base, false);
|
||||
return SCOPED_MEMORY_ACCESS.weakCompareAndSet$RawType$Release(bb.sessionImpl(),
|
||||
bb.unsafeGetBase(),
|
||||
offsetNonPlain(bb, base, handle.alignmentMask),
|
||||
offsetNonPlain(bb, base, offset, handle.alignmentMask),
|
||||
convEndian(handle.be, expected), convEndian(handle.be, value));
|
||||
}
|
||||
|
||||
@ForceInline
|
||||
static $type$ getAndSet(VarHandle ob, Object obb, long base, $type$ value) {
|
||||
static $type$ getAndSet(VarHandle ob, Object obb, Object encl, long base, long offset, $type$ value) {
|
||||
VarHandleSegmentViewBase handle = (VarHandleSegmentViewBase)ob;
|
||||
AbstractMemorySegmentImpl bb = checkReadOnly(obb, false);
|
||||
AbstractMemorySegmentImpl bb = checkSegment(obb, encl, base, false);
|
||||
return convEndian(handle.be,
|
||||
SCOPED_MEMORY_ACCESS.getAndSet$RawType$(bb.sessionImpl(),
|
||||
bb.unsafeGetBase(),
|
||||
offsetNonPlain(bb, base, handle.alignmentMask),
|
||||
offsetNonPlain(bb, base, offset, handle.alignmentMask),
|
||||
convEndian(handle.be, value)));
|
||||
}
|
||||
|
||||
@ForceInline
|
||||
static $type$ getAndSetAcquire(VarHandle ob, Object obb, long base, $type$ value) {
|
||||
static $type$ getAndSetAcquire(VarHandle ob, Object obb, Object encl, long base, long offset, $type$ value) {
|
||||
VarHandleSegmentViewBase handle = (VarHandleSegmentViewBase)ob;
|
||||
AbstractMemorySegmentImpl bb = checkReadOnly(obb, false);
|
||||
AbstractMemorySegmentImpl bb = checkSegment(obb, encl, base, false);
|
||||
return convEndian(handle.be,
|
||||
SCOPED_MEMORY_ACCESS.getAndSet$RawType$Acquire(bb.sessionImpl(),
|
||||
bb.unsafeGetBase(),
|
||||
offsetNonPlain(bb, base, handle.alignmentMask),
|
||||
offsetNonPlain(bb, base, offset, handle.alignmentMask),
|
||||
convEndian(handle.be, value)));
|
||||
}
|
||||
|
||||
@ForceInline
|
||||
static $type$ getAndSetRelease(VarHandle ob, Object obb, long base, $type$ value) {
|
||||
static $type$ getAndSetRelease(VarHandle ob, Object obb, Object encl, long base, long offset, $type$ value) {
|
||||
VarHandleSegmentViewBase handle = (VarHandleSegmentViewBase)ob;
|
||||
AbstractMemorySegmentImpl bb = checkReadOnly(obb, false);
|
||||
AbstractMemorySegmentImpl bb = checkSegment(obb, encl, base, false);
|
||||
return convEndian(handle.be,
|
||||
SCOPED_MEMORY_ACCESS.getAndSet$RawType$Release(bb.sessionImpl(),
|
||||
bb.unsafeGetBase(),
|
||||
offsetNonPlain(bb, base, handle.alignmentMask),
|
||||
offsetNonPlain(bb, base, offset, handle.alignmentMask),
|
||||
convEndian(handle.be, value)));
|
||||
}
|
||||
#end[CAS]
|
||||
#if[AtomicAdd]
|
||||
|
||||
@ForceInline
|
||||
static $type$ getAndAdd(VarHandle ob, Object obb, long base, $type$ delta) {
|
||||
static $type$ getAndAdd(VarHandle ob, Object obb, Object encl, long base, long offset, $type$ delta) {
|
||||
VarHandleSegmentViewBase handle = (VarHandleSegmentViewBase)ob;
|
||||
AbstractMemorySegmentImpl bb = checkReadOnly(obb, false);
|
||||
AbstractMemorySegmentImpl bb = checkSegment(obb, encl, base, false);
|
||||
if (handle.be == BE) {
|
||||
return SCOPED_MEMORY_ACCESS.getAndAdd$RawType$(bb.sessionImpl(),
|
||||
bb.unsafeGetBase(),
|
||||
offsetNonPlain(bb, base, handle.alignmentMask),
|
||||
offsetNonPlain(bb, base, offset, handle.alignmentMask),
|
||||
delta);
|
||||
} else {
|
||||
return getAndAddConvEndianWithCAS(bb, offsetNonPlain(bb, base, handle.alignmentMask), delta);
|
||||
return getAndAddConvEndianWithCAS(bb, offsetNonPlain(bb, base, offset, handle.alignmentMask), delta);
|
||||
}
|
||||
}
|
||||
|
||||
@ForceInline
|
||||
static $type$ getAndAddAcquire(VarHandle ob, Object obb, long base, $type$ delta) {
|
||||
static $type$ getAndAddAcquire(VarHandle ob, Object obb, Object encl, long base, long offset, $type$ delta) {
|
||||
VarHandleSegmentViewBase handle = (VarHandleSegmentViewBase)ob;
|
||||
AbstractMemorySegmentImpl bb = checkReadOnly(obb, false);
|
||||
AbstractMemorySegmentImpl bb = checkSegment(obb, encl, base, false);
|
||||
if (handle.be == BE) {
|
||||
return SCOPED_MEMORY_ACCESS.getAndAdd$RawType$Acquire(bb.sessionImpl(),
|
||||
bb.unsafeGetBase(),
|
||||
offsetNonPlain(bb, base, handle.alignmentMask),
|
||||
offsetNonPlain(bb, base, offset, handle.alignmentMask),
|
||||
delta);
|
||||
} else {
|
||||
return getAndAddConvEndianWithCAS(bb, offsetNonPlain(bb, base, handle.alignmentMask), delta);
|
||||
return getAndAddConvEndianWithCAS(bb, offsetNonPlain(bb, base, offset, handle.alignmentMask), delta);
|
||||
}
|
||||
}
|
||||
|
||||
@ForceInline
|
||||
static $type$ getAndAddRelease(VarHandle ob, Object obb, long base, $type$ delta) {
|
||||
static $type$ getAndAddRelease(VarHandle ob, Object obb, Object encl, long base, long offset, $type$ delta) {
|
||||
VarHandleSegmentViewBase handle = (VarHandleSegmentViewBase)ob;
|
||||
AbstractMemorySegmentImpl bb = checkReadOnly(obb, false);
|
||||
AbstractMemorySegmentImpl bb = checkSegment(obb, encl, base, false);
|
||||
if (handle.be == BE) {
|
||||
return SCOPED_MEMORY_ACCESS.getAndAdd$RawType$Release(bb.sessionImpl(),
|
||||
bb.unsafeGetBase(),
|
||||
offsetNonPlain(bb, base, handle.alignmentMask),
|
||||
offsetNonPlain(bb, base, offset, handle.alignmentMask),
|
||||
delta);
|
||||
} else {
|
||||
return getAndAddConvEndianWithCAS(bb, offsetNonPlain(bb, base, handle.alignmentMask), delta);
|
||||
return getAndAddConvEndianWithCAS(bb, offsetNonPlain(bb, base, offset, handle.alignmentMask), delta);
|
||||
}
|
||||
}
|
||||
|
||||
@ -403,44 +405,44 @@ final class VarHandleSegmentAs$Type$s extends VarHandleSegmentViewBase {
|
||||
#if[Bitwise]
|
||||
|
||||
@ForceInline
|
||||
static $type$ getAndBitwiseOr(VarHandle ob, Object obb, long base, $type$ value) {
|
||||
static $type$ getAndBitwiseOr(VarHandle ob, Object obb, Object encl, long base, long offset, $type$ value) {
|
||||
VarHandleSegmentViewBase handle = (VarHandleSegmentViewBase)ob;
|
||||
AbstractMemorySegmentImpl bb = checkReadOnly(obb, false);
|
||||
AbstractMemorySegmentImpl bb = checkSegment(obb, encl, base, false);
|
||||
if (handle.be == BE) {
|
||||
return SCOPED_MEMORY_ACCESS.getAndBitwiseOr$RawType$(bb.sessionImpl(),
|
||||
bb.unsafeGetBase(),
|
||||
offsetNonPlain(bb, base, handle.alignmentMask),
|
||||
offsetNonPlain(bb, base, offset, handle.alignmentMask),
|
||||
value);
|
||||
} else {
|
||||
return getAndBitwiseOrConvEndianWithCAS(bb, offsetNonPlain(bb, base, handle.alignmentMask), value);
|
||||
return getAndBitwiseOrConvEndianWithCAS(bb, offsetNonPlain(bb, base, offset, handle.alignmentMask), value);
|
||||
}
|
||||
}
|
||||
|
||||
@ForceInline
|
||||
static $type$ getAndBitwiseOrRelease(VarHandle ob, Object obb, long base, $type$ value) {
|
||||
static $type$ getAndBitwiseOrRelease(VarHandle ob, Object obb, Object encl, long base, long offset, $type$ value) {
|
||||
VarHandleSegmentViewBase handle = (VarHandleSegmentViewBase)ob;
|
||||
AbstractMemorySegmentImpl bb = checkReadOnly(obb, false);
|
||||
AbstractMemorySegmentImpl bb = checkSegment(obb, encl, base, false);
|
||||
if (handle.be == BE) {
|
||||
return SCOPED_MEMORY_ACCESS.getAndBitwiseOr$RawType$Release(bb.sessionImpl(),
|
||||
bb.unsafeGetBase(),
|
||||
offsetNonPlain(bb, base, handle.alignmentMask),
|
||||
offsetNonPlain(bb, base, offset, handle.alignmentMask),
|
||||
value);
|
||||
} else {
|
||||
return getAndBitwiseOrConvEndianWithCAS(bb, offsetNonPlain(bb, base, handle.alignmentMask), value);
|
||||
return getAndBitwiseOrConvEndianWithCAS(bb, offsetNonPlain(bb, base, offset, handle.alignmentMask), value);
|
||||
}
|
||||
}
|
||||
|
||||
@ForceInline
|
||||
static $type$ getAndBitwiseOrAcquire(VarHandle ob, Object obb, long base, $type$ value) {
|
||||
static $type$ getAndBitwiseOrAcquire(VarHandle ob, Object obb, Object encl, long base, long offset, $type$ value) {
|
||||
VarHandleSegmentViewBase handle = (VarHandleSegmentViewBase)ob;
|
||||
AbstractMemorySegmentImpl bb = checkReadOnly(obb, false);
|
||||
AbstractMemorySegmentImpl bb = checkSegment(obb, encl, base, false);
|
||||
if (handle.be == BE) {
|
||||
return SCOPED_MEMORY_ACCESS.getAndBitwiseOr$RawType$Acquire(bb.sessionImpl(),
|
||||
bb.unsafeGetBase(),
|
||||
offsetNonPlain(bb, base, handle.alignmentMask),
|
||||
offsetNonPlain(bb, base, offset, handle.alignmentMask),
|
||||
value);
|
||||
} else {
|
||||
return getAndBitwiseOrConvEndianWithCAS(bb, offsetNonPlain(bb, base, handle.alignmentMask), value);
|
||||
return getAndBitwiseOrConvEndianWithCAS(bb, offsetNonPlain(bb, base, offset, handle.alignmentMask), value);
|
||||
}
|
||||
}
|
||||
|
||||
@ -457,44 +459,44 @@ final class VarHandleSegmentAs$Type$s extends VarHandleSegmentViewBase {
|
||||
}
|
||||
|
||||
@ForceInline
|
||||
static $type$ getAndBitwiseAnd(VarHandle ob, Object obb, long base, $type$ value) {
|
||||
static $type$ getAndBitwiseAnd(VarHandle ob, Object obb, Object encl, long base, long offset, $type$ value) {
|
||||
VarHandleSegmentViewBase handle = (VarHandleSegmentViewBase)ob;
|
||||
AbstractMemorySegmentImpl bb = checkReadOnly(obb, false);
|
||||
AbstractMemorySegmentImpl bb = checkSegment(obb, encl, base, false);
|
||||
if (handle.be == BE) {
|
||||
return SCOPED_MEMORY_ACCESS.getAndBitwiseAnd$RawType$(bb.sessionImpl(),
|
||||
bb.unsafeGetBase(),
|
||||
offsetNonPlain(bb, base, handle.alignmentMask),
|
||||
offsetNonPlain(bb, base, offset, handle.alignmentMask),
|
||||
value);
|
||||
} else {
|
||||
return getAndBitwiseAndConvEndianWithCAS(bb, offsetNonPlain(bb, base, handle.alignmentMask), value);
|
||||
return getAndBitwiseAndConvEndianWithCAS(bb, offsetNonPlain(bb, base, offset, handle.alignmentMask), value);
|
||||
}
|
||||
}
|
||||
|
||||
@ForceInline
|
||||
static $type$ getAndBitwiseAndRelease(VarHandle ob, Object obb, long base, $type$ value) {
|
||||
static $type$ getAndBitwiseAndRelease(VarHandle ob, Object obb, Object encl, long base, long offset, $type$ value) {
|
||||
VarHandleSegmentViewBase handle = (VarHandleSegmentViewBase)ob;
|
||||
AbstractMemorySegmentImpl bb = checkReadOnly(obb, false);
|
||||
AbstractMemorySegmentImpl bb = checkSegment(obb, encl, base, false);
|
||||
if (handle.be == BE) {
|
||||
return SCOPED_MEMORY_ACCESS.getAndBitwiseAnd$RawType$Release(bb.sessionImpl(),
|
||||
bb.unsafeGetBase(),
|
||||
offsetNonPlain(bb, base, handle.alignmentMask),
|
||||
offsetNonPlain(bb, base, offset, handle.alignmentMask),
|
||||
value);
|
||||
} else {
|
||||
return getAndBitwiseAndConvEndianWithCAS(bb, offsetNonPlain(bb, base, handle.alignmentMask), value);
|
||||
return getAndBitwiseAndConvEndianWithCAS(bb, offsetNonPlain(bb, base, offset, handle.alignmentMask), value);
|
||||
}
|
||||
}
|
||||
|
||||
@ForceInline
|
||||
static $type$ getAndBitwiseAndAcquire(VarHandle ob, Object obb, long base, $type$ value) {
|
||||
static $type$ getAndBitwiseAndAcquire(VarHandle ob, Object obb, Object encl, long base, long offset, $type$ value) {
|
||||
VarHandleSegmentViewBase handle = (VarHandleSegmentViewBase)ob;
|
||||
AbstractMemorySegmentImpl bb = checkReadOnly(obb, false);
|
||||
AbstractMemorySegmentImpl bb = checkSegment(obb, encl, base, false);
|
||||
if (handle.be == BE) {
|
||||
return SCOPED_MEMORY_ACCESS.getAndBitwiseAnd$RawType$Acquire(bb.sessionImpl(),
|
||||
bb.unsafeGetBase(),
|
||||
offsetNonPlain(bb, base, handle.alignmentMask),
|
||||
offsetNonPlain(bb, base, offset, handle.alignmentMask),
|
||||
value);
|
||||
} else {
|
||||
return getAndBitwiseAndConvEndianWithCAS(bb, offsetNonPlain(bb, base, handle.alignmentMask), value);
|
||||
return getAndBitwiseAndConvEndianWithCAS(bb, offsetNonPlain(bb, base, offset, handle.alignmentMask), value);
|
||||
}
|
||||
}
|
||||
|
||||
@ -512,44 +514,44 @@ final class VarHandleSegmentAs$Type$s extends VarHandleSegmentViewBase {
|
||||
|
||||
|
||||
@ForceInline
|
||||
static $type$ getAndBitwiseXor(VarHandle ob, Object obb, long base, $type$ value) {
|
||||
static $type$ getAndBitwiseXor(VarHandle ob, Object obb, Object encl, long base, long offset, $type$ value) {
|
||||
VarHandleSegmentViewBase handle = (VarHandleSegmentViewBase)ob;
|
||||
AbstractMemorySegmentImpl bb = checkReadOnly(obb, false);
|
||||
AbstractMemorySegmentImpl bb = checkSegment(obb, encl, base, false);
|
||||
if (handle.be == BE) {
|
||||
return SCOPED_MEMORY_ACCESS.getAndBitwiseXor$RawType$(bb.sessionImpl(),
|
||||
bb.unsafeGetBase(),
|
||||
offsetNonPlain(bb, base, handle.alignmentMask),
|
||||
offsetNonPlain(bb, base, offset, handle.alignmentMask),
|
||||
value);
|
||||
} else {
|
||||
return getAndBitwiseXorConvEndianWithCAS(bb, offsetNonPlain(bb, base, handle.alignmentMask), value);
|
||||
return getAndBitwiseXorConvEndianWithCAS(bb, offsetNonPlain(bb, base, offset, handle.alignmentMask), value);
|
||||
}
|
||||
}
|
||||
|
||||
@ForceInline
|
||||
static $type$ getAndBitwiseXorRelease(VarHandle ob, Object obb, long base, $type$ value) {
|
||||
static $type$ getAndBitwiseXorRelease(VarHandle ob, Object obb, Object encl, long base, long offset, $type$ value) {
|
||||
VarHandleSegmentViewBase handle = (VarHandleSegmentViewBase)ob;
|
||||
AbstractMemorySegmentImpl bb = checkReadOnly(obb, false);
|
||||
AbstractMemorySegmentImpl bb = checkSegment(obb, encl, base, false);
|
||||
if (handle.be == BE) {
|
||||
return SCOPED_MEMORY_ACCESS.getAndBitwiseXor$RawType$Release(bb.sessionImpl(),
|
||||
bb.unsafeGetBase(),
|
||||
offsetNonPlain(bb, base, handle.alignmentMask),
|
||||
offsetNonPlain(bb, base, offset, handle.alignmentMask),
|
||||
value);
|
||||
} else {
|
||||
return getAndBitwiseXorConvEndianWithCAS(bb, offsetNonPlain(bb, base, handle.alignmentMask), value);
|
||||
return getAndBitwiseXorConvEndianWithCAS(bb, offsetNonPlain(bb, base, offset, handle.alignmentMask), value);
|
||||
}
|
||||
}
|
||||
|
||||
@ForceInline
|
||||
static $type$ getAndBitwiseXorAcquire(VarHandle ob, Object obb, long base, $type$ value) {
|
||||
static $type$ getAndBitwiseXorAcquire(VarHandle ob, Object obb, Object encl, long base, long offset, $type$ value) {
|
||||
VarHandleSegmentViewBase handle = (VarHandleSegmentViewBase)ob;
|
||||
AbstractMemorySegmentImpl bb = checkReadOnly(obb, false);
|
||||
AbstractMemorySegmentImpl bb = checkSegment(obb, encl, base, false);
|
||||
if (handle.be == BE) {
|
||||
return SCOPED_MEMORY_ACCESS.getAndBitwiseXor$RawType$Acquire(bb.sessionImpl(),
|
||||
bb.unsafeGetBase(),
|
||||
offsetNonPlain(bb, base, handle.alignmentMask),
|
||||
offsetNonPlain(bb, base, offset, handle.alignmentMask),
|
||||
value);
|
||||
} else {
|
||||
return getAndBitwiseXorConvEndianWithCAS(bb, offsetNonPlain(bb, base, handle.alignmentMask), value);
|
||||
return getAndBitwiseXorConvEndianWithCAS(bb, offsetNonPlain(bb, base, offset, handle.alignmentMask), value);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -373,6 +373,16 @@ public abstract sealed class AbstractMemorySegmentImpl
|
||||
sessionImpl().checkValidState();
|
||||
}
|
||||
|
||||
@ForceInline
|
||||
public final void checkEnclosingLayout(long offset, MemoryLayout enclosing, boolean readOnly) {
|
||||
checkAccess(offset, enclosing.byteSize(), readOnly);
|
||||
if (!isAlignedForElement(offset, enclosing)) {
|
||||
throw new IllegalArgumentException(String.format(
|
||||
"Target offset %d is incompatible with alignment constraint %d (of %s) for segment %s"
|
||||
, offset, enclosing.byteAlignment(), enclosing, this));
|
||||
}
|
||||
}
|
||||
|
||||
public abstract long unsafeGetOffset();
|
||||
|
||||
public abstract Object unsafeGetBase();
|
||||
|
||||
@ -80,7 +80,7 @@ public class LayoutPath {
|
||||
MH_CHECK_ENCL_LAYOUT = lookup.findStatic(LayoutPath.class, "checkEnclosingLayout",
|
||||
MethodType.methodType(void.class, MemorySegment.class, long.class, MemoryLayout.class));
|
||||
MH_SEGMENT_RESIZE = lookup.findStatic(LayoutPath.class, "resizeSegment",
|
||||
MethodType.methodType(MemorySegment.class, MemorySegment.class, MemoryLayout.class));
|
||||
MethodType.methodType(MemorySegment.class, MemorySegment.class));
|
||||
MH_ADD = lookup.findStatic(Long.class, "sum",
|
||||
MethodType.methodType(long.class, long.class, long.class));
|
||||
} catch (Throwable ex) {
|
||||
@ -180,13 +180,14 @@ public class LayoutPath {
|
||||
}
|
||||
MemoryLayout derefLayout = addressLayout.targetLayout().get();
|
||||
MethodHandle handle = dereferenceHandle(false).toMethodHandle(VarHandle.AccessMode.GET);
|
||||
handle = MethodHandles.filterReturnValue(handle,
|
||||
MethodHandles.insertArguments(MH_SEGMENT_RESIZE, 1, derefLayout));
|
||||
handle = MethodHandles.filterReturnValue(handle, MH_SEGMENT_RESIZE);
|
||||
return derefPath(derefLayout, handle, this);
|
||||
}
|
||||
|
||||
private static MemorySegment resizeSegment(MemorySegment segment, MemoryLayout layout) {
|
||||
return Utils.longToAddress(segment.address(), layout.byteSize(), layout.byteAlignment());
|
||||
private static MemorySegment resizeSegment(MemorySegment segment) {
|
||||
// Avoid adapting for specific target layout. The check for the root layout
|
||||
// size and alignment will be inserted by LayoutPath::dereferenceHandle anyway.
|
||||
return Utils.longToAddress(segment.address(), Long.MAX_VALUE, 1);
|
||||
}
|
||||
|
||||
// Layout path projections
|
||||
@ -205,19 +206,15 @@ public class LayoutPath {
|
||||
String.format("Path does not select a value layout: %s", breadcrumbs()));
|
||||
}
|
||||
|
||||
VarHandle handle = Utils.makeRawSegmentViewVarHandle(valueLayout);
|
||||
handle = MethodHandles.collectCoordinates(handle, 1, offsetHandle());
|
||||
|
||||
// we only have to check the alignment of the root layout for the first dereference we do,
|
||||
// as each dereference checks the alignment of the target address when constructing its segment
|
||||
// (see Utils::longToAddress)
|
||||
if (derefAdapters.length == 0) {
|
||||
// insert align check for the root layout on the initial MS + offset
|
||||
List<Class<?>> coordinateTypes = handle.coordinateTypes();
|
||||
MethodHandle alignCheck = MethodHandles.insertArguments(MH_CHECK_ENCL_LAYOUT, 2, rootLayout());
|
||||
handle = MethodHandles.collectCoordinates(handle, 0, alignCheck);
|
||||
int[] reorder = IntStream.concat(IntStream.of(0, 1), IntStream.range(0, coordinateTypes.size())).toArray();
|
||||
handle = MethodHandles.permuteCoordinates(handle, coordinateTypes, reorder);
|
||||
VarHandle handle = Utils.makeRawSegmentViewVarHandle(valueLayout); // (MS, ML, long, long)
|
||||
handle = MethodHandles.insertCoordinates(handle, 1, rootLayout()); // (MS, long, long)
|
||||
if (strides.length > 0) {
|
||||
MethodHandle offsetAdapter = offsetHandle();
|
||||
offsetAdapter = MethodHandles.insertArguments(offsetAdapter, 0, 0L);
|
||||
handle = MethodHandles.collectCoordinates(handle, 2, offsetAdapter); // (MS, long)
|
||||
} else {
|
||||
// simpler adaptation
|
||||
handle = MethodHandles.insertCoordinates(handle, 2, offset); // (MS, long)
|
||||
}
|
||||
|
||||
if (adapt) {
|
||||
@ -241,6 +238,8 @@ public class LayoutPath {
|
||||
@ForceInline
|
||||
private static long addScaledOffset(long base, long index, long stride, long bound) {
|
||||
Objects.checkIndex(index, bound);
|
||||
// note: the below can overflow, depending on 'base'. When constructing var handles
|
||||
// through the layout API, this is never the case, as the injected 'base' is always 0.
|
||||
return base + (stride * index);
|
||||
}
|
||||
|
||||
@ -285,12 +284,7 @@ public class LayoutPath {
|
||||
}
|
||||
|
||||
private static void checkEnclosingLayout(MemorySegment segment, long offset, MemoryLayout enclosing) {
|
||||
((AbstractMemorySegmentImpl)segment).checkAccess(offset, enclosing.byteSize(), true);
|
||||
if (!((AbstractMemorySegmentImpl) segment).isAlignedForElement(offset, enclosing)) {
|
||||
throw new IllegalArgumentException(String.format(
|
||||
"Target offset %d is incompatible with alignment constraint %d (of %s) for segment %s"
|
||||
, offset, enclosing.byteAlignment(), enclosing, segment));
|
||||
}
|
||||
((AbstractMemorySegmentImpl)segment).checkEnclosingLayout(offset, enclosing, true);
|
||||
}
|
||||
|
||||
public MemoryLayout layout() {
|
||||
|
||||
@ -63,7 +63,8 @@ public final class Utils {
|
||||
private static final MethodHandle BYTE_TO_BOOL;
|
||||
private static final MethodHandle BOOL_TO_BYTE;
|
||||
private static final MethodHandle ADDRESS_TO_LONG;
|
||||
private static final MethodHandle LONG_TO_ADDRESS;
|
||||
private static final MethodHandle LONG_TO_ADDRESS_TARGET;
|
||||
private static final MethodHandle LONG_TO_ADDRESS_NO_TARGET;
|
||||
|
||||
static {
|
||||
try {
|
||||
@ -74,8 +75,10 @@ public final class Utils {
|
||||
MethodType.methodType(byte.class, boolean.class));
|
||||
ADDRESS_TO_LONG = lookup.findStatic(SharedUtils.class, "unboxSegment",
|
||||
MethodType.methodType(long.class, MemorySegment.class));
|
||||
LONG_TO_ADDRESS = lookup.findStatic(Utils.class, "longToAddress",
|
||||
MethodType.methodType(MemorySegment.class, long.class, long.class, long.class));
|
||||
LONG_TO_ADDRESS_TARGET = lookup.findStatic(Utils.class, "longToAddress",
|
||||
MethodType.methodType(MemorySegment.class, long.class, AddressLayout.class));
|
||||
LONG_TO_ADDRESS_NO_TARGET = lookup.findStatic(Utils.class, "longToAddress",
|
||||
MethodType.methodType(MemorySegment.class, long.class));
|
||||
} catch (Throwable ex) {
|
||||
throw new ExceptionInInitializerError(ex);
|
||||
}
|
||||
@ -129,11 +132,10 @@ public final class Utils {
|
||||
if (layout.carrier() == boolean.class) {
|
||||
handle = MethodHandles.filterValue(handle, BOOL_TO_BYTE, BYTE_TO_BOOL);
|
||||
} else if (layout instanceof AddressLayout addressLayout) {
|
||||
handle = MethodHandles.filterValue(handle,
|
||||
MethodHandles.explicitCastArguments(ADDRESS_TO_LONG, MethodType.methodType(baseCarrier, MemorySegment.class)),
|
||||
MethodHandles.explicitCastArguments(MethodHandles.insertArguments(LONG_TO_ADDRESS, 1,
|
||||
pointeeByteSize(addressLayout), pointeeByteAlign(addressLayout)),
|
||||
MethodType.methodType(MemorySegment.class, baseCarrier)));
|
||||
MethodHandle longToAddressAdapter = addressLayout.targetLayout().isPresent() ?
|
||||
MethodHandles.insertArguments(LONG_TO_ADDRESS_TARGET, 1, addressLayout) :
|
||||
LONG_TO_ADDRESS_NO_TARGET;
|
||||
handle = MethodHandles.filterValue(handle, ADDRESS_TO_LONG, longToAddressAdapter);
|
||||
}
|
||||
return handle;
|
||||
}
|
||||
@ -146,6 +148,16 @@ public final class Utils {
|
||||
return b ? (byte)1 : (byte)0;
|
||||
}
|
||||
|
||||
@ForceInline
|
||||
public static MemorySegment longToAddress(long addr) {
|
||||
return longToAddress(addr, 0, 1);
|
||||
}
|
||||
|
||||
@ForceInline
|
||||
public static MemorySegment longToAddress(long addr, AddressLayout layout) {
|
||||
return longToAddress(addr, pointeeByteSize(layout), pointeeByteAlign(layout));
|
||||
}
|
||||
|
||||
@ForceInline
|
||||
public static MemorySegment longToAddress(long addr, long size, long align) {
|
||||
if (!isAligned(addr, align)) {
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user