From 9583e3657e43cc1c6f2101a64534564db2a9bd84 Mon Sep 17 00:00:00 2001 From: Alan Bateman Date: Sat, 7 May 2022 08:06:16 +0000 Subject: [PATCH] 8284161: Implementation of Virtual Threads (Preview) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Ron Pressler Co-authored-by: Alan Bateman Co-authored-by: Erik Österlund Co-authored-by: Andrew Haley Co-authored-by: Rickard Bäckman Co-authored-by: Markus Grönlund Co-authored-by: Leonid Mesnik Co-authored-by: Serguei Spitsyn Co-authored-by: Chris Plummer Co-authored-by: Coleen Phillimore Co-authored-by: Robbin Ehn Co-authored-by: Stefan Karlsson Co-authored-by: Thomas Schatzl Co-authored-by: Sergey Kuksenko Reviewed-by: lancea, eosterlund, rehn, sspitsyn, stefank, tschatzl, dfuchs, lmesnik, dcubed, kevinw, amenkov, dlong, mchung, psandoz, bpb, coleenp, smarks, egahlin, mseledtsov, coffeys, darcy --- make/data/hotspot-symbols/symbols-unix | 16 + make/test/BuildMicrobenchmark.gmk | 8 +- make/test/JtregNativeHotspot.gmk | 10 +- src/hotspot/cpu/aarch64/aarch64.ad | 8 +- .../aarch64/abstractInterpreter_aarch64.cpp | 29 +- .../cpu/aarch64/c1_LIRAssembler_aarch64.cpp | 15 + .../cpu/aarch64/c1_LIRGenerator_aarch64.cpp | 13 +- .../continuationEntry_aarch64.inline.hpp | 51 + .../continuationFreezeThaw_aarch64.inline.hpp | 301 ++ .../continuationHelper_aarch64.inline.hpp | 140 + src/hotspot/cpu/aarch64/frame_aarch64.cpp | 171 +- src/hotspot/cpu/aarch64/frame_aarch64.hpp | 39 +- .../cpu/aarch64/frame_aarch64.inline.hpp | 262 +- .../gc/g1/g1BarrierSetAssembler_aarch64.cpp | 8 +- .../gc/shared/barrierSetAssembler_aarch64.cpp | 60 +- .../gc/shared/barrierSetAssembler_aarch64.hpp | 7 +- .../gc/shared/barrierSetNMethod_aarch64.cpp | 88 +- .../shenandoahBarrierSetAssembler_aarch64.hpp | 2 + .../gc/z/zBarrierSetAssembler_aarch64.hpp | 4 +- .../cpu/aarch64/interp_masm_aarch64.cpp | 2 + .../cpu/aarch64/javaFrameAnchor_aarch64.hpp | 4 +- .../cpu/aarch64/macroAssembler_aarch64.cpp | 74 +- .../cpu/aarch64/macroAssembler_aarch64.hpp | 16 +- .../cpu/aarch64/nativeInst_aarch64.cpp | 26 + .../cpu/aarch64/nativeInst_aarch64.hpp | 50 +- .../cpu/aarch64/registerMap_aarch64.cpp | 6 +- .../cpu/aarch64/sharedRuntime_aarch64.cpp | 125 +- .../smallRegisterMap_aarch64.inline.hpp | 91 + .../stackChunkFrameStream_aarch64.inline.hpp | 149 + .../aarch64/stackChunkOop_aarch64.inline.hpp | 42 + .../cpu/aarch64/stubGenerator_aarch64.cpp | 386 ++- .../templateInterpreterGenerator_aarch64.cpp | 16 + .../cpu/aarch64/templateTable_aarch64.cpp | 4 + src/hotspot/cpu/arm/c1_LIRGenerator_arm.cpp | 6 +- .../cpu/arm/continuationEntry_arm.inline.hpp | 47 + .../arm/continuationFreezeThaw_arm.inline.hpp | 98 + .../cpu/arm/continuationHelper_arm.inline.hpp | 125 + src/hotspot/cpu/arm/frame_arm.cpp | 63 +- src/hotspot/cpu/arm/frame_arm.hpp | 13 +- src/hotspot/cpu/arm/frame_arm.inline.hpp | 110 +- src/hotspot/cpu/arm/nativeInst_arm_32.hpp | 33 +- src/hotspot/cpu/arm/registerMap_arm.hpp | 2 +- .../cpu/arm/smallRegisterMap_arm.inline.hpp | 87 + .../arm/stackChunkFrameStream_arm.inline.hpp | 115 + .../cpu/arm/stackChunkOop_arm.inline.hpp | 36 + src/hotspot/cpu/arm/stubGenerator_arm.cpp | 4 +- .../arm/templateInterpreterGenerator_arm.cpp | 5 + src/hotspot/cpu/ppc/assembler_ppc.hpp | 3 + src/hotspot/cpu/ppc/c1_LIRGenerator_ppc.cpp | 6 +- .../cpu/ppc/continuationEntry_ppc.inline.hpp | 46 + .../ppc/continuationFreezeThaw_ppc.inline.hpp | 99 + .../cpu/ppc/continuationHelper_ppc.inline.hpp | 124 + src/hotspot/cpu/ppc/frame_ppc.cpp | 61 +- src/hotspot/cpu/ppc/frame_ppc.hpp | 18 +- src/hotspot/cpu/ppc/frame_ppc.inline.hpp | 133 +- .../ppc/gc/shared/barrierSetNMethod_ppc.cpp | 6 +- src/hotspot/cpu/ppc/nativeInst_ppc.hpp | 33 +- src/hotspot/cpu/ppc/registerMap_ppc.hpp | 2 +- .../cpu/ppc/smallRegisterMap_ppc.inline.hpp | 87 + .../ppc/stackChunkFrameStream_ppc.inline.hpp | 115 + .../cpu/ppc/stackChunkOop_ppc.inline.hpp | 36 + src/hotspot/cpu/ppc/stubGenerator_ppc.cpp | 4 +- .../ppc/templateInterpreterGenerator_ppc.cpp | 5 + .../cpu/riscv/c1_LIRGenerator_riscv.cpp | 6 +- .../riscv/continuationEntry_riscv.inline.hpp | 46 + .../continuationFreezeThaw_riscv.inline.hpp | 99 + .../riscv/continuationHelper_riscv.inline.hpp | 124 + src/hotspot/cpu/riscv/frame_riscv.cpp | 118 +- src/hotspot/cpu/riscv/frame_riscv.hpp | 21 +- src/hotspot/cpu/riscv/frame_riscv.inline.hpp | 169 +- .../gc/shared/barrierSetNMethod_riscv.cpp | 9 +- src/hotspot/cpu/riscv/nativeInst_riscv.hpp | 33 +- src/hotspot/cpu/riscv/registerMap_riscv.cpp | 8 +- .../riscv/smallRegisterMap_riscv.inline.hpp | 87 + .../stackChunkFrameStream_riscv.inline.hpp | 115 + .../cpu/riscv/stackChunkOop_riscv.inline.hpp | 36 + src/hotspot/cpu/riscv/stubGenerator_riscv.cpp | 4 +- .../templateInterpreterGenerator_riscv.cpp | 7 +- src/hotspot/cpu/s390/assembler_s390.hpp | 5 +- src/hotspot/cpu/s390/c1_LIRGenerator_s390.cpp | 4 + .../s390/continuationEntry_s390.inline.hpp | 46 + .../continuationFreezeThaw_s390.inline.hpp | 98 + .../s390/continuationHelper_s390.inline.hpp | 125 + src/hotspot/cpu/s390/frame_s390.cpp | 51 +- src/hotspot/cpu/s390/frame_s390.hpp | 12 +- src/hotspot/cpu/s390/frame_s390.inline.hpp | 130 +- src/hotspot/cpu/s390/nativeInst_s390.hpp | 33 +- src/hotspot/cpu/s390/registerMap_s390.hpp | 4 +- .../cpu/s390/smallRegisterMap_s390.inline.hpp | 87 + .../stackChunkFrameStream_s390.inline.hpp | 115 + .../cpu/s390/stackChunkOop_s390.inline.hpp | 36 + src/hotspot/cpu/s390/stubGenerator_s390.cpp | 4 +- .../templateInterpreterGenerator_s390.cpp | 5 + src/hotspot/cpu/x86/assembler_x86.hpp | 4 +- src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp | 38 +- src/hotspot/cpu/x86/c1_LIRGenerator_x86.cpp | 12 +- .../cpu/x86/continuationEntry_x86.inline.hpp | 50 + .../x86/continuationFreezeThaw_x86.inline.hpp | 289 ++ .../cpu/x86/continuationHelper_x86.inline.hpp | 139 + src/hotspot/cpu/x86/frame_x86.cpp | 171 +- src/hotspot/cpu/x86/frame_x86.hpp | 44 +- src/hotspot/cpu/x86/frame_x86.inline.hpp | 277 +- .../x86/gc/g1/g1BarrierSetAssembler_x86.cpp | 38 +- .../x86/gc/shared/barrierSetNMethod_x86.cpp | 11 +- src/hotspot/cpu/x86/globals_x86.hpp | 4 +- src/hotspot/cpu/x86/interp_masm_x86.cpp | 6 + src/hotspot/cpu/x86/javaFrameAnchor_x86.hpp | 4 +- src/hotspot/cpu/x86/macroAssembler_x86.cpp | 123 +- src/hotspot/cpu/x86/macroAssembler_x86.hpp | 20 + src/hotspot/cpu/x86/nativeInst_x86.cpp | 39 +- src/hotspot/cpu/x86/nativeInst_x86.hpp | 56 +- src/hotspot/cpu/x86/registerMap_x86.cpp | 6 +- src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp | 137 +- .../cpu/x86/smallRegisterMap_x86.inline.hpp | 91 + .../x86/stackChunkFrameStream_x86.inline.hpp | 149 + .../cpu/x86/stackChunkOop_x86.inline.hpp | 42 + src/hotspot/cpu/x86/stubGenerator_x86_32.cpp | 6 +- src/hotspot/cpu/x86/stubGenerator_x86_64.cpp | 353 ++- src/hotspot/cpu/x86/stubRoutines_x86.hpp | 6 +- .../x86/templateInterpreterGenerator_x86.cpp | 40 +- .../templateInterpreterGenerator_x86_64.cpp | 4 +- src/hotspot/cpu/x86/templateTable_x86.cpp | 13 + src/hotspot/cpu/x86/vm_version_x86.hpp | 29 +- src/hotspot/cpu/x86/x86.ad | 2 +- src/hotspot/cpu/x86/x86_32.ad | 23 +- src/hotspot/cpu/x86/x86_64.ad | 38 +- .../zero/continuationEntry_zero.inline.hpp | 47 + .../continuationFreezeThaw_zero.inline.hpp | 98 + .../zero/continuationHelper_zero.inline.hpp | 123 + src/hotspot/cpu/zero/frame_zero.cpp | 26 +- src/hotspot/cpu/zero/frame_zero.hpp | 17 +- src/hotspot/cpu/zero/frame_zero.inline.hpp | 83 +- src/hotspot/cpu/zero/nativeInst_zero.hpp | 34 +- src/hotspot/cpu/zero/registerMap_zero.hpp | 2 +- .../cpu/zero/smallRegisterMap_zero.inline.hpp | 87 + .../stackChunkFrameStream_zero.inline.hpp | 115 + .../cpu/zero/stackChunkOop_zero.inline.hpp | 36 + src/hotspot/cpu/zero/stubGenerator_zero.cpp | 6 +- src/hotspot/os/posix/os_posix.cpp | 3 +- src/hotspot/os/posix/signals_posix.cpp | 28 + src/hotspot/os/windows/os_windows.cpp | 20 + src/hotspot/share/c1/c1_CodeStubs.hpp | 10 +- src/hotspot/share/c1/c1_Compilation.cpp | 11 +- src/hotspot/share/c1/c1_Compilation.hpp | 6 +- src/hotspot/share/c1/c1_Compiler.cpp | 5 +- src/hotspot/share/c1/c1_GraphBuilder.cpp | 1 + src/hotspot/share/c1/c1_LIRAssembler.cpp | 2 + src/hotspot/share/c1/c1_LIRAssembler.hpp | 2 + src/hotspot/share/c1/c1_LIRGenerator.cpp | 66 +- src/hotspot/share/c1/c1_LIRGenerator.hpp | 6 +- src/hotspot/share/c1/c1_Runtime1.cpp | 5 +- src/hotspot/share/cds/cppVtables.cpp | 4 +- src/hotspot/share/cds/heapShared.cpp | 2 + src/hotspot/share/ci/ciEnv.cpp | 11 + src/hotspot/share/ci/ciEnv.hpp | 4 + src/hotspot/share/ci/ciMethod.hpp | 13 +- .../share/classfile/classFileParser.cpp | 16 + src/hotspot/share/classfile/javaClasses.cpp | 621 ++++- src/hotspot/share/classfile/javaClasses.hpp | 302 +- .../share/classfile/javaClasses.inline.hpp | 187 +- src/hotspot/share/classfile/vmClassMacros.hpp | 8 + src/hotspot/share/classfile/vmClasses.cpp | 3 + src/hotspot/share/classfile/vmIntrinsics.cpp | 15 +- src/hotspot/share/classfile/vmIntrinsics.hpp | 22 + src/hotspot/share/classfile/vmSymbols.hpp | 70 +- src/hotspot/share/code/codeBlob.cpp | 10 +- src/hotspot/share/code/codeBlob.hpp | 23 +- src/hotspot/share/code/codeBlob.inline.hpp | 39 + src/hotspot/share/code/codeCache.cpp | 56 +- src/hotspot/share/code/codeCache.hpp | 81 +- src/hotspot/share/code/codeCache.inline.hpp | 59 + src/hotspot/share/code/compiledIC.hpp | 6 +- src/hotspot/share/code/compiledMethod.cpp | 70 +- src/hotspot/share/code/compiledMethod.hpp | 39 +- src/hotspot/share/code/nmethod.cpp | 156 +- src/hotspot/share/code/nmethod.hpp | 18 +- src/hotspot/share/code/relocInfo.hpp | 19 +- src/hotspot/share/compiler/compilerOracle.hpp | 3 + src/hotspot/share/compiler/oopMap.cpp | 541 ++-- src/hotspot/share/compiler/oopMap.hpp | 89 +- src/hotspot/share/compiler/oopMap.inline.hpp | 157 ++ src/hotspot/share/gc/epsilon/epsilonHeap.hpp | 2 + src/hotspot/share/gc/g1/g1BarrierSet.cpp | 3 +- src/hotspot/share/gc/g1/g1CodeBlobClosure.cpp | 27 +- src/hotspot/share/gc/g1/g1CollectedHeap.cpp | 17 + src/hotspot/share/gc/g1/g1CollectedHeap.hpp | 5 +- .../share/gc/g1/g1CollectedHeap.inline.hpp | 7 +- src/hotspot/share/gc/g1/g1ConcurrentMark.cpp | 8 +- src/hotspot/share/gc/g1/g1FullCollector.cpp | 6 + .../share/gc/g1/g1FullGCCompactTask.cpp | 4 +- src/hotspot/share/gc/g1/g1FullGCMarkTask.cpp | 4 +- .../share/gc/g1/g1FullGCMarker.inline.hpp | 3 + .../share/gc/g1/g1FullGCOopClosures.hpp | 4 +- .../gc/g1/g1FullGCOopClosures.inline.hpp | 10 +- .../share/gc/g1/g1ParScanThreadState.cpp | 6 + src/hotspot/share/gc/g1/g1Policy.cpp | 8 +- .../gc/parallel/parallelScavengeHeap.cpp | 6 +- .../gc/parallel/parallelScavengeHeap.hpp | 4 +- .../share/gc/parallel/psParallelCompact.cpp | 11 +- .../gc/parallel/psParallelCompact.inline.hpp | 4 +- .../share/gc/parallel/psPromotionManager.cpp | 3 + .../gc/parallel/psPromotionManager.inline.hpp | 6 + src/hotspot/share/gc/parallel/psScavenge.cpp | 6 +- .../share/gc/serial/defNewGeneration.cpp | 8 +- src/hotspot/share/gc/serial/markSweep.hpp | 6 +- .../share/gc/serial/markSweep.inline.hpp | 9 +- src/hotspot/share/gc/serial/serialHeap.cpp | 4 +- src/hotspot/share/gc/shared/barrierSet.cpp | 36 +- src/hotspot/share/gc/shared/barrierSet.hpp | 11 +- .../share/gc/shared/barrierSetNMethod.cpp | 67 + .../share/gc/shared/barrierSetNMethod.hpp | 17 +- src/hotspot/share/gc/shared/collectedHeap.cpp | 2 + src/hotspot/share/gc/shared/collectedHeap.hpp | 15 + .../share/gc/shared/continuationGCSupport.hpp | 40 + .../shared/continuationGCSupport.inline.hpp | 56 + .../share/gc/shared/copyFailedInfo.hpp | 4 +- src/hotspot/share/gc/shared/gcCause.cpp | 5 +- src/hotspot/share/gc/shared/gcCause.hpp | 1 + .../share/gc/shared/genCollectedHeap.cpp | 34 +- .../share/gc/shared/genCollectedHeap.hpp | 6 +- .../share/gc/shared/genOopClosures.hpp | 6 +- src/hotspot/share/gc/shared/generation.cpp | 24 +- src/hotspot/share/gc/shared/memAllocator.cpp | 42 +- src/hotspot/share/gc/shared/memAllocator.hpp | 14 +- src/hotspot/share/gc/shared/space.cpp | 10 +- .../share/gc/shenandoah/shenandoahFullGC.cpp | 10 +- .../share/gc/shenandoah/shenandoahHeap.cpp | 10 + .../share/gc/shenandoah/shenandoahHeap.hpp | 2 + .../gc/shenandoah/shenandoahHeap.inline.hpp | 3 + .../shenandoahRootProcessor.inline.hpp | 2 +- src/hotspot/share/gc/z/zBarrier.cpp | 4 +- src/hotspot/share/gc/z/zBarrier.inline.hpp | 20 +- src/hotspot/share/gc/z/zBarrierSetNMethod.cpp | 6 +- src/hotspot/share/gc/z/zCollectedHeap.cpp | 24 +- src/hotspot/share/gc/z/zCollectedHeap.hpp | 3 +- src/hotspot/share/gc/z/zDriver.cpp | 3 +- src/hotspot/share/gc/z/zHeap.cpp | 7 +- src/hotspot/share/gc/z/zHeap.hpp | 5 +- src/hotspot/share/gc/z/zHeapIterator.cpp | 6 +- src/hotspot/share/gc/z/zMark.cpp | 32 +- src/hotspot/share/gc/z/zNMethod.cpp | 19 +- src/hotspot/share/gc/z/zNMethod.hpp | 1 + src/hotspot/share/gc/z/zVerify.cpp | 6 +- src/hotspot/share/include/jvm.h | 51 +- .../share/interpreter/abstractInterpreter.cpp | 4 + .../share/interpreter/abstractInterpreter.hpp | 5 +- .../share/interpreter/bytecodeTracer.cpp | 10 +- .../share/interpreter/interpreterRuntime.cpp | 7 +- .../share/interpreter/linkResolver.cpp | 21 +- .../share/interpreter/linkResolver.hpp | 4 +- src/hotspot/share/interpreter/oopMapCache.cpp | 4 + src/hotspot/share/interpreter/oopMapCache.hpp | 2 + .../templateInterpreterGenerator.cpp | 10 +- .../templateInterpreterGenerator.hpp | 3 +- src/hotspot/share/jfr/jfr.cpp | 43 +- src/hotspot/share/jfr/jfr.hpp | 25 +- src/hotspot/share/jfr/jni/jfrJavaSupport.cpp | 261 +- src/hotspot/share/jfr/jni/jfrJavaSupport.hpp | 23 +- src/hotspot/share/jfr/jni/jfrJniMethod.cpp | 8 +- .../jfr/jni/jfrJniMethodRegistration.cpp | 4 +- .../leakprofiler/checkpoint/eventEmitter.cpp | 9 +- .../checkpoint/objectSampleCheckpoint.cpp | 10 +- .../checkpoint/objectSampleCheckpoint.hpp | 3 +- .../leakprofiler/sampling/objectSample.cpp | 3 +- .../leakprofiler/sampling/objectSample.hpp | 49 +- .../leakprofiler/sampling/objectSampler.cpp | 43 +- .../leakprofiler/sampling/objectSampler.hpp | 3 +- src/hotspot/share/jfr/metadata/metadata.xml | 43 +- .../share/jfr/periodic/jfrPeriodic.cpp | 5 +- .../jfr/periodic/jfrThreadCPULoadEvent.cpp | 15 +- .../periodic/sampling/jfrThreadSampler.cpp | 17 +- .../checkpoint/jfrCheckpointManager.cpp | 148 +- .../checkpoint/jfrCheckpointManager.hpp | 14 +- .../checkpoint/jfrCheckpointWriter.cpp | 2 +- .../checkpoint/types/jfrThreadGroup.cpp | 15 +- .../checkpoint/types/jfrThreadState.cpp | 43 +- .../checkpoint/types/jfrThreadState.hpp | 6 +- .../jfr/recorder/checkpoint/types/jfrType.cpp | 73 +- .../jfr/recorder/checkpoint/types/jfrType.hpp | 7 +- .../checkpoint/types/jfrTypeManager.cpp | 56 +- .../checkpoint/types/jfrTypeManager.hpp | 10 +- .../types/traceid/jfrOopTraceId.hpp | 45 + .../types/traceid/jfrOopTraceId.inline.hpp | 75 + .../checkpoint/types/traceid/jfrTraceId.cpp | 17 +- .../checkpoint/types/traceid/jfrTraceId.hpp | 3 - .../types/traceid/jfrTraceId.inline.hpp | 7 +- .../types/traceid/jfrTraceIdEpoch.cpp | 20 +- .../types/traceid/jfrTraceIdEpoch.hpp | 13 + .../types/traceid/jfrTraceIdLoadBarrier.hpp | 2 +- .../share/jfr/recorder/jfrRecorder.cpp | 5 +- .../recorder/repository/jfrEmergencyDump.cpp | 13 +- .../recorder/repository/jfrEmergencyDump.hpp | 10 +- .../share/jfr/recorder/service/jfrEvent.hpp | 64 +- .../recorder/service/jfrRecorderThread.cpp | 10 +- .../jfr/recorder/stacktrace/jfrStackTrace.cpp | 187 +- .../jfr/recorder/stacktrace/jfrStackTrace.hpp | 7 +- .../stacktrace/jfrStackTraceRepository.cpp | 26 +- .../stacktrace/jfrStackTraceRepository.hpp | 6 +- .../share/jfr/recorder/storage/jfrBuffer.cpp | 28 +- .../share/jfr/recorder/storage/jfrBuffer.hpp | 6 +- .../storage/jfrEpochStorage.inline.hpp | 8 +- .../storage/jfrMemorySpace.inline.hpp | 21 +- .../share/jfr/recorder/storage/jfrStorage.cpp | 37 +- .../jfr/recorder/storage/jfrStorageUtils.hpp | 12 +- .../jfr/support/jfrEpochSynchronization.hpp | 4 +- .../share/jfr/support/jfrIntrinsics.cpp | 96 + .../share/jfr/support/jfrIntrinsics.hpp | 37 +- .../share/jfr/support/jfrThreadExtension.hpp | 12 +- src/hotspot/share/jfr/support/jfrThreadId.hpp | 8 +- .../share/jfr/support/jfrThreadId.inline.hpp | 64 + .../share/jfr/support/jfrThreadLocal.cpp | 342 ++- .../share/jfr/support/jfrThreadLocal.hpp | 89 +- .../share/jfr/writers/jfrJavaEventWriter.cpp | 116 +- .../share/jfr/writers/jfrJavaEventWriter.hpp | 4 + .../share/jfr/writers/jfrStorageAdapter.hpp | 4 +- .../share/jfr/writers/jfrStreamWriterHost.hpp | 3 +- .../writers/jfrStreamWriterHost.inline.hpp | 5 + .../share/jfr/writers/jfrWriterHost.hpp | 3 +- .../jfr/writers/jfrWriterHost.inline.hpp | 9 +- .../share/jvmci/jvmciCodeInstaller.cpp | 7 +- .../share/jvmci/jvmciCodeInstaller.hpp | 3 +- src/hotspot/share/jvmci/jvmciCompilerToVM.cpp | 4 +- src/hotspot/share/jvmci/jvmciRuntime.cpp | 2 + src/hotspot/share/jvmci/jvmciRuntime.hpp | 1 + src/hotspot/share/jvmci/vmStructs_jvmci.cpp | 5 + src/hotspot/share/logging/logTag.hpp | 2 + src/hotspot/share/memory/iterator.cpp | 20 +- src/hotspot/share/memory/iterator.hpp | 33 +- src/hotspot/share/memory/iterator.inline.hpp | 35 +- src/hotspot/share/oops/accessBackend.cpp | 23 +- src/hotspot/share/oops/accessBackend.hpp | 17 +- src/hotspot/share/oops/constMethod.hpp | 7 +- src/hotspot/share/oops/constantPool.cpp | 26 + src/hotspot/share/oops/constantPool.hpp | 3 +- src/hotspot/share/oops/cpCache.cpp | 8 +- src/hotspot/share/oops/cpCache.hpp | 8 +- src/hotspot/share/oops/cpCache.inline.hpp | 5 +- .../share/oops/instanceClassLoaderKlass.hpp | 4 +- src/hotspot/share/oops/instanceKlass.cpp | 16 +- .../share/oops/instanceStackChunkKlass.cpp | 265 ++ .../share/oops/instanceStackChunkKlass.hpp | 178 ++ .../oops/instanceStackChunkKlass.inline.hpp | 172 ++ src/hotspot/share/oops/klass.hpp | 4 +- src/hotspot/share/oops/method.cpp | 24 +- src/hotspot/share/oops/method.hpp | 53 +- src/hotspot/share/oops/objArrayOop.hpp | 5 +- src/hotspot/share/oops/oop.cpp | 22 +- src/hotspot/share/oops/oop.hpp | 10 +- src/hotspot/share/oops/oop.inline.hpp | 21 +- src/hotspot/share/oops/oopsHierarchy.hpp | 5 +- src/hotspot/share/oops/stackChunkOop.cpp | 652 +++++ src/hotspot/share/oops/stackChunkOop.hpp | 211 ++ .../share/oops/stackChunkOop.inline.hpp | 374 +++ src/hotspot/share/opto/buildOopMap.cpp | 14 +- src/hotspot/share/opto/bytecodeInfo.cpp | 7 + src/hotspot/share/opto/c2compiler.cpp | 6 +- src/hotspot/share/opto/compile.cpp | 1 + src/hotspot/share/opto/compile.hpp | 3 + src/hotspot/share/opto/library_call.cpp | 574 +++- src/hotspot/share/opto/library_call.hpp | 19 +- src/hotspot/share/opto/locknode.cpp | 8 +- src/hotspot/share/opto/macro.cpp | 37 +- src/hotspot/share/opto/memnode.cpp | 25 +- src/hotspot/share/opto/output.cpp | 2 + src/hotspot/share/opto/parse1.cpp | 4 + src/hotspot/share/opto/runtime.cpp | 58 +- src/hotspot/share/opto/runtime.hpp | 9 +- src/hotspot/share/prims/forte.cpp | 5 +- src/hotspot/share/prims/jni.cpp | 21 +- src/hotspot/share/prims/jniCheck.cpp | 17 +- src/hotspot/share/prims/jvm.cpp | 216 +- src/hotspot/share/prims/jvmti.xml | 503 +++- src/hotspot/share/prims/jvmti.xsl | 4 +- src/hotspot/share/prims/jvmtiEnter.xsl | 4 +- src/hotspot/share/prims/jvmtiEnv.cpp | 1398 +++++++--- src/hotspot/share/prims/jvmtiEnvBase.cpp | 1040 +++++-- src/hotspot/share/prims/jvmtiEnvBase.hpp | 336 ++- .../share/prims/jvmtiEnvThreadState.cpp | 114 +- .../share/prims/jvmtiEnvThreadState.hpp | 18 +- .../share/prims/jvmtiEventController.cpp | 132 +- .../share/prims/jvmtiEventController.hpp | 9 +- src/hotspot/share/prims/jvmtiExport.cpp | 376 ++- src/hotspot/share/prims/jvmtiExport.hpp | 21 +- src/hotspot/share/prims/jvmtiExtensions.cpp | 197 +- src/hotspot/share/prims/jvmtiImpl.cpp | 246 +- src/hotspot/share/prims/jvmtiImpl.hpp | 76 +- .../share/prims/jvmtiManageCapabilities.cpp | 7 +- src/hotspot/share/prims/jvmtiRawMonitor.cpp | 11 +- src/hotspot/share/prims/jvmtiTagMap.cpp | 3 +- src/hotspot/share/prims/jvmtiThreadState.cpp | 413 ++- src/hotspot/share/prims/jvmtiThreadState.hpp | 105 +- .../share/prims/jvmtiThreadState.inline.hpp | 80 +- src/hotspot/share/prims/jvmtiTrace.cpp | 8 +- src/hotspot/share/prims/jvmtiUtil.hpp | 4 +- src/hotspot/share/prims/nativeEntryPoint.cpp | 4 +- src/hotspot/share/prims/stackwalk.cpp | 122 +- src/hotspot/share/prims/stackwalk.hpp | 61 +- src/hotspot/share/runtime/continuation.cpp | 474 ++++ src/hotspot/share/runtime/continuation.hpp | 135 + .../share/runtime/continuationEntry.cpp | 113 + .../share/runtime/continuationEntry.hpp | 147 + .../runtime/continuationEntry.inline.hpp | 40 + .../share/runtime/continuationFreezeThaw.cpp | 2472 +++++++++++++++++ .../share/runtime/continuationHelper.hpp | 148 + .../runtime/continuationHelper.inline.hpp | 215 ++ .../share/runtime/continuationWrapper.cpp | 99 + .../runtime/continuationWrapper.inline.hpp | 193 ++ src/hotspot/share/runtime/deoptimization.cpp | 26 +- .../flags/jvmFlagConstraintsRuntime.cpp | 14 +- .../flags/jvmFlagConstraintsRuntime.hpp | 3 +- src/hotspot/share/runtime/frame.cpp | 521 +++- src/hotspot/share/runtime/frame.hpp | 144 +- src/hotspot/share/runtime/frame.inline.hpp | 50 +- src/hotspot/share/runtime/globals.hpp | 28 + src/hotspot/share/runtime/handles.cpp | 15 +- src/hotspot/share/runtime/handles.hpp | 12 +- src/hotspot/share/runtime/handles.inline.hpp | 10 +- src/hotspot/share/runtime/handshake.cpp | 1 + src/hotspot/share/runtime/init.cpp | 8 +- src/hotspot/share/runtime/java.cpp | 5 +- src/hotspot/share/runtime/javaFrameAnchor.hpp | 4 +- src/hotspot/share/runtime/mutexLocker.cpp | 12 +- src/hotspot/share/runtime/mutexLocker.hpp | 3 + src/hotspot/share/runtime/registerMap.hpp | 65 +- src/hotspot/share/runtime/sharedRuntime.cpp | 56 +- src/hotspot/share/runtime/sharedRuntime.hpp | 1 + src/hotspot/share/runtime/signature.cpp | 85 +- src/hotspot/share/runtime/signature.hpp | 13 +- .../share/runtime/smallRegisterMap.inline.hpp | 32 + .../share/runtime/stackChunkFrameStream.cpp | 39 + .../share/runtime/stackChunkFrameStream.hpp | 137 + .../runtime/stackChunkFrameStream.inline.hpp | 410 +++ .../share/runtime/stackFrameStream.cpp | 9 +- .../share/runtime/stackFrameStream.hpp | 4 +- src/hotspot/share/runtime/stackOverflow.hpp | 2 + src/hotspot/share/runtime/stackValue.cpp | 100 +- src/hotspot/share/runtime/stackValue.hpp | 13 +- .../share/runtime/stackWatermark.inline.hpp | 3 +- src/hotspot/share/runtime/stubRoutines.cpp | 37 +- src/hotspot/share/runtime/stubRoutines.hpp | 27 +- src/hotspot/share/runtime/sweeper.cpp | 9 +- src/hotspot/share/runtime/synchronizer.cpp | 1 + src/hotspot/share/runtime/thread.cpp | 260 +- src/hotspot/share/runtime/thread.hpp | 124 +- src/hotspot/share/runtime/thread.inline.hpp | 25 +- .../share/runtime/threadIdentifier.cpp | 41 + .../share/runtime/threadIdentifier.hpp | 42 + src/hotspot/share/runtime/threadSMR.cpp | 3 +- src/hotspot/share/runtime/vframe.cpp | 160 +- src/hotspot/share/runtime/vframe.hpp | 32 +- src/hotspot/share/runtime/vframe.inline.hpp | 85 +- src/hotspot/share/runtime/vframeArray.cpp | 7 +- src/hotspot/share/runtime/vframe_hp.cpp | 54 +- src/hotspot/share/runtime/vmOperation.hpp | 7 +- src/hotspot/share/runtime/vmOperations.cpp | 4 +- src/hotspot/share/runtime/vmStructs.cpp | 10 +- .../share/services/diagnosticCommand.cpp | 75 + .../share/services/diagnosticCommand.hpp | 28 +- src/hotspot/share/services/heapDumper.cpp | 2 +- src/hotspot/share/services/threadIdTable.cpp | 2 +- src/hotspot/share/services/threadService.cpp | 39 +- src/hotspot/share/services/threadService.hpp | 6 +- src/hotspot/share/utilities/bitMap.hpp | 29 +- src/hotspot/share/utilities/bitMap.inline.hpp | 6 +- src/hotspot/share/utilities/debug.cpp | 10 + src/hotspot/share/utilities/debug.hpp | 9 +- src/hotspot/share/utilities/macros.hpp | 4 + .../sun/nio/ch/DefaultPollerProvider.java | 44 + .../linux/classes/sun/nio/ch/EPoll.java | 9 +- .../linux/classes/sun/nio/ch/EPollPoller.java | 82 + .../classes/sun/nio/ch/EPollSelectorImpl.java | 10 +- .../sun/nio/ch/DefaultPollerProvider.java | 44 + .../macosx/classes/sun/nio/ch/KQueue.java | 9 +- .../classes/sun/nio/ch/KQueuePoller.java | 76 + .../sun/nio/ch/KQueueSelectorImpl.java | 10 +- .../classes/java/io/BufferedInputStream.java | 119 +- .../classes/java/io/BufferedOutputStream.java | 133 +- .../share/classes/java/io/BufferedReader.java | 411 ++- .../share/classes/java/io/BufferedWriter.java | 266 +- .../classes/java/io/FileInputStream.java | 59 +- .../classes/java/io/FileOutputStream.java | 37 +- .../classes/java/io/InputStreamReader.java | 29 +- .../classes/java/io/OutputStreamWriter.java | 27 +- .../share/classes/java/io/PrintStream.java | 334 ++- .../share/classes/java/io/PrintWriter.java | 379 ++- .../classes/java/io/PushbackInputStream.java | 37 +- .../classes/java/io/RandomAccessFile.java | 77 +- .../share/classes/java/io/Reader.java | 55 +- .../share/classes/java/io/Writer.java | 79 +- .../java/lang/ApplicationShutdownHooks.java | 13 +- .../java/lang/InheritableThreadLocal.java | 6 +- .../classes/java/lang/LiveStackFrame.java | 69 +- .../share/classes/java/lang/Object.java | 26 +- .../java/lang/PinnedThreadPrinter.java | 132 + .../share/classes/java/lang/Runnable.java | 39 +- .../classes/java/lang/StackFrameInfo.java | 12 +- .../classes/java/lang/StackStreamFactory.java | 105 +- .../classes/java/lang/StackTraceElement.java | 25 +- .../share/classes/java/lang/StackWalker.java | 125 +- .../share/classes/java/lang/System.java | 124 +- .../share/classes/java/lang/Thread.java | 2015 ++++++++++---- .../classes/java/lang/ThreadBuilders.java | 359 +++ .../share/classes/java/lang/ThreadGroup.java | 988 +++---- .../share/classes/java/lang/ThreadLocal.java | 80 +- .../share/classes/java/lang/Throwable.java | 71 +- .../classes/java/lang/VirtualThread.java | 1105 ++++++++ .../java/lang/WrongThreadException.java | 73 + .../classes/java/lang/invoke/MethodType.java | 3 +- .../classes/java/lang/ref/Finalizer.java | 24 +- .../java/lang/ref/NativeReferenceQueue.java | 92 + .../classes/java/lang/ref/Reference.java | 48 +- .../classes/java/lang/ref/ReferenceQueue.java | 172 +- .../classes/java/net/DatagramSocket.java | 35 +- .../share/classes/java/net/InetAddress.java | 29 +- .../classes/java/net/MulticastSocket.java | 5 +- .../share/classes/java/net/ServerSocket.java | 35 +- .../share/classes/java/net/Socket.java | 106 +- src/java.base/share/classes/java/net/URL.java | 22 +- .../classes/java/nio/MappedMemoryUtils.java | 11 +- .../spi/AbstractSelectableChannel.java | 4 +- .../classes/java/nio/charset/Charset.java | 26 +- .../java/util/concurrent/Exchanger.java | 4 +- .../java/util/concurrent/Executors.java | 38 + .../java/util/concurrent/ForkJoinPool.java | 52 +- .../java/util/concurrent/ThreadFactory.java | 7 +- .../util/concurrent/ThreadLocalRandom.java | 17 +- .../concurrent/ThreadPerTaskExecutor.java | 601 ++++ .../util/concurrent/ThreadPoolExecutor.java | 13 +- .../util/concurrent/locks/LockSupport.java | 73 +- .../share/classes/java/util/jar/JarFile.java | 23 +- .../access/JavaIOPrintStreamAccess.java | 31 + .../access/JavaIOPrintWriterAccess.java | 31 + .../jdk/internal/access/JavaLangAccess.java | 118 +- .../internal/access/JavaLangRefAccess.java | 16 +- .../access/JavaUtilConcurrentFJPAccess.java | 32 + .../access/JavaUtilConcurrentTLRAccess.java | 30 + .../jdk/internal/access/SharedSecrets.java | 59 +- .../jdk/internal/event/ThreadSleepEvent.java | 43 + .../internal/event/VirtualThreadEndEvent.java | 41 + .../event/VirtualThreadPinnedEvent.java | 31 + .../event/VirtualThreadStartEvent.java | 41 + .../event/VirtualThreadSubmitFailedEvent.java | 33 + .../jdk/internal/javac/PreviewFeature.java | 3 +- .../classes/jdk/internal/misc/Blocker.java | 125 + .../jdk/internal/misc/CarrierThread.java | 127 + .../jdk/internal/misc/InnocuousThread.java | 4 +- .../jdk/internal/misc/InternalLock.java | 84 + .../jdk/internal/misc/PreviewFeatures.java | 55 + .../misc/StructureViolationExceptions.java | 90 + .../internal/misc/TerminatingThreadLocal.java | 8 +- .../jdk/internal/misc/ThreadFlock.java | 592 ++++ .../jdk/internal/misc/ThreadTracker.java | 93 + .../share/classes/jdk/internal/misc/VM.java | 4 +- .../jdk/internal/misc/VirtualThreads.java | 95 + .../classes/jdk/internal/vm/Continuation.java | 508 ++++ .../jdk/internal/vm/ContinuationScope.java | 73 + .../jdk/internal/vm/ExtentLocalContainer.java | 216 ++ .../internal/vm/SharedThreadContainer.java | 184 ++ .../classes/jdk/internal/vm/StackChunk.java | 40 + .../jdk/internal/vm/StackableScope.java | 261 ++ .../jdk/internal/vm/ThreadContainer.java | 97 + .../jdk/internal/vm/ThreadContainers.java | 281 ++ .../classes/jdk/internal/vm/ThreadDumper.java | 360 +++ .../vm/annotation/ChangesCurrentThread.java | 44 + .../vm/annotation/JvmtiMountTransition.java | 40 + src/java.base/share/classes/module-info.java | 9 +- .../classes/sun/net/www/http/HttpClient.java | 3 + .../sun/nio/ch/DatagramChannelImpl.java | 124 +- .../sun/nio/ch/DatagramSocketAdaptor.java | 89 +- .../classes/sun/nio/ch/FileChannelImpl.java | 133 +- .../classes/sun/nio/ch/IOVecWrapper.java | 14 +- .../classes/sun/nio/ch/NativeThreadSet.java | 19 +- .../classes/sun/nio/ch/NioSocketImpl.java | 155 +- .../share/classes/sun/nio/ch/Poller.java | 469 ++++ .../classes/sun/nio/ch/PollerProvider.java | 70 + .../share/classes/sun/nio/ch/SelChImpl.java | 18 +- .../sun/nio/ch/ServerSocketChannelImpl.java | 49 +- .../classes/sun/nio/ch/SocketAdaptor.java | 12 +- .../classes/sun/nio/ch/SocketChannelImpl.java | 96 +- .../share/classes/sun/nio/ch/Util.java | 16 +- .../classes/sun/nio/cs/StreamDecoder.java | 185 +- .../classes/sun/nio/cs/StreamEncoder.java | 162 +- .../classes/sun/nio/fs/NativeBuffers.java | 14 +- .../classes/sun/security/ssl/SSLLogger.java | 6 +- src/java.base/share/native/include/jni.h | 13 +- .../share/native/libjava/Continuation.c | 39 + .../share/native/libjava/PreviewFeatures.c | 34 + .../share/native/libjava/RandomAccessFile.c | 12 +- .../share/native/libjava/StackStreamFactory.c | 25 +- .../share/native/libjava/StackTraceElement.c | 6 +- src/java.base/share/native/libjava/Thread.c | 21 +- .../share/native/libjava/VirtualThread.c | 44 + .../unix/classes/java/io/UnixFileSystem.java | 150 +- .../unix/classes/java/lang/ProcessImpl.java | 83 +- .../unix/classes/sun/nio/ch/NativeThread.java | 57 +- .../classes/sun/nio/ch/PollSelectorImpl.java | 9 +- .../classes/sun/nio/ch/SinkChannelImpl.java | 13 +- .../classes/sun/nio/ch/SourceChannelImpl.java | 13 +- .../unix/classes/sun/nio/fs/UnixCopyFile.java | 10 +- .../sun/nio/fs/UnixNativeDispatcher.java | 462 +-- .../unix/native/libjava/UnixFileSystem_md.c | 47 +- .../unix/native/libnio/ch/NativeThread.c | 6 +- .../native/libnio/fs/UnixNativeDispatcher.c | 17 +- .../classes/java/io/WinNTFileSystem.java | 128 +- .../classes/java/lang/ProcessImpl.java | 15 +- .../sun/nio/ch/DefaultPollerProvider.java | 44 + .../classes/sun/nio/ch/NativeThread.java | 49 +- .../windows/classes/sun/nio/ch/PipeImpl.java | 12 +- .../windows/classes/sun/nio/ch/WEPoll.java | 9 +- .../classes/sun/nio/ch/WEPollPoller.java | 76 + .../sun/nio/ch/WEPollSelectorImpl.java | 10 +- .../sun/nio/fs/WindowsNativeDispatcher.java | 218 +- .../classes/sun/nio/fs/WindowsSecurity.java | 11 +- .../native/libjava/WinNTFileSystem_md.c | 44 +- .../libnio/fs/WindowsNativeDispatcher.c | 11 +- .../classes/com/sun/media/sound/Printer.java | 4 +- .../share/classes/sun/awt/AppContext.java | 13 +- .../classes/java/util/logging/LogRecord.java | 2 +- .../remote/rmi/RMIConnectionImpl.java | 6 +- .../management/remote/rmi/RMIConnector.java | 11 +- .../internal/ServerCommunicatorAdmin.java | 4 +- .../java/lang/management/ThreadInfo.java | 6 +- .../java/lang/management/ThreadMXBean.java | 145 +- .../classes/sun/management/ThreadImpl.java | 119 +- .../classes/sun/rmi/transport/Transport.java | 6 +- src/java.se/share/data/jdwp/jdwp.spec | 134 +- .../sun/jvm/hotspot/memory/FileMapInfo.java | 11 +- .../jvm/hotspot/memory/SystemDictionary.java | 10 +- .../hotspot/oops/InstanceStackChunkKlass.java | 55 + .../sun/jvm/hotspot/oops/Metadata.java | 3 +- .../sun/jvm/hotspot/oops/OopUtilities.java | 91 +- .../src/jdk/vm/ci/amd64/AMD64.java | 5 +- .../com/sun/jdi/NativeMethodException.java | 5 +- .../com/sun/jdi/OpaqueFrameException.java | 52 + .../share/classes/com/sun/jdi/StackFrame.java | 13 +- .../com/sun/jdi/ThreadGroupReference.java | 23 +- .../classes/com/sun/jdi/ThreadReference.java | 71 +- .../classes/com/sun/jdi/VirtualMachine.java | 34 +- .../sun/jdi/request/ThreadDeathRequest.java | 20 +- .../sun/jdi/request/ThreadStartRequest.java | 20 +- .../sun/tools/example/debug/tty/Commands.java | 16 +- .../com/sun/tools/example/debug/tty/Env.java | 6 +- .../tools/example/debug/tty/EventHandler.java | 49 +- .../com/sun/tools/example/debug/tty/TTY.java | 15 +- .../tools/example/debug/tty/TTYResources.java | 2 + .../tools/example/debug/tty/ThreadInfo.java | 39 +- .../example/debug/tty/ThreadIterator.java | 28 +- .../tools/example/debug/tty/VMConnection.java | 12 +- .../tools/jdi/EventRequestManagerImpl.java | 17 +- .../com/sun/tools/jdi/JDWPException.java | 5 +- .../com/sun/tools/jdi/StackFrameImpl.java | 11 +- .../sun/tools/jdi/SunCommandLineLauncher.java | 14 +- .../sun/tools/jdi/ThreadReferenceImpl.java | 34 +- .../com/sun/tools/jdi/VirtualMachineImpl.java | 6 +- .../sun/tools/jdi/resources/jdi.properties | 8 +- src/jdk.jdi/share/man/jdb.1 | 31 + .../share/native/libjdwp/EventRequestImpl.c | 6 +- .../native/libjdwp/ThreadReferenceImpl.c | 30 +- .../share/native/libjdwp/VirtualMachineImpl.c | 25 +- .../share/native/libjdwp/debugInit.c | 22 +- .../share/native/libjdwp/error_messages.c | 6 +- .../share/native/libjdwp/eventFilter.c | 46 +- .../share/native/libjdwp/eventFilter.h | 3 +- .../share/native/libjdwp/eventHandler.c | 66 + .../share/native/libjdwp/eventHelper.c | 11 +- .../share/native/libjdwp/standardHandlers.c | 8 +- .../share/native/libjdwp/threadControl.c | 344 ++- .../share/native/libjdwp/threadControl.h | 4 +- .../share/native/libjdwp/util.c | 26 +- .../share/native/libjdwp/util.h | 9 +- .../jdk/jfr/consumer/RecordedThread.java | 14 +- .../jdk/jfr/events/ThreadSleepEvent.java | 44 + .../jdk/jfr/events/VirtualThreadEndEvent.java | 44 + .../jfr/events/VirtualThreadPinnedEvent.java | 40 + .../jfr/events/VirtualThreadStartEvent.java | 44 + .../VirtualThreadSubmitFailedEvent.java | 49 + .../jdk/jfr/internal/EventHandlerCreator.java | 8 +- .../classes/jdk/jfr/internal/EventWriter.java | 17 +- .../share/classes/jdk/jfr/internal/JVM.java | 5 +- .../jfr/internal/instrument/JDKEvents.java | 17 +- .../jdk/jfr/internal/tool/PrettyWriter.java | 4 +- src/jdk.jfr/share/conf/jfr/default.jfc | 48 + src/jdk.jfr/share/conf/jfr/profile.jfc | 48 + .../execution/LocalExecutionControl.java | 13 +- .../management/HotSpotDiagnosticMXBean.java | 56 +- .../com/sun/management/ThreadMXBean.java | 14 +- .../internal/HotSpotDiagnostic.java | 53 +- .../src/share/conf/common.properties | 6 +- .../gtest/classfile/test_symbolTable.cpp | 4 +- .../hotspot/gtest/oops/test_instanceKlass.cpp | 2 + test/hotspot/jtreg/ProblemList-Xcomp.txt | 12 +- .../hotspot/jtreg/ProblemList-svc-vthread.txt | 185 ++ test/hotspot/jtreg/ProblemList-zgc.txt | 6 +- test/hotspot/jtreg/ProblemList.txt | 2 + .../compiler/codecache/stress/Helper.java | 35 +- .../stress/OverloadCompileQueueTest.java | 10 +- .../UnexpectedDeoptimizationAllTest.java | 16 +- .../stress/UnexpectedDeoptimizationTest.java | 16 +- .../java.base/java/lang/Object.java | 12 +- .../OptimizeImplicitExceptions.java | 1 + .../TestSkeletonPredicateNegation.java | 1 + .../compiler/uncommontrap/Decompile.java | 3 +- .../TestPerfCountersAndMemoryPools.java | 23 +- .../runtime/LocalLong/LocalLongHelper.java | 6 +- .../runtime/LocalLong/LocalLongTest.java | 11 +- .../RedefineRunningMethods_Shared.java | 12 +- .../cds/appcds/test-classes/ParallelLoad.java | 5 +- .../jni/IsVirtualThread/IsVirtualThread.java | 70 + .../jni/IsVirtualThread/libIsVirtualThread.c | 29 + .../TestRegisterNativesWarning.java | 8 +- .../libregisterNativesWarning.c | 6 +- .../vthread/JNIMonitor/JNIMonitor.java | 62 + .../vthread/JNIMonitor/libJNIMonitor.cpp | 36 + .../jtreg/runtime/vthread/RedefineClass.java | 178 ++ .../jtreg/runtime/vthread/StackChunk.java | 40 + .../vthread/StackChunkClassLoaderTest.java | 78 + .../TestObjectAllocationSampleEvent.java | 53 + .../dcmd/thread/ThreadDumpToFileTest.java | 134 + .../libGetLocalWithoutSuspendTest.cpp | 11 +- .../GetOwnedMonitorInfoTest.java | 14 +- .../GetOwnedMonitorStackDepthInfoTest.java | 17 +- .../MyPackage/HeapMonitorVMEventsTest.java | 46 +- .../RedefineAddLambdaExpression.java | 27 +- .../RedefineClasses/RedefineDoubleDelete.java | 20 +- .../RedefineClasses/RedefineFinalizer.java | 15 +- .../RedefineInterfaceCall.java | 15 +- .../RedefineInterfaceMethods.java | 48 +- .../jvmti/RedefineClasses/RedefineLeak.java | 3 +- .../RedefinePreviousVersions.java | 27 +- .../RedefineRunningMethods.java | 108 +- .../RedefineRunningMethodsWithBacktrace.java | 88 +- .../RedefineSubtractLambdaExpression.java | 23 +- .../RedefineClasses/TestRedefineCondy.java | 12 +- .../libSuspendWithCurrentThread.cpp | 77 +- .../Breakpoint/breakpoint01/breakpoint01.java | 125 + .../breakpoint01/libbreakpoint01.cpp | 290 ++ .../ClassLoad/classload01/classload01.java | 136 + .../ClassLoad/classload01/libclassload01.cpp | 218 ++ .../ClassPrepare/classprep01/classprep01.java | 129 + .../classprep01/libclassprep01.cpp | 344 +++ .../Exception/exception01/exception01.java | 92 + .../Exception/exception01/exception01a.jasm | 154 + .../Exception/exception01/libexception01.cpp | 287 ++ .../ExceptionCatch/excatch01/excatch01.java | 89 + .../ExceptionCatch/excatch01/excatch01a.jasm | 154 + .../ExceptionCatch/excatch01/libexcatch01.cpp | 250 ++ .../FieldAccess/fieldacc01/fieldacc01.java | 96 + .../FieldAccess/fieldacc01/fieldacc01a.jasm | 191 ++ .../FieldAccess/fieldacc01/libfieldacc01.cpp | 346 +++ .../FieldAccess/fieldacc02/fieldacc02.java | 118 + .../FieldAccess/fieldacc02/libfieldacc02.cpp | 370 +++ .../FieldAccess/fieldacc03/fieldacc03.java | 127 + .../FieldAccess/fieldacc03/libfieldacc03.cpp | 318 +++ .../FieldAccess/fieldacc04/fieldacc04.java | 109 + .../FieldAccess/fieldacc04/libfieldacc04.cpp | 306 ++ .../fieldmod01/fieldmod01.java | 101 + .../fieldmod01/fieldmod01a.jasm | 106 + .../fieldmod01/libfieldmod01.cpp | 440 +++ .../fieldmod02/fieldmod02.java | 115 + .../fieldmod02/libfieldmod02.cpp | 481 ++++ .../FramePop/framepop01/framepop01.java | 89 + .../FramePop/framepop01/framepop01a.java | 29 + .../FramePop/framepop01/libframepop01.cpp | 280 ++ .../FramePop/framepop02/framepop02.java | 97 + .../FramePop/framepop02/libframepop02.cpp | 308 ++ .../MethodEntry/mentry01/libmentry01.cpp | 243 ++ .../events/MethodEntry/mentry01/mentry01.java | 101 + .../MethodEntry/mentry02/libmentry02.cpp | 180 ++ .../events/MethodEntry/mentry02/mentry02.java | 104 + .../events/MethodExit/mexit01/libmexit01.cpp | 256 ++ .../events/MethodExit/mexit01/mexit01.java | 97 + .../events/MethodExit/mexit01/mexit01a.jasm | 38 + .../events/MethodExit/mexit02/libmexit02.cpp | 245 ++ .../events/MethodExit/mexit02/mexit02.java | 94 + .../events/MethodExit/mexit02/mexit02a.jasm | 38 + .../mcontenter01/libmcontenter01.cpp | 213 ++ .../mcontenter01/mcontenter01.java | 147 + .../mcontentered01/libmcontentered01.cpp | 251 ++ .../mcontentered01/mcontentered01.java | 147 + .../monitorwait01/libmonitorwait01.cpp | 205 ++ .../monitorwait01/monitorwait01.java | 142 + .../monitorwaited01/libmonitorwaited01.cpp | 209 ++ .../monitorwaited01/monitorwaited01.java | 144 + .../nativemethbind01/libnativemethbind01.cpp | 223 ++ .../nativemethbind01/nativemethbind01.java | 95 + .../nativemethbind02/libnativemethbind02.cpp | 174 ++ .../nativemethbind02/nativemethbind02.java | 78 + .../nativemethbind03/libnativemethbind03.cpp | 231 ++ .../nativemethbind03/nativemethbind03.java | 84 + .../nativemethbind04/libnativemethbind04.cpp | 199 ++ .../nativemethbind04/nativemethbind04.java | 81 + .../singlestep01/libsinglestep01.cpp | 319 +++ .../SingleStep/singlestep01/singlestep01.java | 101 + .../singlestep02/libsinglestep02.cpp | 138 + .../SingleStep/singlestep02/singlestep02.java | 66 + .../singlestep03/libsinglestep03.cpp | 345 +++ .../SingleStep/singlestep03/singlestep03.java | 102 + .../ThreadEnd/threadend01/libthreadend01.cpp | 141 + .../ThreadEnd/threadend01/threadend01.java | 102 + .../ThreadEnd/threadend02/libthreadend02.cpp | 166 ++ .../ThreadEnd/threadend02/threadend02.java | 87 + .../threadstart01/libthreadstart01.cpp | 143 + .../threadstart01/threadstart01.java | 101 + .../threadstart02/libthreadstart02.cpp | 405 +++ .../threadstart02/threadstart02.java | 122 + .../threadstart03/libthreadstart03.cpp | 200 ++ .../threadstart03/threadstart03.java | 75 + .../vmobjalloc01/libvmobjalloc01.cpp | 147 + .../vmobjalloc01/vmobjalloc01.java | 68 + .../GetAllThreadsNullTest.java | 63 + .../libGetAllThreadsNullTest.cpp | 85 + .../jvmti/negative/contmon03/contmon03.java | 57 + .../jvmti/negative/contmon03/libcontmon03.cpp | 92 + .../jvmti/negative/framecnt02/framecnt02.java | 86 + .../negative/framecnt02/libframecnt02.cpp | 75 + .../jvmti/negative/framecnt03/framecnt03.java | 70 + .../negative/framecnt03/libframecnt03.cpp | 70 + .../jvmti/negative/frameloc03/frameloc03.java | 114 + .../negative/frameloc03/libframeloc03.cpp | 155 ++ .../negative/getstacktr02/getstacktr02.java | 73 + .../negative/getstacktr02/libgetstacktr02.cpp | 111 + .../negative/getstacktr09/getstacktr09.java | 90 + .../negative/getstacktr09/libgetstacktr09.cpp | 93 + .../jvmti/negative/thrinfo02/libthrinfo02.cpp | 105 + .../jvmti/negative/thrinfo02/thrinfo02.java | 70 + .../jvmti/negative/thrstat04/libthrstat04.cpp | 88 + .../jvmti/negative/thrstat04/thrstat04.java | 64 + .../GetStackTraceNotSuspendedStressTest.java | 103 + .../libGetStackTraceNotSuspendedStress.cpp | 141 + .../GetStackTraceSuspendedStressTest.java | 124 + .../libGetStackTraceSuspendedStress.cpp | 265 ++ .../SetGetThreadLocalStorageStressTest.java | 114 + .../libSetGetThreadLocalStorageStress.cpp | 269 ++ .../GetAllThreads/allthr01/allthr01.java | 146 + .../GetAllThreads/allthr01/liballthr01.cpp | 188 ++ .../contmon01/contmon01.java | 192 ++ .../contmon01/libcontmon01.cpp | 70 + .../contmon02/contmon02.java | 116 + .../contmon02/libcontmon02.cpp | 64 + .../GetFrameCount/framecnt01/framecnt01.java | 166 ++ .../framecnt01/libframecnt01.cpp | 80 + .../frameloc01/frameloc01.java | 120 + .../frameloc01/frameloc01a.jasm | 115 + .../frameloc01/libframeloc01.cpp | 161 ++ .../frameloc02/frameloc02.java | 70 + .../frameloc02/libframeloc02.cpp | 126 + .../GetStackTraceCurrentThreadTest.java | 71 + .../libGetStackTraceCurrentThreadTest.cpp | 79 + .../thread/GetStackTrace/get_stack_trace.h | 106 + .../getstacktr03/getstacktr03.java | 104 + .../getstacktr03/libgetstacktr03.cpp | 97 + .../getstacktr04/getstacktr04.java | 87 + .../getstacktr04/libgetstacktr04.cpp | 116 + .../getstacktr05/getstacktr05.java | 88 + .../getstacktr05/libgetstacktr05.cpp | 140 + .../getstacktr06/getstacktr06.java | 95 + .../getstacktr06/libgetstacktr06.cpp | 137 + .../getstacktr07/getstacktr07.java | 101 + .../getstacktr07/libgetstacktr07.cpp | 146 + .../getstacktr08/getstacktr08.java | 118 + .../getstacktr08/libgetstacktr08.cpp | 190 ++ .../GetThreadInfo/thrinfo01/libthrinfo01.cpp | 107 + .../GetThreadInfo/thrinfo01/thrinfo01.java | 129 + .../GetThreadState/thrstat01/libthrstat01.cpp | 151 + .../GetThreadState/thrstat01/thrstat01.java | 166 ++ .../GetThreadState/thrstat02/libthrstat02.cpp | 242 ++ .../GetThreadState/thrstat02/thrstat02.java | 229 ++ .../GetThreadState/thrstat03/libthrstat03.cpp | 95 + .../GetThreadState/thrstat03/thrstat03.java | 107 + .../GetThreadState/thrstat05/libthrstat05.cpp | 114 + .../GetThreadState/thrstat05/thrstat05.java | 375 +++ .../resumethrd01/libresumethrd01.cpp | 131 + .../resumethrd01/resumethrd01.java | 141 + .../resumethrd02/libresumethrd02.cpp | 182 ++ .../resumethrd02/resumethrd02.java | 151 + .../resumethrdlst01/libresumethrdlst01.cpp | 228 ++ .../resumethrdlst01/resumethrdlst01.java | 168 ++ .../resumethrdlst02/libresumethrdlst02.cpp | 282 ++ .../resumethrdlst02/resumethrdlst02.java | 167 ++ .../suspendthrd01/libsuspendthrd01.cpp | 131 + .../suspendthrd01/suspendthrd01.java | 149 + .../suspendthrd02/libsuspendthrd02.cpp | 177 ++ .../suspendthrd02/suspendthrd02.java | 156 ++ .../suspendthrd03/libsuspendthrd03.cpp | 176 ++ .../suspendthrd03/suspendthrd03.java | 316 +++ .../suspendthrdlst01/libsuspendthrdlst01.cpp | 227 ++ .../suspendthrdlst01/suspendthrdlst01.java | 167 ++ .../suspendthrdlst02/libsuspendthrdlst02.cpp | 269 ++ .../suspendthrdlst02/suspendthrdlst02.java | 171 ++ .../BreakpointInYieldTest.java | 95 + .../libBreakpointInYieldTest.cpp | 323 +++ .../ContFramePopTest/ContFramePopTest.java | 123 + .../ContFramePopTest/libContFramePopTest.cpp | 223 ++ .../ContStackDepthTest.java | 114 + .../libContStackDepthTest.cpp | 306 ++ .../ContYieldBreakPointTest.java | 102 + .../libContYieldBreakPointTest.cpp | 205 ++ .../ContinuationTest/ContinuationTest.java | 118 + .../ContinuationTest/libContinuationTest.cpp | 179 ++ .../GetSetLocalTest/GetSetLocalTest.java | 105 + .../GetSetLocalTest/libGetSetLocalTest.cpp | 493 ++++ .../InterruptThreadTest.java | 81 + .../libInterruptThreadTest.cpp | 78 + .../MethodExitTest/MethodExitTest.java | 112 + .../MethodExitTest/libMethodExitTest.cpp | 616 ++++ .../NullAsCurrentThreadTest.java | 76 + .../libNullAsCurrentThreadTest.cpp | 407 +++ .../PinnedTaskTest/PinnedTaskTest.java | 54 + .../PinnedTaskTest/libPinnedTaskTest.cpp | 58 + .../RawMonitorTest/RawMonitorTest.java | 74 + .../RawMonitorTest/libRawMonitorTest.cpp | 115 + .../RedefineRunningMethods.java | 184 ++ .../SelfSuspendDisablerTest.java | 75 + .../libSelfSuspendDisablerTest.cpp | 86 + .../SuspendResume1/SuspendResume1.java | 170 ++ .../SuspendResume1/libSuspendResume1.cpp | 422 +++ .../SuspendResume2/SuspendResume2.java | 164 ++ .../SuspendResume2/libSuspendResume2.cpp | 337 +++ .../SuspendResumeAll/SuspendResumeAll.java | 176 ++ .../SuspendResumeAll/libSuspendResumeAll.cpp | 394 +++ .../VThreadMonitorTest.java | 144 + .../libVThreadMonitorTest.cpp | 360 +++ .../VThreadNotifyFramePopTest.java | 116 + .../libVThreadNotifyFramePopTest.cpp | 301 ++ .../vthread/VThreadTest/VThreadTest.java | 87 + .../vthread/VThreadTest/libVThreadTest.cpp | 631 +++++ .../VThreadUnsupportedTest.java | 69 + .../libVThreadUnsupportedTest.cpp | 200 ++ .../WaitNotifySuspendedVThreadTest.java | 86 + .../libWaitNotifySuspendedVThread.cpp | 181 ++ .../vthread/premain/AgentWithVThread.java | 39 + .../vthread/premain/AgentWithVThreadTest.java | 44 + .../serviceability/sa/ClhsdbInspect.java | 8 +- .../tmtools/jstack/WaitNotifyThreadTest.java | 4 +- .../tmtools/jstack/utils/DefaultFormat.java | 14 +- .../testlibrary/jvmti/TransformerAgent.java | 8 +- .../gc/g1/unloading/UnloadingTest.java | 3 +- .../TestDescription.java | 3 +- .../TestDescription.java | 3 +- .../TestDescription.java | 3 +- .../nsk/jdb/kill/kill001/kill001.java | 38 +- .../nsk/jdb/kill/kill001/kill001a.java | 34 +- .../jdb/monitor/monitor001/monitor001.java | 10 +- .../jdb/threads/threads002/threads002.java | 4 +- .../jdb/threads/threads002/threads002a.java | 13 +- .../jdb/threads/threads003/threads003.java | 130 + .../jdb/threads/threads003/threads003a.java | 131 + .../nsk/jdb/trace/trace001/trace001.java | 7 +- .../nsk/jdb/trace/trace001/trace001a.java | 10 +- .../unmonitor/unmonitor001/unmonitor001.java | 10 +- .../stepRequests/stepreq001t.java | 16 +- .../referringObjects003.java | 9 +- .../waitingThreads/waitingthreads002a.java | 7 +- .../jdi/StackFrame/_bounds_/bounds002.java | 31 +- .../threads/threads001.java | 43 +- .../forceEarlyReturn002.java | 27 +- .../forceEarlyReturn002a.java | 6 +- .../popFrames/popframes001.java | 30 +- .../ThreadStartEvent/thread/thread001.java | 17 +- .../ownedMonitorsStackDepthInfo001a.java | 25 +- .../thrgrpinfo001/thrgrpinfo001.cpp | 10 +- .../nsk/jvmti/GetThreadInfo/thrinfo001.java | 35 +- .../thrinfo001/TestDescription.java | 3 +- .../GetThreadInfo/thrinfo001/thrinfo001.cpp | 13 +- .../jvmti/RedefineClasses/StressRedefine.java | 31 +- .../StressRedefineVirtual.java | 40 + .../threadstart002/threadstart002.cpp | 45 +- .../java.base/java/lang/Object.java | 20 +- .../capability/CM01/cm01t011/cm01t011.cpp | 22 +- .../extension/EX03/ex03t001/ex03t001.cpp | 8 +- .../followref003/followref003.cpp | 4 +- .../monitoring/share/ThreadController.java | 19 +- .../share/thread/LockingThreads.java | 4 +- .../share/thread/RunningThread.java | 5 +- .../share/thread/SleepingThread.java | 6 +- .../thread/SynchronizerLockingThreads.java | 5 +- .../thread/ThreadMonitoringScenarioBase.java | 4 +- .../share/thread/TimedWaitingThread.java | 3 +- .../share/thread/WaitingThread.java | 5 +- .../monitoring/stress/thread/strace001.java | 55 +- .../vmTestbase/nsk/share/MainWrapper.java | 93 + .../jtreg/vmTestbase/nsk/share/jdb/Jdb.java | 52 +- .../vmTestbase/nsk/share/jdb/Launcher.java | 14 +- .../vmTestbase/nsk/share/jdi/Binder.java | 16 +- .../vmTestbase/nsk/share/jdi/Debugee.java | 5 +- .../nsk/share/jdi/JDIThreadFactory.java | 23 +- .../nsk/share/jdi/OwnedMonitorsDebugger.java | 43 +- .../nsk/share/jpda/DebugeeBinder.java | 16 +- .../nsk/share/locks/LockingThread.java | 5 +- .../vmTestbase/nsk/share/log/LogSupport.java | 8 +- .../nsk/share/runner/RunParams.java | 13 +- .../nsk/share/runner/ThreadsRunner.java | 30 +- .../nsk/share/test/LocalRandom.java | 5 +- .../nsk/stress/strace/strace001.java | 15 +- .../nsk/stress/strace/strace002.java | 15 +- .../nsk/stress/strace/strace003.java | 19 +- .../nsk/stress/strace/strace004.java | 19 +- .../nsk/stress/strace/strace005.java | 16 +- .../nsk/stress/strace/strace006.java | 16 +- .../nsk/stress/strace/strace007.java | 10 +- .../nsk/stress/strace/strace008.java | 12 +- .../nsk/stress/strace/strace009.java | 10 +- .../nsk/stress/strace/strace010.java | 10 +- .../nsk/stress/strace/strace011.java | 10 +- .../nsk/stress/strace/strace012.java | 10 +- .../nsk/stress/strace/strace013.java | 5 +- .../nsk/stress/strace/strace014.java | 7 +- .../nsk/stress/strace/strace015.java | 5 +- test/jdk/ProblemList-Xcomp.txt | 7 +- test/jdk/ProblemList-svc-vthread.txt | 46 + test/jdk/ProblemList.txt | 4 +- test/jdk/TEST.groups | 3 +- test/jdk/com/sun/jdi/EATests.java | 4 +- test/jdk/com/sun/jdi/ResumeOneThreadTest.java | 7 +- test/jdk/com/sun/jdi/TestScaffold.java | 80 +- test/jdk/com/sun/jdi/lib/jdb/Jdb.java | 8 +- .../HotSpotDiagnosticMXBean/DumpThreads.java | 236 ++ .../8028054/TestConstructorFinder.java | 4 +- .../XMLDecoder/8028054/TestMethodFinder.java | 4 +- test/jdk/java/lang/ProcessBuilder/Basic.java | 24 +- test/jdk/java/lang/Thread/BuilderTest.java | 917 ++++++ .../java/lang/Thread/JoinWithDuration.java | 214 ++ .../java/lang/Thread/SleepWithDuration.java | 126 + .../lang/Thread/UncaughtExceptionsTest.java | 10 +- .../java/lang/Thread/virtual/Collectable.java | 70 + .../lang/Thread/virtual/CustomScheduler.java | 251 ++ .../lang/Thread/virtual/GetStackTrace.java | 129 + .../virtual/GetStackTraceWhenRunnable.java | 82 + .../java/lang/Thread/virtual/HoldsLock.java | 181 ++ .../java/lang/Thread/virtual/JfrEvents.java | 198 ++ .../jdk/java/lang/Thread/virtual/Locking.java | 218 ++ .../virtual/ParkWithFixedThreadPool.java | 79 + .../jdk/java/lang/Thread/virtual/Parking.java | 362 +++ .../virtual/PreviewFeaturesNotEnabled.java | 112 + .../java/lang/Thread/virtual/Reflection.java | 377 +++ .../lang/Thread/virtual/ShutdownHook.java | 54 + .../java/lang/Thread/virtual/StackTraces.java | 82 + .../java/lang/Thread/virtual/ThreadAPI.java | 2288 +++++++++++++++ .../lang/Thread/virtual/ThreadBuilders.java | 52 + .../lang/Thread/virtual/ThreadLocals.java | 201 ++ .../Thread/virtual/TracePinnedThreads.java | 65 + .../java/lang/Thread/virtual/WaitNotify.java | 164 ++ .../virtual/stress/GetStackTraceALot.java | 124 + .../lang/Thread/virtual/stress/PinALot.java | 76 + .../lang/Thread/virtual/stress/PingPong.java | 124 + .../lang/Thread/virtual/stress/Skynet.java | 120 + .../lang/Thread/virtual/stress/SleepALot.java | 73 + .../Thread/virtual/stress/TEST.properties | 24 + .../lang/Thread/virtual/stress/TimedGet.java | 109 + .../lang/Thread/virtual/stress/YieldALot.java | 70 + .../java/lang/Thread/virtual/stress/skynet.md | 25 + .../jdk/java/lang/ThreadGroup/BasicTests.java | 846 ++++++ test/jdk/java/lang/ThreadGroup/Daemon.java | 60 - test/jdk/java/lang/ThreadGroup/Destroy.java | 115 - .../java/lang/ThreadGroup/NullThreadName.java | 85 - .../java/lang/ThreadGroup/SetMaxPriority.java | 69 - test/jdk/java/lang/ThreadGroup/Stop.java | 68 - test/jdk/java/lang/ThreadGroup/Suspend.java | 77 - .../Throwable/OverrideFillInStackTrace.java | 41 + .../instrument/ParallelTransformerLoader.sh | 78 - .../ParallelTransformerLoaderAgent.java | 122 +- .../ParallelTransformerLoaderApp.java | 90 - .../ParallelTransformerLoaderTest.java | 90 + test/jdk/java/lang/instrument/TestClass1.java | 11 +- test/jdk/java/lang/instrument/TestClass2.java | 11 +- test/jdk/java/lang/instrument/TestClass3.java | 11 +- .../ThreadMXBean/VirtualThreadDeadlocks.java | 103 + .../ThreadMXBean/VirtualThreads.java | 220 ++ test/jdk/java/lang/ref/CleanerTest.java | 42 +- .../java/lang/ref/OOMEInReferenceHandler.java | 4 +- test/jdk/java/net/Socket/Timeouts.java | 10 +- .../java/net/vthread/BlockingSocketOps.java | 842 ++++++ test/jdk/java/net/vthread/HttpALot.java | 108 + test/jdk/java/net/vthread/InterruptHttp.java | 132 + .../etc/AdaptorCloseAndInterrupt.java | 2 +- .../channels/vthread/BlockingChannelOps.java | 909 ++++++ .../concurrent/ExecutorService/CloseTest.java | 181 ++ .../concurrent/Future/DefaultMethods.java | 227 ++ .../ThreadPerTaskExecutorTest.java | 1020 +++++++ .../lib/DelegatingExecutorService.java | 126 + .../util/concurrent/tck/ExecutorsTest.java | 10 + .../util/concurrent/tck/ForkJoinPoolTest.java | 40 +- .../util/concurrent/tck/JSR166TestCase.java | 4 +- .../misc/ThreadFlock/ThreadFlockTest.java | 1054 +++++++ .../jdk/internal/vm/Continuation/Basic.java | 401 +++ .../vm/Continuation/ClassUnloading.java | 116 + .../jdk/internal/vm/Continuation/Fuzz.java | 1012 +++++++ .../vm/Continuation/HumongousStack.java | 91 + .../vm/Continuation/LiveFramesDriver.java | 40 + .../jdk/internal/vm/Continuation/Scoped.java | 145 + .../internal/vm/Continuation/TEST.properties | 1 + .../jdk/jdk/internal/vm/Continuation/fuzz.dat | 20 + .../java.base/java/lang/LiveFrames.java | 113 + .../java/lang/StackWalkerHelper.java | 169 ++ .../internal/vm/Continuation/libBasicJNI.c | 31 + .../recordingstream/TestRecursive.java | 4 +- .../TestGCCauseWithG1ConcurrentMark.java | 2 +- .../TestGCCauseWithG1FullCollection.java | 2 +- .../metadata/TestDefaultConfigurations.java | 6 +- .../metadata/TestLookForUntestedEvents.java | 9 +- .../jdk/jfr/event/runtime/LatchedThread.java | 31 +- .../event/runtime/TestActiveSettingEvent.java | 6 +- .../jfr/event/runtime/TestThreadEndEvent.java | 7 +- .../event/runtime/TestThreadSleepEvent.java | 50 +- .../event/runtime/TestThreadStartEvent.java | 7 +- .../runtime/TestVirtualThreadEndEvent.java | 95 + .../runtime/TestVirtualThreadStartEvent.java | 96 + test/jdk/jdk/jfr/jvm/LatchedThread.java | 91 + test/jdk/jdk/jfr/jvm/TestGetStackTraceId.java | 5 +- test/jdk/jdk/jfr/jvm/TestJFRIntrinsic.java | 9 +- test/jdk/jdk/jfr/jvm/TestThreadExclusion.java | 52 +- .../jfr/jvm/TestVirtualThreadExclusion.java | 115 + .../threading/TestDeepVirtualStackTrace.java | 123 + .../jfr/threading/TestManyVirtualThreads.java | 106 + .../threading/TestNestedVirtualThreads.java | 85 + test/langtools/TEST.groups | 2 +- .../jdk/test/whitebox/CPUInfoTest.java | 5 +- test/lib/jdk/test/lib/Utils.java | 8 +- test/lib/jdk/test/lib/jfr/EventNames.java | 6 +- test/lib/jdk/test/lib/jvmti/DebugeeClass.java | 80 + test/lib/jdk/test/lib/jvmti/jvmti_common.h | 859 ++++++ test/lib/jdk/test/lib/jvmti/jvmti_thread.h | 338 +++ .../jdk/test/lib/process/ProcessTools.java | 145 +- .../jdk/test/lib/thread/VThreadRunner.java | 143 + .../math/BigIntegerMersennePrimeMultiply.java | 8 +- test/micro/org/openjdk/bench/loom/Skynet.java | 114 + .../bench/loom/obsolete/FreezeAndThaw.java | 149 + .../openjdk/bench/loom/obsolete/OneShot.java | 252 ++ .../bench/loom/obsolete/Oscillation.java | 104 + .../org/openjdk/bench/loom/ring/Channel.java | 30 + .../org/openjdk/bench/loom/ring/Channels.java | 536 ++++ .../org/openjdk/bench/loom/ring/Ring.java | 167 ++ test/micro/org/openjdk/bench/loom/skynet.md | 25 + 1133 files changed, 95935 insertions(+), 8335 deletions(-) create mode 100644 src/hotspot/cpu/aarch64/continuationEntry_aarch64.inline.hpp create mode 100644 src/hotspot/cpu/aarch64/continuationFreezeThaw_aarch64.inline.hpp create mode 100644 src/hotspot/cpu/aarch64/continuationHelper_aarch64.inline.hpp create mode 100644 src/hotspot/cpu/aarch64/smallRegisterMap_aarch64.inline.hpp create mode 100644 src/hotspot/cpu/aarch64/stackChunkFrameStream_aarch64.inline.hpp create mode 100644 src/hotspot/cpu/aarch64/stackChunkOop_aarch64.inline.hpp create mode 100644 src/hotspot/cpu/arm/continuationEntry_arm.inline.hpp create mode 100644 src/hotspot/cpu/arm/continuationFreezeThaw_arm.inline.hpp create mode 100644 src/hotspot/cpu/arm/continuationHelper_arm.inline.hpp create mode 100644 src/hotspot/cpu/arm/smallRegisterMap_arm.inline.hpp create mode 100644 src/hotspot/cpu/arm/stackChunkFrameStream_arm.inline.hpp create mode 100644 src/hotspot/cpu/arm/stackChunkOop_arm.inline.hpp create mode 100644 src/hotspot/cpu/ppc/continuationEntry_ppc.inline.hpp create mode 100644 src/hotspot/cpu/ppc/continuationFreezeThaw_ppc.inline.hpp create mode 100644 src/hotspot/cpu/ppc/continuationHelper_ppc.inline.hpp create mode 100644 src/hotspot/cpu/ppc/smallRegisterMap_ppc.inline.hpp create mode 100644 src/hotspot/cpu/ppc/stackChunkFrameStream_ppc.inline.hpp create mode 100644 src/hotspot/cpu/ppc/stackChunkOop_ppc.inline.hpp create mode 100644 src/hotspot/cpu/riscv/continuationEntry_riscv.inline.hpp create mode 100644 src/hotspot/cpu/riscv/continuationFreezeThaw_riscv.inline.hpp create mode 100644 src/hotspot/cpu/riscv/continuationHelper_riscv.inline.hpp create mode 100644 src/hotspot/cpu/riscv/smallRegisterMap_riscv.inline.hpp create mode 100644 src/hotspot/cpu/riscv/stackChunkFrameStream_riscv.inline.hpp create mode 100644 src/hotspot/cpu/riscv/stackChunkOop_riscv.inline.hpp create mode 100644 src/hotspot/cpu/s390/continuationEntry_s390.inline.hpp create mode 100644 src/hotspot/cpu/s390/continuationFreezeThaw_s390.inline.hpp create mode 100644 src/hotspot/cpu/s390/continuationHelper_s390.inline.hpp create mode 100644 src/hotspot/cpu/s390/smallRegisterMap_s390.inline.hpp create mode 100644 src/hotspot/cpu/s390/stackChunkFrameStream_s390.inline.hpp create mode 100644 src/hotspot/cpu/s390/stackChunkOop_s390.inline.hpp create mode 100644 src/hotspot/cpu/x86/continuationEntry_x86.inline.hpp create mode 100644 src/hotspot/cpu/x86/continuationFreezeThaw_x86.inline.hpp create mode 100644 src/hotspot/cpu/x86/continuationHelper_x86.inline.hpp create mode 100644 src/hotspot/cpu/x86/smallRegisterMap_x86.inline.hpp create mode 100644 src/hotspot/cpu/x86/stackChunkFrameStream_x86.inline.hpp create mode 100644 src/hotspot/cpu/x86/stackChunkOop_x86.inline.hpp create mode 100644 src/hotspot/cpu/zero/continuationEntry_zero.inline.hpp create mode 100644 src/hotspot/cpu/zero/continuationFreezeThaw_zero.inline.hpp create mode 100644 src/hotspot/cpu/zero/continuationHelper_zero.inline.hpp create mode 100644 src/hotspot/cpu/zero/smallRegisterMap_zero.inline.hpp create mode 100644 src/hotspot/cpu/zero/stackChunkFrameStream_zero.inline.hpp create mode 100644 src/hotspot/cpu/zero/stackChunkOop_zero.inline.hpp create mode 100644 src/hotspot/share/code/codeBlob.inline.hpp create mode 100644 src/hotspot/share/code/codeCache.inline.hpp create mode 100644 src/hotspot/share/compiler/oopMap.inline.hpp create mode 100644 src/hotspot/share/gc/shared/continuationGCSupport.hpp create mode 100644 src/hotspot/share/gc/shared/continuationGCSupport.inline.hpp create mode 100644 src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrOopTraceId.hpp create mode 100644 src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrOopTraceId.inline.hpp create mode 100644 src/hotspot/share/jfr/support/jfrIntrinsics.cpp create mode 100644 src/hotspot/share/jfr/support/jfrThreadId.inline.hpp create mode 100644 src/hotspot/share/oops/instanceStackChunkKlass.cpp create mode 100644 src/hotspot/share/oops/instanceStackChunkKlass.hpp create mode 100644 src/hotspot/share/oops/instanceStackChunkKlass.inline.hpp create mode 100644 src/hotspot/share/oops/stackChunkOop.cpp create mode 100644 src/hotspot/share/oops/stackChunkOop.hpp create mode 100644 src/hotspot/share/oops/stackChunkOop.inline.hpp create mode 100644 src/hotspot/share/runtime/continuation.cpp create mode 100644 src/hotspot/share/runtime/continuation.hpp create mode 100644 src/hotspot/share/runtime/continuationEntry.cpp create mode 100644 src/hotspot/share/runtime/continuationEntry.hpp create mode 100644 src/hotspot/share/runtime/continuationEntry.inline.hpp create mode 100644 src/hotspot/share/runtime/continuationFreezeThaw.cpp create mode 100644 src/hotspot/share/runtime/continuationHelper.hpp create mode 100644 src/hotspot/share/runtime/continuationHelper.inline.hpp create mode 100644 src/hotspot/share/runtime/continuationWrapper.cpp create mode 100644 src/hotspot/share/runtime/continuationWrapper.inline.hpp create mode 100644 src/hotspot/share/runtime/smallRegisterMap.inline.hpp create mode 100644 src/hotspot/share/runtime/stackChunkFrameStream.cpp create mode 100644 src/hotspot/share/runtime/stackChunkFrameStream.hpp create mode 100644 src/hotspot/share/runtime/stackChunkFrameStream.inline.hpp create mode 100644 src/hotspot/share/runtime/threadIdentifier.cpp create mode 100644 src/hotspot/share/runtime/threadIdentifier.hpp create mode 100644 src/java.base/linux/classes/sun/nio/ch/DefaultPollerProvider.java create mode 100644 src/java.base/linux/classes/sun/nio/ch/EPollPoller.java create mode 100644 src/java.base/macosx/classes/sun/nio/ch/DefaultPollerProvider.java create mode 100644 src/java.base/macosx/classes/sun/nio/ch/KQueuePoller.java create mode 100644 src/java.base/share/classes/java/lang/PinnedThreadPrinter.java create mode 100644 src/java.base/share/classes/java/lang/ThreadBuilders.java create mode 100644 src/java.base/share/classes/java/lang/VirtualThread.java create mode 100644 src/java.base/share/classes/java/lang/WrongThreadException.java create mode 100644 src/java.base/share/classes/java/lang/ref/NativeReferenceQueue.java create mode 100644 src/java.base/share/classes/java/util/concurrent/ThreadPerTaskExecutor.java create mode 100644 src/java.base/share/classes/jdk/internal/access/JavaIOPrintStreamAccess.java create mode 100644 src/java.base/share/classes/jdk/internal/access/JavaIOPrintWriterAccess.java create mode 100644 src/java.base/share/classes/jdk/internal/access/JavaUtilConcurrentFJPAccess.java create mode 100644 src/java.base/share/classes/jdk/internal/access/JavaUtilConcurrentTLRAccess.java create mode 100644 src/java.base/share/classes/jdk/internal/event/ThreadSleepEvent.java create mode 100644 src/java.base/share/classes/jdk/internal/event/VirtualThreadEndEvent.java create mode 100644 src/java.base/share/classes/jdk/internal/event/VirtualThreadPinnedEvent.java create mode 100644 src/java.base/share/classes/jdk/internal/event/VirtualThreadStartEvent.java create mode 100644 src/java.base/share/classes/jdk/internal/event/VirtualThreadSubmitFailedEvent.java create mode 100644 src/java.base/share/classes/jdk/internal/misc/Blocker.java create mode 100644 src/java.base/share/classes/jdk/internal/misc/CarrierThread.java create mode 100644 src/java.base/share/classes/jdk/internal/misc/InternalLock.java create mode 100644 src/java.base/share/classes/jdk/internal/misc/PreviewFeatures.java create mode 100644 src/java.base/share/classes/jdk/internal/misc/StructureViolationExceptions.java create mode 100644 src/java.base/share/classes/jdk/internal/misc/ThreadFlock.java create mode 100644 src/java.base/share/classes/jdk/internal/misc/ThreadTracker.java create mode 100644 src/java.base/share/classes/jdk/internal/misc/VirtualThreads.java create mode 100644 src/java.base/share/classes/jdk/internal/vm/Continuation.java create mode 100644 src/java.base/share/classes/jdk/internal/vm/ContinuationScope.java create mode 100644 src/java.base/share/classes/jdk/internal/vm/ExtentLocalContainer.java create mode 100644 src/java.base/share/classes/jdk/internal/vm/SharedThreadContainer.java create mode 100644 src/java.base/share/classes/jdk/internal/vm/StackChunk.java create mode 100644 src/java.base/share/classes/jdk/internal/vm/StackableScope.java create mode 100644 src/java.base/share/classes/jdk/internal/vm/ThreadContainer.java create mode 100644 src/java.base/share/classes/jdk/internal/vm/ThreadContainers.java create mode 100644 src/java.base/share/classes/jdk/internal/vm/ThreadDumper.java create mode 100644 src/java.base/share/classes/jdk/internal/vm/annotation/ChangesCurrentThread.java create mode 100644 src/java.base/share/classes/jdk/internal/vm/annotation/JvmtiMountTransition.java create mode 100644 src/java.base/share/classes/sun/nio/ch/Poller.java create mode 100644 src/java.base/share/classes/sun/nio/ch/PollerProvider.java create mode 100644 src/java.base/share/native/libjava/Continuation.c create mode 100644 src/java.base/share/native/libjava/PreviewFeatures.c create mode 100644 src/java.base/share/native/libjava/VirtualThread.c create mode 100644 src/java.base/windows/classes/sun/nio/ch/DefaultPollerProvider.java create mode 100644 src/java.base/windows/classes/sun/nio/ch/WEPollPoller.java create mode 100644 src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/oops/InstanceStackChunkKlass.java create mode 100644 src/jdk.jdi/share/classes/com/sun/jdi/OpaqueFrameException.java create mode 100644 src/jdk.jfr/share/classes/jdk/jfr/events/ThreadSleepEvent.java create mode 100644 src/jdk.jfr/share/classes/jdk/jfr/events/VirtualThreadEndEvent.java create mode 100644 src/jdk.jfr/share/classes/jdk/jfr/events/VirtualThreadPinnedEvent.java create mode 100644 src/jdk.jfr/share/classes/jdk/jfr/events/VirtualThreadStartEvent.java create mode 100644 src/jdk.jfr/share/classes/jdk/jfr/events/VirtualThreadSubmitFailedEvent.java create mode 100644 test/hotspot/jtreg/ProblemList-svc-vthread.txt create mode 100644 test/hotspot/jtreg/runtime/jni/IsVirtualThread/IsVirtualThread.java create mode 100644 test/hotspot/jtreg/runtime/jni/IsVirtualThread/libIsVirtualThread.c create mode 100644 test/hotspot/jtreg/runtime/vthread/JNIMonitor/JNIMonitor.java create mode 100644 test/hotspot/jtreg/runtime/vthread/JNIMonitor/libJNIMonitor.cpp create mode 100644 test/hotspot/jtreg/runtime/vthread/RedefineClass.java create mode 100644 test/hotspot/jtreg/runtime/vthread/StackChunk.java create mode 100644 test/hotspot/jtreg/runtime/vthread/StackChunkClassLoaderTest.java create mode 100644 test/hotspot/jtreg/runtime/vthread/TestObjectAllocationSampleEvent.java create mode 100644 test/hotspot/jtreg/serviceability/dcmd/thread/ThreadDumpToFileTest.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/events/Breakpoint/breakpoint01/breakpoint01.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/events/Breakpoint/breakpoint01/libbreakpoint01.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/events/ClassLoad/classload01/classload01.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/events/ClassLoad/classload01/libclassload01.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/events/ClassPrepare/classprep01/classprep01.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/events/ClassPrepare/classprep01/libclassprep01.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/events/Exception/exception01/exception01.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/events/Exception/exception01/exception01a.jasm create mode 100644 test/hotspot/jtreg/serviceability/jvmti/events/Exception/exception01/libexception01.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/events/ExceptionCatch/excatch01/excatch01.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/events/ExceptionCatch/excatch01/excatch01a.jasm create mode 100644 test/hotspot/jtreg/serviceability/jvmti/events/ExceptionCatch/excatch01/libexcatch01.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/events/FieldAccess/fieldacc01/fieldacc01.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/events/FieldAccess/fieldacc01/fieldacc01a.jasm create mode 100644 test/hotspot/jtreg/serviceability/jvmti/events/FieldAccess/fieldacc01/libfieldacc01.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/events/FieldAccess/fieldacc02/fieldacc02.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/events/FieldAccess/fieldacc02/libfieldacc02.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/events/FieldAccess/fieldacc03/fieldacc03.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/events/FieldAccess/fieldacc03/libfieldacc03.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/events/FieldAccess/fieldacc04/fieldacc04.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/events/FieldAccess/fieldacc04/libfieldacc04.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/events/FieldModification/fieldmod01/fieldmod01.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/events/FieldModification/fieldmod01/fieldmod01a.jasm create mode 100644 test/hotspot/jtreg/serviceability/jvmti/events/FieldModification/fieldmod01/libfieldmod01.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/events/FieldModification/fieldmod02/fieldmod02.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/events/FieldModification/fieldmod02/libfieldmod02.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/events/FramePop/framepop01/framepop01.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/events/FramePop/framepop01/framepop01a.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/events/FramePop/framepop01/libframepop01.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/events/FramePop/framepop02/framepop02.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/events/FramePop/framepop02/libframepop02.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/events/MethodEntry/mentry01/libmentry01.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/events/MethodEntry/mentry01/mentry01.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/events/MethodEntry/mentry02/libmentry02.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/events/MethodEntry/mentry02/mentry02.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/events/MethodExit/mexit01/libmexit01.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/events/MethodExit/mexit01/mexit01.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/events/MethodExit/mexit01/mexit01a.jasm create mode 100644 test/hotspot/jtreg/serviceability/jvmti/events/MethodExit/mexit02/libmexit02.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/events/MethodExit/mexit02/mexit02.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/events/MethodExit/mexit02/mexit02a.jasm create mode 100644 test/hotspot/jtreg/serviceability/jvmti/events/MonitorContendedEnter/mcontenter01/libmcontenter01.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/events/MonitorContendedEnter/mcontenter01/mcontenter01.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/events/MonitorContendedEntered/mcontentered01/libmcontentered01.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/events/MonitorContendedEntered/mcontentered01/mcontentered01.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/events/MonitorWait/monitorwait01/libmonitorwait01.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/events/MonitorWait/monitorwait01/monitorwait01.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/events/MonitorWaited/monitorwaited01/libmonitorwaited01.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/events/MonitorWaited/monitorwaited01/monitorwaited01.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/events/NativeMethodBind/nativemethbind01/libnativemethbind01.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/events/NativeMethodBind/nativemethbind01/nativemethbind01.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/events/NativeMethodBind/nativemethbind02/libnativemethbind02.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/events/NativeMethodBind/nativemethbind02/nativemethbind02.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/events/NativeMethodBind/nativemethbind03/libnativemethbind03.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/events/NativeMethodBind/nativemethbind03/nativemethbind03.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/events/NativeMethodBind/nativemethbind04/libnativemethbind04.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/events/NativeMethodBind/nativemethbind04/nativemethbind04.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/events/SingleStep/singlestep01/libsinglestep01.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/events/SingleStep/singlestep01/singlestep01.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/events/SingleStep/singlestep02/libsinglestep02.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/events/SingleStep/singlestep02/singlestep02.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/events/SingleStep/singlestep03/libsinglestep03.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/events/SingleStep/singlestep03/singlestep03.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/events/ThreadEnd/threadend01/libthreadend01.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/events/ThreadEnd/threadend01/threadend01.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/events/ThreadEnd/threadend02/libthreadend02.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/events/ThreadEnd/threadend02/threadend02.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/events/ThreadStart/threadstart01/libthreadstart01.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/events/ThreadStart/threadstart01/threadstart01.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/events/ThreadStart/threadstart02/libthreadstart02.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/events/ThreadStart/threadstart02/threadstart02.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/events/ThreadStart/threadstart03/libthreadstart03.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/events/ThreadStart/threadstart03/threadstart03.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/events/VMObjectAlloc/vmobjalloc01/libvmobjalloc01.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/events/VMObjectAlloc/vmobjalloc01/vmobjalloc01.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/negative/GetAllThreadsNullTest/GetAllThreadsNullTest.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/negative/GetAllThreadsNullTest/libGetAllThreadsNullTest.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/negative/contmon03/contmon03.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/negative/contmon03/libcontmon03.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/negative/framecnt02/framecnt02.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/negative/framecnt02/libframecnt02.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/negative/framecnt03/framecnt03.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/negative/framecnt03/libframecnt03.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/negative/frameloc03/frameloc03.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/negative/frameloc03/libframeloc03.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/negative/getstacktr02/getstacktr02.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/negative/getstacktr02/libgetstacktr02.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/negative/getstacktr09/getstacktr09.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/negative/getstacktr09/libgetstacktr09.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/negative/thrinfo02/libthrinfo02.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/negative/thrinfo02/thrinfo02.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/negative/thrstat04/libthrstat04.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/negative/thrstat04/thrstat04.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/stress/StackTrace/NotSuspended/GetStackTraceNotSuspendedStressTest.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/stress/StackTrace/NotSuspended/libGetStackTraceNotSuspendedStress.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/stress/StackTrace/Suspended/GetStackTraceSuspendedStressTest.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/stress/StackTrace/Suspended/libGetStackTraceSuspendedStress.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/stress/ThreadLocalStorage/SetGetThreadLocalStorageStressTest/SetGetThreadLocalStorageStressTest.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/stress/ThreadLocalStorage/SetGetThreadLocalStorageStressTest/libSetGetThreadLocalStorageStress.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/thread/GetAllThreads/allthr01/allthr01.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/thread/GetAllThreads/allthr01/liballthr01.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/thread/GetCurrentContendedMonitor/contmon01/contmon01.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/thread/GetCurrentContendedMonitor/contmon01/libcontmon01.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/thread/GetCurrentContendedMonitor/contmon02/contmon02.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/thread/GetCurrentContendedMonitor/contmon02/libcontmon02.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/thread/GetFrameCount/framecnt01/framecnt01.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/thread/GetFrameCount/framecnt01/libframecnt01.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/thread/GetFrameLocation/frameloc01/frameloc01.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/thread/GetFrameLocation/frameloc01/frameloc01a.jasm create mode 100644 test/hotspot/jtreg/serviceability/jvmti/thread/GetFrameLocation/frameloc01/libframeloc01.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/thread/GetFrameLocation/frameloc02/frameloc02.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/thread/GetFrameLocation/frameloc02/libframeloc02.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/thread/GetStackTrace/GetStackTraceCurrentThreadTest/GetStackTraceCurrentThreadTest.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/thread/GetStackTrace/GetStackTraceCurrentThreadTest/libGetStackTraceCurrentThreadTest.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/thread/GetStackTrace/get_stack_trace.h create mode 100644 test/hotspot/jtreg/serviceability/jvmti/thread/GetStackTrace/getstacktr03/getstacktr03.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/thread/GetStackTrace/getstacktr03/libgetstacktr03.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/thread/GetStackTrace/getstacktr04/getstacktr04.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/thread/GetStackTrace/getstacktr04/libgetstacktr04.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/thread/GetStackTrace/getstacktr05/getstacktr05.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/thread/GetStackTrace/getstacktr05/libgetstacktr05.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/thread/GetStackTrace/getstacktr06/getstacktr06.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/thread/GetStackTrace/getstacktr06/libgetstacktr06.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/thread/GetStackTrace/getstacktr07/getstacktr07.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/thread/GetStackTrace/getstacktr07/libgetstacktr07.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/thread/GetStackTrace/getstacktr08/getstacktr08.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/thread/GetStackTrace/getstacktr08/libgetstacktr08.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/thread/GetThreadInfo/thrinfo01/libthrinfo01.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/thread/GetThreadInfo/thrinfo01/thrinfo01.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/thread/GetThreadState/thrstat01/libthrstat01.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/thread/GetThreadState/thrstat01/thrstat01.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/thread/GetThreadState/thrstat02/libthrstat02.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/thread/GetThreadState/thrstat02/thrstat02.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/thread/GetThreadState/thrstat03/libthrstat03.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/thread/GetThreadState/thrstat03/thrstat03.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/thread/GetThreadState/thrstat05/libthrstat05.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/thread/GetThreadState/thrstat05/thrstat05.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/thread/ResumeThread/resumethrd01/libresumethrd01.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/thread/ResumeThread/resumethrd01/resumethrd01.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/thread/ResumeThread/resumethrd02/libresumethrd02.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/thread/ResumeThread/resumethrd02/resumethrd02.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/thread/ResumeThreadList/resumethrdlst01/libresumethrdlst01.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/thread/ResumeThreadList/resumethrdlst01/resumethrdlst01.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/thread/ResumeThreadList/resumethrdlst02/libresumethrdlst02.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/thread/ResumeThreadList/resumethrdlst02/resumethrdlst02.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/thread/SuspendThread/suspendthrd01/libsuspendthrd01.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/thread/SuspendThread/suspendthrd01/suspendthrd01.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/thread/SuspendThread/suspendthrd02/libsuspendthrd02.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/thread/SuspendThread/suspendthrd02/suspendthrd02.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/thread/SuspendThread/suspendthrd03/libsuspendthrd03.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/thread/SuspendThread/suspendthrd03/suspendthrd03.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/thread/SuspendThreadList/suspendthrdlst01/libsuspendthrdlst01.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/thread/SuspendThreadList/suspendthrdlst01/suspendthrdlst01.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/thread/SuspendThreadList/suspendthrdlst02/libsuspendthrdlst02.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/thread/SuspendThreadList/suspendthrdlst02/suspendthrdlst02.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/vthread/BreakpointInYieldTest/BreakpointInYieldTest.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/vthread/BreakpointInYieldTest/libBreakpointInYieldTest.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/vthread/ContFramePopTest/ContFramePopTest.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/vthread/ContFramePopTest/libContFramePopTest.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/vthread/ContStackDepthTest/ContStackDepthTest.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/vthread/ContStackDepthTest/libContStackDepthTest.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/vthread/ContYieldBreakPointTest/ContYieldBreakPointTest.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/vthread/ContYieldBreakPointTest/libContYieldBreakPointTest.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/vthread/ContinuationTest/ContinuationTest.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/vthread/ContinuationTest/libContinuationTest.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/vthread/GetSetLocalTest/GetSetLocalTest.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/vthread/GetSetLocalTest/libGetSetLocalTest.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/vthread/InterruptThreadTest/InterruptThreadTest.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/vthread/InterruptThreadTest/libInterruptThreadTest.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/vthread/MethodExitTest/MethodExitTest.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/vthread/MethodExitTest/libMethodExitTest.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/vthread/NullAsCurrentThreadTest/NullAsCurrentThreadTest.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/vthread/NullAsCurrentThreadTest/libNullAsCurrentThreadTest.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/vthread/PinnedTaskTest/PinnedTaskTest.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/vthread/PinnedTaskTest/libPinnedTaskTest.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/vthread/RawMonitorTest/RawMonitorTest.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/vthread/RawMonitorTest/libRawMonitorTest.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/vthread/RedefineClasses/RedefineRunningMethods.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/vthread/SelfSuspendDisablerTest/SelfSuspendDisablerTest.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/vthread/SelfSuspendDisablerTest/libSelfSuspendDisablerTest.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/vthread/SuspendResume1/SuspendResume1.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/vthread/SuspendResume1/libSuspendResume1.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/vthread/SuspendResume2/SuspendResume2.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/vthread/SuspendResume2/libSuspendResume2.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/vthread/SuspendResumeAll/SuspendResumeAll.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/vthread/SuspendResumeAll/libSuspendResumeAll.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/vthread/VThreadMonitorTest/VThreadMonitorTest.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/vthread/VThreadMonitorTest/libVThreadMonitorTest.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/vthread/VThreadNotifyFramePopTest/VThreadNotifyFramePopTest.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/vthread/VThreadNotifyFramePopTest/libVThreadNotifyFramePopTest.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/vthread/VThreadTest/VThreadTest.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/vthread/VThreadTest/libVThreadTest.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/vthread/VThreadUnsupportedTest/VThreadUnsupportedTest.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/vthread/VThreadUnsupportedTest/libVThreadUnsupportedTest.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/vthread/WaitNotifySuspendedVThreadTest/WaitNotifySuspendedVThreadTest.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/vthread/WaitNotifySuspendedVThreadTest/libWaitNotifySuspendedVThread.cpp create mode 100644 test/hotspot/jtreg/serviceability/jvmti/vthread/premain/AgentWithVThread.java create mode 100644 test/hotspot/jtreg/serviceability/jvmti/vthread/premain/AgentWithVThreadTest.java create mode 100644 test/hotspot/jtreg/vmTestbase/nsk/jdb/threads/threads003/threads003.java create mode 100644 test/hotspot/jtreg/vmTestbase/nsk/jdb/threads/threads003/threads003a.java create mode 100644 test/hotspot/jtreg/vmTestbase/nsk/jvmti/RedefineClasses/StressRedefineVirtual/StressRedefineVirtual.java create mode 100644 test/hotspot/jtreg/vmTestbase/nsk/share/MainWrapper.java create mode 100644 test/jdk/ProblemList-svc-vthread.txt create mode 100644 test/jdk/com/sun/management/HotSpotDiagnosticMXBean/DumpThreads.java create mode 100644 test/jdk/java/lang/Thread/BuilderTest.java create mode 100644 test/jdk/java/lang/Thread/JoinWithDuration.java create mode 100644 test/jdk/java/lang/Thread/SleepWithDuration.java create mode 100644 test/jdk/java/lang/Thread/virtual/Collectable.java create mode 100644 test/jdk/java/lang/Thread/virtual/CustomScheduler.java create mode 100644 test/jdk/java/lang/Thread/virtual/GetStackTrace.java create mode 100644 test/jdk/java/lang/Thread/virtual/GetStackTraceWhenRunnable.java create mode 100644 test/jdk/java/lang/Thread/virtual/HoldsLock.java create mode 100644 test/jdk/java/lang/Thread/virtual/JfrEvents.java create mode 100644 test/jdk/java/lang/Thread/virtual/Locking.java create mode 100644 test/jdk/java/lang/Thread/virtual/ParkWithFixedThreadPool.java create mode 100644 test/jdk/java/lang/Thread/virtual/Parking.java create mode 100644 test/jdk/java/lang/Thread/virtual/PreviewFeaturesNotEnabled.java create mode 100644 test/jdk/java/lang/Thread/virtual/Reflection.java create mode 100644 test/jdk/java/lang/Thread/virtual/ShutdownHook.java create mode 100644 test/jdk/java/lang/Thread/virtual/StackTraces.java create mode 100644 test/jdk/java/lang/Thread/virtual/ThreadAPI.java create mode 100644 test/jdk/java/lang/Thread/virtual/ThreadBuilders.java create mode 100644 test/jdk/java/lang/Thread/virtual/ThreadLocals.java create mode 100644 test/jdk/java/lang/Thread/virtual/TracePinnedThreads.java create mode 100644 test/jdk/java/lang/Thread/virtual/WaitNotify.java create mode 100644 test/jdk/java/lang/Thread/virtual/stress/GetStackTraceALot.java create mode 100644 test/jdk/java/lang/Thread/virtual/stress/PinALot.java create mode 100644 test/jdk/java/lang/Thread/virtual/stress/PingPong.java create mode 100644 test/jdk/java/lang/Thread/virtual/stress/Skynet.java create mode 100644 test/jdk/java/lang/Thread/virtual/stress/SleepALot.java create mode 100644 test/jdk/java/lang/Thread/virtual/stress/TEST.properties create mode 100644 test/jdk/java/lang/Thread/virtual/stress/TimedGet.java create mode 100644 test/jdk/java/lang/Thread/virtual/stress/YieldALot.java create mode 100644 test/jdk/java/lang/Thread/virtual/stress/skynet.md create mode 100644 test/jdk/java/lang/ThreadGroup/BasicTests.java delete mode 100644 test/jdk/java/lang/ThreadGroup/Daemon.java delete mode 100644 test/jdk/java/lang/ThreadGroup/Destroy.java delete mode 100644 test/jdk/java/lang/ThreadGroup/NullThreadName.java delete mode 100644 test/jdk/java/lang/ThreadGroup/SetMaxPriority.java delete mode 100644 test/jdk/java/lang/ThreadGroup/Stop.java delete mode 100644 test/jdk/java/lang/ThreadGroup/Suspend.java create mode 100644 test/jdk/java/lang/Throwable/OverrideFillInStackTrace.java delete mode 100644 test/jdk/java/lang/instrument/ParallelTransformerLoader.sh delete mode 100644 test/jdk/java/lang/instrument/ParallelTransformerLoaderApp.java create mode 100644 test/jdk/java/lang/instrument/ParallelTransformerLoaderTest.java create mode 100644 test/jdk/java/lang/management/ThreadMXBean/VirtualThreadDeadlocks.java create mode 100644 test/jdk/java/lang/management/ThreadMXBean/VirtualThreads.java create mode 100644 test/jdk/java/net/vthread/BlockingSocketOps.java create mode 100644 test/jdk/java/net/vthread/HttpALot.java create mode 100644 test/jdk/java/net/vthread/InterruptHttp.java create mode 100644 test/jdk/java/nio/channels/vthread/BlockingChannelOps.java create mode 100644 test/jdk/java/util/concurrent/ExecutorService/CloseTest.java create mode 100644 test/jdk/java/util/concurrent/Future/DefaultMethods.java create mode 100644 test/jdk/java/util/concurrent/ThreadPerTaskExecutor/ThreadPerTaskExecutorTest.java create mode 100644 test/jdk/java/util/concurrent/lib/DelegatingExecutorService.java create mode 100644 test/jdk/jdk/internal/misc/ThreadFlock/ThreadFlockTest.java create mode 100644 test/jdk/jdk/internal/vm/Continuation/Basic.java create mode 100644 test/jdk/jdk/internal/vm/Continuation/ClassUnloading.java create mode 100644 test/jdk/jdk/internal/vm/Continuation/Fuzz.java create mode 100644 test/jdk/jdk/internal/vm/Continuation/HumongousStack.java create mode 100644 test/jdk/jdk/internal/vm/Continuation/LiveFramesDriver.java create mode 100644 test/jdk/jdk/internal/vm/Continuation/Scoped.java create mode 100644 test/jdk/jdk/internal/vm/Continuation/TEST.properties create mode 100644 test/jdk/jdk/internal/vm/Continuation/fuzz.dat create mode 100644 test/jdk/jdk/internal/vm/Continuation/java.base/java/lang/LiveFrames.java create mode 100644 test/jdk/jdk/internal/vm/Continuation/java.base/java/lang/StackWalkerHelper.java create mode 100644 test/jdk/jdk/internal/vm/Continuation/libBasicJNI.c create mode 100644 test/jdk/jdk/jfr/event/runtime/TestVirtualThreadEndEvent.java create mode 100644 test/jdk/jdk/jfr/event/runtime/TestVirtualThreadStartEvent.java create mode 100644 test/jdk/jdk/jfr/jvm/LatchedThread.java create mode 100644 test/jdk/jdk/jfr/jvm/TestVirtualThreadExclusion.java create mode 100644 test/jdk/jdk/jfr/threading/TestDeepVirtualStackTrace.java create mode 100644 test/jdk/jdk/jfr/threading/TestManyVirtualThreads.java create mode 100644 test/jdk/jdk/jfr/threading/TestNestedVirtualThreads.java create mode 100644 test/lib/jdk/test/lib/jvmti/DebugeeClass.java create mode 100644 test/lib/jdk/test/lib/jvmti/jvmti_common.h create mode 100644 test/lib/jdk/test/lib/jvmti/jvmti_thread.h create mode 100644 test/lib/jdk/test/lib/thread/VThreadRunner.java create mode 100644 test/micro/org/openjdk/bench/loom/Skynet.java create mode 100644 test/micro/org/openjdk/bench/loom/obsolete/FreezeAndThaw.java create mode 100644 test/micro/org/openjdk/bench/loom/obsolete/OneShot.java create mode 100644 test/micro/org/openjdk/bench/loom/obsolete/Oscillation.java create mode 100644 test/micro/org/openjdk/bench/loom/ring/Channel.java create mode 100644 test/micro/org/openjdk/bench/loom/ring/Channels.java create mode 100644 test/micro/org/openjdk/bench/loom/ring/Ring.java create mode 100644 test/micro/org/openjdk/bench/loom/skynet.md diff --git a/make/data/hotspot-symbols/symbols-unix b/make/data/hotspot-symbols/symbols-unix index d51fcb3727d..48a51f8c940 100644 --- a/make/data/hotspot-symbols/symbols-unix +++ b/make/data/hotspot-symbols/symbols-unix @@ -46,7 +46,9 @@ JVM_ConstantPoolGetSize JVM_ConstantPoolGetStringAt JVM_ConstantPoolGetTagAt JVM_ConstantPoolGetUTF8At +JVM_CurrentCarrierThread JVM_CurrentThread +JVM_SetCurrentThread JVM_CurrentTimeMillis JVM_DefineClass JVM_DefineClassWithSource @@ -120,6 +122,7 @@ JVM_GetMethodTypeAnnotations JVM_GetNanoTimeAdjustment JVM_GetNestHost JVM_GetNestMembers +JVM_GetNextThreadIdOffset JVM_GetPermittedSubclasses JVM_GetPrimitiveArrayElement JVM_GetProperties @@ -135,6 +138,7 @@ JVM_GetVmArguments JVM_Halt JVM_HasReferencePendingList JVM_HoldsLock +JVM_GetStackTrace JVM_IHashCode JVM_InitClassName JVM_InitStackTraceElement @@ -150,6 +154,7 @@ JVM_IsDumpingClassList JVM_IsFinalizationEnabled JVM_IsHiddenClass JVM_IsInterface +JVM_IsPreviewEnabled JVM_IsPrimitiveClass JVM_IsRecord JVM_IsSameClassPackage @@ -182,15 +187,19 @@ JVM_RawMonitorEnter JVM_RawMonitorExit JVM_ReferenceClear JVM_ReferenceRefersTo +JVM_RegisterContinuationMethods JVM_RegisterLambdaProxyClassForArchiving JVM_RegisterSignal JVM_ReleaseUTF JVM_ReportFinalizationComplete JVM_ResumeThread +JVM_ExtentLocalCache +JVM_SetExtentLocalCache JVM_SetArrayElement JVM_SetClassSigners JVM_SetNativeThreadName JVM_SetPrimitiveArrayElement +JVM_SetStackWalkContinuation JVM_SetThreadPriority JVM_Sleep JVM_StartThread @@ -210,3 +219,10 @@ JVM_AddReadsModule JVM_DefineArchivedModules JVM_DefineModule JVM_SetBootLoaderUnnamedModule + +# Virtual thread notifications for JVMTI +JVM_VirtualThreadMountBegin +JVM_VirtualThreadMountEnd +JVM_VirtualThreadUnmountBegin +JVM_VirtualThreadUnmountEnd +# diff --git a/make/test/BuildMicrobenchmark.gmk b/make/test/BuildMicrobenchmark.gmk index 72ee7523f0e..0a0bd39cb69 100644 --- a/make/test/BuildMicrobenchmark.gmk +++ b/make/test/BuildMicrobenchmark.gmk @@ -95,8 +95,12 @@ $(eval $(call SetupJavaCompilation, BUILD_JDK_MICROBENCHMARK, \ SRC := $(MICROBENCHMARK_SRC), \ BIN := $(MICROBENCHMARK_CLASSES), \ JAVAC_FLAGS := --add-exports java.base/sun.security.util=ALL-UNNAMED \ - --add-exports java.base/sun.invoke.util=ALL-UNNAMED, \ - JAVA_FLAGS := --add-modules jdk.unsupported --limit-modules java.management, \ + --add-exports java.base/sun.invoke.util=ALL-UNNAMED \ + --add-exports java.base/jdk.internal.vm=ALL-UNNAMED \ + --enable-preview, \ + JAVA_FLAGS := --add-modules jdk.unsupported --limit-modules java.management \ + --add-exports java.base/jdk.internal.vm=ALL-UNNAMED \ + --enable-preview, \ )) $(BUILD_JDK_MICROBENCHMARK): $(JMH_COMPILE_JARS) diff --git a/make/test/JtregNativeHotspot.gmk b/make/test/JtregNativeHotspot.gmk index 58390110251..6b5c2770af1 100644 --- a/make/test/JtregNativeHotspot.gmk +++ b/make/test/JtregNativeHotspot.gmk @@ -1,5 +1,5 @@ # -# Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2015, 2022, Oracle and/or its affiliates. All rights reserved. # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. # # This code is free software; you can redistribute it and/or modify it @@ -148,8 +148,10 @@ BUILD_HOTSPOT_JTREG_LIBRARIES_CFLAGS_libNoFramePointer := $(NO_FRAMEPOINTER_CFLA # Optimization -O3 needed, HIGH == -O3 BUILD_HOTSPOT_JTREG_LIBRARIES_OPTIMIZATION_libNoFramePointer := HIGH -BUILD_HOTSPOT_JTREG_LIBRARIES_CFLAGS := -D__STDC_FORMAT_MACROS -D__STDC_LIMIT_MACROS -D__STDC_CONSTANT_MACROS -BUILD_HOTSPOT_JTREG_EXECUTABLES_CFLAGS := -D__STDC_FORMAT_MACROS -D__STDC_LIMIT_MACROS -D__STDC_CONSTANT_MACROS +JVMTI_COMMON_INCLUDES=-I$(TOPDIR)/test/lib/jdk/test/lib/jvmti + +BUILD_HOTSPOT_JTREG_LIBRARIES_CFLAGS := -D__STDC_FORMAT_MACROS -D__STDC_LIMIT_MACROS -D__STDC_CONSTANT_MACROS $(JVMTI_COMMON_INCLUDES) +BUILD_HOTSPOT_JTREG_EXECUTABLES_CFLAGS := -D__STDC_FORMAT_MACROS -D__STDC_LIMIT_MACROS -D__STDC_CONSTANT_MACROS $(JVMTI_COMMON_INCLUDES) BUILD_HOTSPOT_JTREG_LIBRARIES_CFLAGS_libProcessUtils := $(VM_SHARE_INCLUDES) @@ -346,6 +348,7 @@ BUILD_HOTSPOT_JTREG_LIBRARIES_CFLAGS_libvminit001 := $(NSK_JVMTI_AGENT_INCLUDES) BUILD_HOTSPOT_JTREG_LIBRARIES_CFLAGS_libsuspendthrd001 := $(NSK_JVMTI_AGENT_INCLUDES) BUILD_HOTSPOT_JTREG_LIBRARIES_CFLAGS_libsuspendthrd002 := $(NSK_JVMTI_AGENT_INCLUDES) BUILD_HOTSPOT_JTREG_LIBRARIES_CFLAGS_libsuspendthrd003 := $(NSK_JVMTI_AGENT_INCLUDES) +BUILD_HOTSPOT_JTREG_LIBRARIES_CFLAGS_libsuspendvthr001 := $(NSK_JVMTI_AGENT_INCLUDES) BUILD_HOTSPOT_JTREG_LIBRARIES_CFLAGS_libdeclcls002 := $(NSK_JVMTI_AGENT_INCLUDES) BUILD_HOTSPOT_JTREG_LIBRARIES_CFLAGS_libdeclcls003 := $(NSK_JVMTI_AGENT_INCLUDES) BUILD_HOTSPOT_JTREG_LIBRARIES_CFLAGS_libdeclcls001 := $(NSK_JVMTI_AGENT_INCLUDES) @@ -1012,6 +1015,7 @@ else BUILD_HOTSPOT_JTREG_LIBRARIES_LIBS_libsuspendthrd001 += -lpthread BUILD_HOTSPOT_JTREG_LIBRARIES_LIBS_libsuspendthrd002 += -lpthread BUILD_HOTSPOT_JTREG_LIBRARIES_LIBS_libsuspendthrd003 += -lpthread + BUILD_HOTSPOT_JTREG_LIBRARIES_LIBS_libsuspendvthr001 += -lpthread BUILD_HOTSPOT_JTREG_LIBRARIES_LIBS_libdeclcls002 += -lpthread BUILD_HOTSPOT_JTREG_LIBRARIES_LIBS_libdeclcls003 += -lpthread BUILD_HOTSPOT_JTREG_LIBRARIES_LIBS_libdeclcls001 += -lpthread diff --git a/src/hotspot/cpu/aarch64/aarch64.ad b/src/hotspot/cpu/aarch64/aarch64.ad index 99128351c94..dad07555906 100644 --- a/src/hotspot/cpu/aarch64/aarch64.ad +++ b/src/hotspot/cpu/aarch64/aarch64.ad @@ -3829,6 +3829,8 @@ encode %{ } } + __ post_call_nop(); + // Only non uncommon_trap calls need to reinitialize ptrue. if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) { __ reinitialize_ptrue(); @@ -3842,7 +3844,9 @@ encode %{ if (call == NULL) { ciEnv::current()->record_failure("CodeCache is full"); return; - } else if (Compile::current()->max_vector_size() > 0) { + } + __ post_call_nop(); + if (Compile::current()->max_vector_size() > 0) { __ reinitialize_ptrue(); } %} @@ -3870,6 +3874,7 @@ encode %{ ciEnv::current()->record_failure("CodeCache is full"); return; } + __ post_call_nop(); } else { Label retaddr; __ adr(rscratch2, retaddr); @@ -3878,6 +3883,7 @@ encode %{ __ stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize))); __ blr(rscratch1); __ bind(retaddr); + __ post_call_nop(); __ add(sp, sp, 2 * wordSize); } if (Compile::current()->max_vector_size() > 0) { diff --git a/src/hotspot/cpu/aarch64/abstractInterpreter_aarch64.cpp b/src/hotspot/cpu/aarch64/abstractInterpreter_aarch64.cpp index 61630e37b6b..c2aaca99143 100644 --- a/src/hotspot/cpu/aarch64/abstractInterpreter_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/abstractInterpreter_aarch64.cpp @@ -123,9 +123,9 @@ void AbstractInterpreter::layout_activation(Method* method, // It is also guaranteed to be walkable even though it is in a // skeletal state - int max_locals = method->max_locals() * Interpreter::stackElementWords; - int extra_locals = (method->max_locals() - method->size_of_parameters()) * - Interpreter::stackElementWords; + const int max_locals = method->max_locals() * Interpreter::stackElementWords; + const int params = method->size_of_parameters() * Interpreter::stackElementWords; + const int extra_locals = max_locals - params; #ifdef ASSERT assert(caller->sp() == interpreter_frame->sender_sp(), "Frame not properly walkable"); @@ -144,12 +144,9 @@ void AbstractInterpreter::layout_activation(Method* method, // align the incoming parameters with the caller's temporary // expression stack. For other types of caller frame it doesn't // matter. - intptr_t* locals; - if (caller->is_interpreted_frame()) { - locals = caller->interpreter_frame_last_sp() + caller_actual_parameters - 1; - } else { - locals = interpreter_frame->sender_sp() + max_locals - 1; - } + intptr_t* const locals = caller->is_interpreted_frame() + ? caller->interpreter_frame_last_sp() + caller_actual_parameters - 1 + : interpreter_frame->sender_sp() + max_locals - 1; #ifdef ASSERT if (caller->is_interpreted_frame()) { @@ -171,14 +168,10 @@ void AbstractInterpreter::layout_activation(Method* method, // All frames but the initial (oldest) interpreter frame we fill in have // a value for sender_sp that allows walking the stack but isn't // truly correct. Correct the value here. - if (extra_locals != 0 && - interpreter_frame->sender_sp() == - interpreter_frame->interpreter_frame_sender_sp()) { - interpreter_frame->set_interpreter_frame_sender_sp(caller->sp() + - extra_locals); + if (extra_locals != 0 && interpreter_frame->sender_sp() == interpreter_frame->interpreter_frame_sender_sp()) { + interpreter_frame->set_interpreter_frame_sender_sp(caller->sp() + extra_locals); } - *interpreter_frame->interpreter_frame_cache_addr() = - method->constants()->cache(); - *interpreter_frame->interpreter_frame_mirror_addr() = - method->method_holder()->java_mirror(); + + *interpreter_frame->interpreter_frame_cache_addr() = method->constants()->cache(); + *interpreter_frame->interpreter_frame_mirror_addr() = method->method_holder()->java_mirror(); } diff --git a/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp index dfb89ca762d..9e0bac8c8c2 100644 --- a/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp @@ -438,6 +438,7 @@ int LIR_Assembler::emit_unwind_handler() { __ unlock_object(r5, r4, r0, *stub->entry()); } __ bind(*stub->continuation()); + __ dec_held_monitor_count(rthread); } if (compilation()->env()->dtrace_method_probes()) { @@ -2037,6 +2038,7 @@ void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) { return; } add_call_info(code_offset(), op->info()); + __ post_call_nop(); } @@ -2047,6 +2049,7 @@ void LIR_Assembler::ic_call(LIR_OpJavaCall* op) { return; } add_call_info(code_offset(), op->info()); + __ post_call_nop(); } void LIR_Assembler::emit_static_call_stub() { @@ -2571,7 +2574,18 @@ void LIR_Assembler::emit_lock(LIR_OpLock* op) { } else { Unimplemented(); } + if (op->code() == lir_lock) { + // If deoptimization happens in Runtime1::monitorenter, inc_held_monitor_count after backing from slowpath + // will be skipped. Solution is: + // 1. Increase only in fastpath + // 2. Runtime1::monitorenter increase count after locking + __ inc_held_monitor_count(rthread); + } __ bind(*op->stub()->continuation()); + if (op->code() == lir_unlock) { + // unlock in slowpath is JRT_Leaf stub, no deoptimization can happen + __ dec_held_monitor_count(rthread); + } } void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) { @@ -2899,6 +2913,7 @@ void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* arg if (info != NULL) { add_call_info_here(info); } + __ post_call_nop(); } void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) { diff --git a/src/hotspot/cpu/aarch64/c1_LIRGenerator_aarch64.cpp b/src/hotspot/cpu/aarch64/c1_LIRGenerator_aarch64.cpp index 26298f24440..1250f96e077 100644 --- a/src/hotspot/cpu/aarch64/c1_LIRGenerator_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/c1_LIRGenerator_aarch64.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2005, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2005, 2022, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2014, Red Hat Inc. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * @@ -338,6 +338,17 @@ void LIRGenerator::do_MonitorExit(MonitorExit* x) { monitor_exit(obj_temp, lock, syncTempOpr(), LIR_OprFact::illegalOpr, x->monitor_no()); } +void LIRGenerator::do_continuation_doYield(Intrinsic* x) { + BasicTypeList signature(0); + CallingConvention* cc = frame_map()->java_calling_convention(&signature, true); + + const LIR_Opr result_reg = result_register_for(x->type()); + address entry = StubRoutines::cont_doYield(); + LIR_Opr result = rlock_result(x); + CodeEmitInfo* info = state_for(x, x->state()); + __ call_runtime(entry, LIR_OprFact::illegalOpr, result_reg, cc->args(), info); + __ move(result_reg, result); +} void LIRGenerator::do_NegateOp(NegateOp* x) { diff --git a/src/hotspot/cpu/aarch64/continuationEntry_aarch64.inline.hpp b/src/hotspot/cpu/aarch64/continuationEntry_aarch64.inline.hpp new file mode 100644 index 00000000000..c8abb8c319f --- /dev/null +++ b/src/hotspot/cpu/aarch64/continuationEntry_aarch64.inline.hpp @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_AARCH64_CONTINUATIONENTRY_AARCH64_INLINE_HPP +#define CPU_AARCH64_CONTINUATIONENTRY_AARCH64_INLINE_HPP + +#include "runtime/continuationEntry.hpp" + +#include "code/codeCache.hpp" +#include "runtime/frame.inline.hpp" +#include "runtime/registerMap.hpp" + +inline frame ContinuationEntry::to_frame() const { + static CodeBlob* cb = CodeCache::find_blob_fast(entry_pc()); + assert(cb != nullptr, ""); + assert(cb->as_compiled_method()->method()->is_continuation_enter_intrinsic(), ""); + return frame(entry_sp(), entry_sp(), entry_fp(), entry_pc(), cb); +} + +inline intptr_t* ContinuationEntry::entry_fp() const { + return (intptr_t*)((address)this + size()); +} + +inline void ContinuationEntry::update_register_map(RegisterMap* map) const { + intptr_t** fp = (intptr_t**)(bottom_sender_sp() - frame::sender_sp_offset); + frame::update_map_with_saved_link(map, fp); +} + + +#endif // CPU_AARCH64_CONTINUATIONENTRY_AARCH64_INLINE_HPP diff --git a/src/hotspot/cpu/aarch64/continuationFreezeThaw_aarch64.inline.hpp b/src/hotspot/cpu/aarch64/continuationFreezeThaw_aarch64.inline.hpp new file mode 100644 index 00000000000..8239c649283 --- /dev/null +++ b/src/hotspot/cpu/aarch64/continuationFreezeThaw_aarch64.inline.hpp @@ -0,0 +1,301 @@ +/* + * Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_AARCH64_CONTINUATIONFREEZETHAW_AARCH64_INLINE_HPP +#define CPU_AARCH64_CONTINUATIONFREEZETHAW_AARCH64_INLINE_HPP + +#include "code/codeBlob.inline.hpp" +#include "oops/stackChunkOop.inline.hpp" +#include "runtime/frame.hpp" +#include "runtime/frame.inline.hpp" + + +inline void patch_callee_link(const frame& f, intptr_t* fp) { + DEBUG_ONLY(intptr_t* orig = *ContinuationHelper::Frame::callee_link_address(f)); + *ContinuationHelper::Frame::callee_link_address(f) = fp; +} + +inline void patch_callee_link_relative(const frame& f, intptr_t* fp) { + intptr_t* la = (intptr_t*)ContinuationHelper::Frame::callee_link_address(f); + intptr_t new_value = fp - la; + *la = new_value; +} + +////// Freeze + +// Fast path + +inline void FreezeBase::patch_stack_pd(intptr_t* frame_sp, intptr_t* heap_sp) { + // copy the spilled fp from the heap to the stack + *(frame_sp - frame::sender_sp_offset) = *(heap_sp - frame::sender_sp_offset); +} + +// Slow path + +template +inline frame FreezeBase::sender(const frame& f) { + assert(FKind::is_instance(f), ""); + if (FKind::interpreted) { + return frame(f.sender_sp(), f.interpreter_frame_sender_sp(), f.link(), f.sender_pc()); + } + intptr_t** link_addr = link_address(f); + + intptr_t* sender_sp = (intptr_t*)(link_addr + frame::sender_sp_offset); // f.unextended_sp() + (fsize/wordSize); // + address sender_pc = (address) *(sender_sp-1); + assert(sender_sp != f.sp(), "must have changed"); + + int slot = 0; + CodeBlob* sender_cb = CodeCache::find_blob_and_oopmap(sender_pc, slot); + return sender_cb != nullptr + ? frame(sender_sp, sender_sp, *link_addr, sender_pc, sender_cb, + slot == -1 ? nullptr : sender_cb->oop_map_for_slot(slot, sender_pc), + false /* on_heap ? */) + : frame(sender_sp, sender_sp, *link_addr, sender_pc); +} + +template +frame FreezeBase::new_heap_frame(frame& f, frame& caller) { + assert(FKind::is_instance(f), ""); + assert(!caller.is_interpreted_frame() + || caller.unextended_sp() == (intptr_t*)caller.at(frame::interpreter_frame_last_sp_offset), ""); + + intptr_t *sp, *fp; // sp is really our unextended_sp + if (FKind::interpreted) { + assert((intptr_t*)f.at(frame::interpreter_frame_last_sp_offset) == nullptr + || f.unextended_sp() == (intptr_t*)f.at(frame::interpreter_frame_last_sp_offset), ""); + int locals = f.interpreter_frame_method()->max_locals(); + // If the caller.is_empty(), i.e. we're freezing into an empty chunk, then we set + // the chunk's argsize in finalize_freeze and make room for it above the unextended_sp + bool overlap_caller = caller.is_interpreted_frame() || caller.is_empty(); + fp = caller.unextended_sp() - (locals + frame::sender_sp_offset) + (overlap_caller ? ContinuationHelper::InterpretedFrame::stack_argsize(f) : 0); + sp = fp - (f.fp() - f.unextended_sp()); + assert(sp <= fp, ""); + assert(fp <= caller.unextended_sp(), ""); + caller.set_sp(fp + frame::sender_sp_offset); + + assert(_cont.tail()->is_in_chunk(sp), ""); + + frame hf(sp, sp, fp, f.pc(), nullptr, nullptr, true /* on_heap */); + *hf.addr_at(frame::interpreter_frame_locals_offset) = frame::sender_sp_offset + locals - 1; + return hf; + } else { + // We need to re-read fp out of the frame because it may be an oop and we might have + // had a safepoint in finalize_freeze, after constructing f. + fp = *(intptr_t**)(f.sp() - frame::sender_sp_offset); + + int fsize = FKind::size(f); + sp = caller.unextended_sp() - fsize; + if (caller.is_interpreted_frame()) { + // If the caller is interpreted, our stackargs are not supposed to overlap with it + // so we make more room by moving sp down by argsize + int argsize = FKind::stack_argsize(f); + sp -= argsize; + } + caller.set_sp(sp + fsize); + + assert(_cont.tail()->is_in_chunk(sp), ""); + + return frame(sp, sp, fp, f.pc(), nullptr, nullptr, true /* on_heap */); + } +} + +void FreezeBase::adjust_interpreted_frame_unextended_sp(frame& f) { + assert((f.at(frame::interpreter_frame_last_sp_offset) != 0) || (f.unextended_sp() == f.sp()), ""); + intptr_t* real_unextended_sp = (intptr_t*)f.at(frame::interpreter_frame_last_sp_offset); + if (real_unextended_sp != nullptr) { + f.set_unextended_sp(real_unextended_sp); // can be null at a safepoint + } +} + +static inline void relativize_one(intptr_t* const vfp, intptr_t* const hfp, int offset) { + assert(*(hfp + offset) == *(vfp + offset), ""); + intptr_t* addr = hfp + offset; + intptr_t value = *(intptr_t**)addr - vfp; + *addr = value; +} + +inline void FreezeBase::relativize_interpreted_frame_metadata(const frame& f, const frame& hf) { + intptr_t* vfp = f.fp(); + intptr_t* hfp = hf.fp(); + assert(hfp == hf.unextended_sp() + (f.fp() - f.unextended_sp()), ""); + assert((f.at(frame::interpreter_frame_last_sp_offset) != 0) + || (f.unextended_sp() == f.sp()), ""); + assert(f.fp() > (intptr_t*)f.at(frame::interpreter_frame_initial_sp_offset), ""); + + // on AARCH64, we may insert padding between the locals and the rest of the frame + // (see TemplateInterpreterGenerator::generate_normal_entry, and AbstractInterpreter::layout_activation) + // so we compute locals "from scratch" rather than relativizing the value in the stack frame, which might include padding, + // since we don't freeze the padding word (see recurse_freeze_interpreted_frame). + + // at(frame::interpreter_frame_last_sp_offset) can be NULL at safepoint preempts + *hf.addr_at(frame::interpreter_frame_last_sp_offset) = hf.unextended_sp() - hf.fp(); + *hf.addr_at(frame::interpreter_frame_locals_offset) = frame::sender_sp_offset + f.interpreter_frame_method()->max_locals() - 1; + + relativize_one(vfp, hfp, frame::interpreter_frame_initial_sp_offset); // == block_top == block_bottom + + assert((hf.fp() - hf.unextended_sp()) == (f.fp() - f.unextended_sp()), ""); + assert(hf.unextended_sp() == (intptr_t*)hf.at(frame::interpreter_frame_last_sp_offset), ""); + assert(hf.unextended_sp() <= (intptr_t*)hf.at(frame::interpreter_frame_initial_sp_offset), ""); + assert(hf.fp() > (intptr_t*)hf.at(frame::interpreter_frame_initial_sp_offset), ""); + assert(hf.fp() <= (intptr_t*)hf.at(frame::interpreter_frame_locals_offset), ""); +} + +inline void FreezeBase::set_top_frame_metadata_pd(const frame& hf) { + stackChunkOop chunk = _cont.tail(); + assert(chunk->is_in_chunk(hf.sp() - 1), ""); + assert(chunk->is_in_chunk(hf.sp() - frame::sender_sp_offset), ""); + + *(hf.sp() - 1) = (intptr_t)hf.pc(); + + intptr_t* fp_addr = hf.sp() - frame::sender_sp_offset; + *fp_addr = hf.is_interpreted_frame() ? (intptr_t)(hf.fp() - fp_addr) + : (intptr_t)hf.fp(); +} + +inline void FreezeBase::patch_pd(frame& hf, const frame& caller) { + if (caller.is_interpreted_frame()) { + assert(!caller.is_empty(), ""); + patch_callee_link_relative(caller, caller.fp()); + } else { + // If we're the bottom-most frame frozen in this freeze, the caller might have stayed frozen in the chunk, + // and its oop-containing fp fixed. We've now just overwritten it, so we must patch it back to its value + // as read from the chunk. + patch_callee_link(caller, caller.fp()); + } +} + +//////// Thaw + +// Fast path + +inline void ThawBase::prefetch_chunk_pd(void* start, int size) { + size <<= LogBytesPerWord; + Prefetch::read(start, size); + Prefetch::read(start, size - 64); +} + +void ThawBase::patch_chunk_pd(intptr_t* sp) { + intptr_t* fp = _cont.entryFP(); + *(intptr_t**)(sp - frame::sender_sp_offset) = fp; +} + +// Slow path + +inline frame ThawBase::new_entry_frame() { + intptr_t* sp = _cont.entrySP(); + return frame(sp, sp, _cont.entryFP(), _cont.entryPC()); // TODO PERF: This finds code blob and computes deopt state +} + +template frame ThawBase::new_stack_frame(const frame& hf, frame& caller, bool bottom) { + assert(FKind::is_instance(hf), ""); + // The values in the returned frame object will be written into the callee's stack in patch. + + if (FKind::interpreted) { + intptr_t* heap_sp = hf.unextended_sp(); + const int fsize = ContinuationHelper::InterpretedFrame::frame_bottom(hf) - hf.unextended_sp(); + const int locals = hf.interpreter_frame_method()->max_locals(); + intptr_t* frame_sp = caller.unextended_sp() - fsize; + intptr_t* fp = frame_sp + (hf.fp() - heap_sp); + int padding = 0; + if ((intptr_t)fp % frame::frame_alignment != 0) { + fp--; + frame_sp--; + padding++; + log_develop_trace(continuations)("Adding internal interpreted frame alignment"); + } + DEBUG_ONLY(intptr_t* unextended_sp = fp + *hf.addr_at(frame::interpreter_frame_last_sp_offset);) + assert(frame_sp == unextended_sp, ""); + caller.set_sp(fp + frame::sender_sp_offset); + frame f(frame_sp, frame_sp, fp, hf.pc()); + // it's set again later in set_interpreter_frame_bottom, but we need to set the locals now so that + // we could call ContinuationHelper::InterpretedFrame::frame_bottom + intptr_t offset = *hf.addr_at(frame::interpreter_frame_locals_offset); + assert((int)offset == frame::sender_sp_offset + locals - 1, ""); + // derelativize locals + *(intptr_t**)f.addr_at(frame::interpreter_frame_locals_offset) = fp + padding + offset; + assert((intptr_t)f.fp() % frame::frame_alignment == 0, ""); + return f; + } else { + int fsize = FKind::size(hf); + intptr_t* frame_sp = caller.unextended_sp() - fsize; + if (bottom || caller.is_interpreted_frame()) { + int argsize = hf.compiled_frame_stack_argsize(); + + fsize += argsize; + frame_sp -= argsize; + caller.set_sp(caller.sp() - argsize); + assert(caller.sp() == frame_sp + (fsize-argsize), ""); + + frame_sp = align(hf, frame_sp, caller, bottom); + } + + assert(hf.cb() != nullptr, ""); + assert(hf.oop_map() != nullptr, ""); + intptr_t* fp; + if (PreserveFramePointer) { + // we need to recreate a "real" frame pointer, pointing into the stack + fp = frame_sp + FKind::size(hf) - frame::sender_sp_offset; + } else { + fp = FKind::stub + ? frame_sp + fsize - frame::sender_sp_offset // on AArch64, this value is used for the safepoint stub + : *(intptr_t**)(hf.sp() - frame::sender_sp_offset); // we need to re-read fp because it may be an oop and we might have fixed the frame. + } + return frame(frame_sp, frame_sp, fp, hf.pc(), hf.cb(), hf.oop_map(), false); // TODO PERF : this computes deopt state; is it necessary? + } +} + +inline intptr_t* ThawBase::align(const frame& hf, intptr_t* frame_sp, frame& caller, bool bottom) { +#ifdef _LP64 + if (((intptr_t)frame_sp & 0xf) != 0) { + assert(caller.is_interpreted_frame() || (bottom && hf.compiled_frame_stack_argsize() % 2 != 0), ""); + frame_sp--; + caller.set_sp(caller.sp() - 1); + } + assert(is_aligned(frame_sp, frame::frame_alignment), ""); +#endif + + return frame_sp; +} + +inline void ThawBase::patch_pd(frame& f, const frame& caller) { + patch_callee_link(caller, caller.fp()); +} + +static inline void derelativize_one(intptr_t* const fp, int offset) { + intptr_t* addr = fp + offset; + *addr = (intptr_t)(fp + *addr); +} + +inline void ThawBase::derelativize_interpreted_frame_metadata(const frame& hf, const frame& f) { + intptr_t* vfp = f.fp(); + + derelativize_one(vfp, frame::interpreter_frame_last_sp_offset); + derelativize_one(vfp, frame::interpreter_frame_initial_sp_offset); +} + +inline void ThawBase::set_interpreter_frame_bottom(const frame& f, intptr_t* bottom) { + *(intptr_t**)f.addr_at(frame::interpreter_frame_locals_offset) = bottom - 1; +} + +#endif // CPU_AARCH64_CONTINUATIONFREEZETHAW_AARCH64_INLINE_HPP diff --git a/src/hotspot/cpu/aarch64/continuationHelper_aarch64.inline.hpp b/src/hotspot/cpu/aarch64/continuationHelper_aarch64.inline.hpp new file mode 100644 index 00000000000..47409027717 --- /dev/null +++ b/src/hotspot/cpu/aarch64/continuationHelper_aarch64.inline.hpp @@ -0,0 +1,140 @@ +/* + * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_AARCH64_CONTINUATIONHELPER_AARCH64_INLINE_HPP +#define CPU_AARCH64_CONTINUATIONHELPER_AARCH64_INLINE_HPP + +#include "runtime/continuationHelper.hpp" + +#include "runtime/continuationEntry.inline.hpp" +#include "runtime/frame.inline.hpp" +#include "runtime/registerMap.hpp" +#include "utilities/macros.hpp" + +template +static inline intptr_t** link_address(const frame& f) { + assert(FKind::is_instance(f), ""); + return FKind::interpreted + ? (intptr_t**)(f.fp() + frame::link_offset) + : (intptr_t**)(f.unextended_sp() + f.cb()->frame_size() - frame::sender_sp_offset); +} + +inline int ContinuationHelper::frame_align_words(int size) { +#ifdef _LP64 + return size & 1; +#else + return 0; +#endif +} + +inline intptr_t* ContinuationHelper::frame_align_pointer(intptr_t* sp) { +#ifdef _LP64 + sp = align_down(sp, frame::frame_alignment); +#endif + return sp; +} + +template +inline void ContinuationHelper::update_register_map(const frame& f, RegisterMap* map) { + frame::update_map_with_saved_link(map, link_address(f)); +} + +inline void ContinuationHelper::update_register_map_with_callee(const frame& f, RegisterMap* map) { + frame::update_map_with_saved_link(map, ContinuationHelper::Frame::callee_link_address(f)); +} + +inline void ContinuationHelper::push_pd(const frame& f) { + *(intptr_t**)(f.sp() - frame::sender_sp_offset) = f.fp(); +} + + +inline void ContinuationHelper::set_anchor_to_entry_pd(JavaFrameAnchor* anchor, ContinuationEntry* entry) { + anchor->set_last_Java_fp(entry->entry_fp()); +} + +#ifdef ASSERT +inline void ContinuationHelper::set_anchor_pd(JavaFrameAnchor* anchor, intptr_t* sp) { + intptr_t* fp = *(intptr_t**)(sp - frame::sender_sp_offset); + anchor->set_last_Java_fp(fp); +} + +inline bool ContinuationHelper::Frame::assert_frame_laid_out(frame f) { + intptr_t* sp = f.sp(); + address pc = *(address*)(sp - frame::sender_sp_ret_address_offset()); + intptr_t* fp = *(intptr_t**)(sp - frame::sender_sp_offset); + assert(f.raw_pc() == pc, "f.ra_pc: " INTPTR_FORMAT " actual: " INTPTR_FORMAT, p2i(f.raw_pc()), p2i(pc)); + assert(f.fp() == fp, "f.fp: " INTPTR_FORMAT " actual: " INTPTR_FORMAT, p2i(f.fp()), p2i(fp)); + return f.raw_pc() == pc && f.fp() == fp; +} +#endif + +inline intptr_t** ContinuationHelper::Frame::callee_link_address(const frame& f) { + return (intptr_t**)(f.sp() - frame::sender_sp_offset); +} + +inline address* ContinuationHelper::Frame::return_pc_address(const frame& f) { + return (address*)(f.real_fp() - 1); +} + +inline address* ContinuationHelper::InterpretedFrame::return_pc_address(const frame& f) { + return (address*)(f.fp() + frame::return_addr_offset); +} + +inline void ContinuationHelper::InterpretedFrame::patch_sender_sp(frame& f, intptr_t* sp) { + assert(f.is_interpreted_frame(), ""); + intptr_t* la = f.addr_at(frame::interpreter_frame_sender_sp_offset); + *la = f.is_heap_frame() ? (intptr_t)(sp - f.fp()) : (intptr_t)sp; +} + +inline address ContinuationHelper::Frame::real_pc(const frame& f) { + address* pc_addr = &(((address*) f.sp())[-1]); + return *pc_addr; +} + +inline void ContinuationHelper::Frame::patch_pc(const frame& f, address pc) { + address* pc_addr = &(((address*) f.sp())[-1]); + *pc_addr = pc; +} + +inline intptr_t* ContinuationHelper::InterpretedFrame::frame_top(const frame& f, InterpreterOopMap* mask) { // inclusive; this will be copied with the frame + // interpreter_frame_last_sp_offset, points to unextended_sp includes arguments in the frame + // interpreter_frame_initial_sp_offset excludes expression stack slots + int expression_stack_sz = expression_stack_size(f, mask); + intptr_t* res = *(intptr_t**)f.addr_at(frame::interpreter_frame_initial_sp_offset) - expression_stack_sz; + assert(res == (intptr_t*)f.interpreter_frame_monitor_end() - expression_stack_sz, ""); + assert(res >= f.unextended_sp(), + "res: " INTPTR_FORMAT " initial_sp: " INTPTR_FORMAT " last_sp: " INTPTR_FORMAT " unextended_sp: " INTPTR_FORMAT " expression_stack_size: %d", + p2i(res), p2i(f.addr_at(frame::interpreter_frame_initial_sp_offset)), f.at(frame::interpreter_frame_last_sp_offset), p2i(f.unextended_sp()), expression_stack_sz); + return res; +} + +inline intptr_t* ContinuationHelper::InterpretedFrame::frame_bottom(const frame& f) { // exclusive; this will not be copied with the frame + return (intptr_t*)f.at(frame::interpreter_frame_locals_offset) + 1; // exclusive, so we add 1 word +} + +inline intptr_t* ContinuationHelper::InterpretedFrame::frame_top(const frame& f, int callee_argsize, bool callee_interpreted) { + return f.unextended_sp() + (callee_interpreted ? callee_argsize : 0); +} + +#endif // CPU_AARCH64_CONTINUATIONHELPER_AARCH64_INLINE_HPP diff --git a/src/hotspot/cpu/aarch64/frame_aarch64.cpp b/src/hotspot/cpu/aarch64/frame_aarch64.cpp index 113e4b3251a..0790b7783a8 100644 --- a/src/hotspot/cpu/aarch64/frame_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/frame_aarch64.cpp @@ -56,6 +56,9 @@ void RegisterMap::check_location_valid() { // Profiling/safepoint support bool frame::safe_for_sender(JavaThread *thread) { + if (is_heap_frame()) { + return true; + } address sp = (address)_sp; address fp = (address)_fp; address unextended_sp = (address)_unextended_sp; @@ -156,6 +159,12 @@ bool frame::safe_for_sender(JavaThread *thread) { sender_pc = pauth_strip_verifiable((address) *(sender_sp-1), (address)saved_fp); } + if (Continuation::is_return_barrier_entry(sender_pc)) { + // If our sender_pc is the return barrier, then our "real" sender is the continuation entry + frame s = Continuation::continuation_bottom_sender(thread, *this, sender_sp); + sender_sp = s.sp(); + sender_pc = s.pc(); + } // If the potential sender is the interpreter then we can do some more checking if (Interpreter::contains(sender_pc)) { @@ -271,6 +280,7 @@ void frame::patch_pc(Thread* thread, address pc) { address signing_sp = (((address*) sp())[-2]); address signed_pc = pauth_sign_return_address(pc, (address)signing_sp); address pc_old = pauth_strip_verifiable(*pc_addr, (address)signing_sp); + if (TracePcPatching) { tty->print("patch_pc at address " INTPTR_FORMAT " [" INTPTR_FORMAT " -> " INTPTR_FORMAT "]", p2i(pc_addr), p2i(pc_old), p2i(pc)); @@ -280,30 +290,24 @@ void frame::patch_pc(Thread* thread, address pc) { tty->print_cr(""); } + assert(!Continuation::is_return_barrier_entry(pc_old), "return barrier"); + // Either the return address is the original one or we are going to // patch in the same address that's already there. - assert(_pc == pc_old || pc == pc_old, "must be"); + assert(_pc == pc_old || pc == pc_old || pc_old == 0, ""); + DEBUG_ONLY(address old_pc = _pc;) *pc_addr = signed_pc; + _pc = pc; // must be set before call to get_deopt_original_pc address original_pc = CompiledMethod::get_deopt_original_pc(this); if (original_pc != NULL) { - assert(original_pc == _pc, "expected original PC to be stored before patching"); + assert(original_pc == old_pc, "expected original PC to be stored before patching"); _deopt_state = is_deoptimized; - // leave _pc as is + _pc = original_pc; } else { _deopt_state = not_deoptimized; - _pc = pc; } } -bool frame::is_interpreted_frame() const { - return Interpreter::contains(pc()); -} - -int frame::frame_size(RegisterMap* map) const { - frame sender = this->sender(map); - return sender.sp() - sp(); -} - intptr_t* frame::entry_frame_argument_at(int offset) const { // convert offset to index to deal with tsi int index = (Interpreter::expr_offset_in_bytes(offset)/wordSize); @@ -330,7 +334,7 @@ BasicObjectLock* frame::interpreter_frame_monitor_begin() const { } BasicObjectLock* frame::interpreter_frame_monitor_end() const { - BasicObjectLock* result = (BasicObjectLock*) *addr_at(interpreter_frame_monitor_block_top_offset); + BasicObjectLock* result = (BasicObjectLock*) at(interpreter_frame_monitor_block_top_offset); // make sure the pointer points inside the frame assert(sp() <= (intptr_t*) result, "monitor end should be above the stack pointer"); assert((intptr_t*) result < fp(), "monitor end should be strictly below the frame pointer"); @@ -401,6 +405,7 @@ void frame::verify_deopt_original_pc(CompiledMethod* nm, intptr_t* unextended_sp //------------------------------------------------------------------------------ // frame::adjust_unextended_sp +#ifdef ASSERT void frame::adjust_unextended_sp() { // On aarch64, sites calling method handle intrinsics and lambda forms are treated // as any other call site. Therefore, no special action is needed when we are @@ -412,32 +417,12 @@ void frame::adjust_unextended_sp() { // If the sender PC is a deoptimization point, get the original PC. if (sender_cm->is_deopt_entry(_pc) || sender_cm->is_deopt_mh_entry(_pc)) { - DEBUG_ONLY(verify_deopt_original_pc(sender_cm, _unextended_sp)); + verify_deopt_original_pc(sender_cm, _unextended_sp); } } } } - -//------------------------------------------------------------------------------ -// frame::update_map_with_saved_link -void frame::update_map_with_saved_link(RegisterMap* map, intptr_t** link_addr) { - // The interpreter and compiler(s) always save fp in a known - // location on entry. We must record where that location is - // so that if fp was live on callout from c2 we can find - // the saved copy no matter what it called. - - // Since the interpreter always saves fp if we record where it is then - // we don't have to always save fp on entry and exit to c2 compiled - // code, on entry will be enough. - map->set_location(rfp->as_VMReg(), (address) link_addr); - // this is weird "H" ought to be at a higher address however the - // oopMaps seems to have the "H" regs at the same address and the - // vanilla register. - // XXXX make this go away - if (true) { - map->set_location(rfp->as_VMReg()->next(), (address) link_addr); - } -} +#endif //------------------------------------------------------------------------------ @@ -449,6 +434,7 @@ frame frame::sender_for_interpreter_frame(RegisterMap* map) const { // This is the sp before any possible extension (adapter/locals). intptr_t* unextended_sp = interpreter_frame_sender_sp(); + intptr_t* sender_fp = link(); #if COMPILER2_OR_JVMCI if (map->update_map()) { @@ -459,94 +445,15 @@ frame frame::sender_for_interpreter_frame(RegisterMap* map) const { // For ROP protection, Interpreter will have signed the sender_pc, but there is no requirement to authenticate it here. address sender_pc = pauth_strip_verifiable(sender_pc_maybe_signed(), (address)link()); - return frame(sender_sp, unextended_sp, link(), sender_pc); -} - -//------------------------------------------------------------------------------ -// frame::sender_for_compiled_frame -frame frame::sender_for_compiled_frame(RegisterMap* map) const { - // When the sp of a compiled frame is correct, we can get the correct sender sp - // by unextended sp + frame size. - // For the following two scenarios, the sp of a compiled frame is correct: - // a) This compiled frame is built from the anchor. - // b) This compiled frame is built from a callee frame, and the callee frame can - // calculate its sp correctly. - // - // For b), if the callee frame is a native code frame (such as leaf call), the sp of - // the compiled frame cannot be calculated correctly. There is currently no suitable - // solution to solve this problem perfectly. But when PreserveFramePointer is enabled, - // we can get the correct sender sp by fp + 2 (that is sender_sp()). - - assert(_cb->frame_size() >= 0, "must have non-zero frame size"); - intptr_t* l_sender_sp = (!PreserveFramePointer || _sp_is_trusted) ? unextended_sp() + _cb->frame_size() - : sender_sp(); - intptr_t* unextended_sp = l_sender_sp; - - // the return_address is always the word on the stack - - // For ROP protection, C1/C2 will have signed the sender_pc, but there is no requirement to authenticate it here. - address sender_pc = pauth_strip_verifiable((address) *(l_sender_sp-1), (address) *(l_sender_sp-2)); - - intptr_t** saved_fp_addr = (intptr_t**) (l_sender_sp - frame::sender_sp_offset); - - // assert (sender_sp() == l_sender_sp, "should be"); - // assert (*saved_fp_addr == link(), "should be"); - - if (map->update_map()) { - // Tell GC to use argument oopmaps for some runtime stubs that need it. - // For C1, the runtime stub might not have oop maps, so set this flag - // outside of update_register_map. - map->set_include_argument_oops(_cb->caller_must_gc_arguments(map->thread())); - if (_cb->oop_maps() != NULL) { - OopMapSet::update_register_map(this, map); + if (Continuation::is_return_barrier_entry(sender_pc)) { + if (map->walk_cont()) { // about to walk into an h-stack + return Continuation::top_frame(*this, map); + } else { + return Continuation::continuation_bottom_sender(map->thread(), *this, sender_sp); } - - // Since the prolog does the save and restore of FP there is no - // oopmap for it so we must fill in its location as if there was - // an oopmap entry since if our caller was compiled code there - // could be live jvm state in it. - update_map_with_saved_link(map, saved_fp_addr); } - return frame(l_sender_sp, unextended_sp, *saved_fp_addr, sender_pc); -} - -//------------------------------------------------------------------------------ -// frame::sender_raw -frame frame::sender_raw(RegisterMap* map) const { - // Default is we done have to follow them. The sender_for_xxx will - // update it accordingly - map->set_include_argument_oops(false); - - if (is_entry_frame()) - return sender_for_entry_frame(map); - if (is_interpreted_frame()) - return sender_for_interpreter_frame(map); - assert(_cb == CodeCache::find_blob(pc()),"Must be the same"); - - // This test looks odd: why is it not is_compiled_frame() ? That's - // because stubs also have OOP maps. - if (_cb != NULL) { - return sender_for_compiled_frame(map); - } - - // Must be native-compiled frame, i.e. the marshaling code for native - // methods that exists in the core system. - - // Native code may or may not have signed the return address, we have no way to be sure or what - // signing methods they used. Instead, just ensure the stripped value is used. - - return frame(sender_sp(), link(), sender_pc()); -} - -frame frame::sender(RegisterMap* map) const { - frame result = sender_raw(map); - - if (map->process_frames()) { - StackWatermarkSet::on_iteration(map->thread(), result); - } - - return result; + return frame(sender_sp, unextended_sp, sender_fp, sender_pc); } bool frame::is_interpreted_frame_valid(JavaThread* thread) const { @@ -655,7 +562,6 @@ BasicType frame::interpreter_frame_result(oop* oop_result, jvalue* value_result) return type; } - intptr_t* frame::interpreter_frame_tos_at(jint offset) const { int index = (Interpreter::expr_offset_in_bytes(offset)/wordSize); return &interpreter_frame_tos_address()[index]; @@ -678,6 +584,14 @@ void frame::describe_pd(FrameValues& values, int frame_no) { DESCRIBE_FP_OFFSET(interpreter_frame_bcp); DESCRIBE_FP_OFFSET(interpreter_frame_initial_sp); } + + intptr_t* ret_pc_loc = sp() - return_addr_offset; + address ret_pc = *(address*)ret_pc_loc; + if (Continuation::is_return_barrier_entry(ret_pc)) + values.describe(frame_no, ret_pc_loc, "return address (return barrier)"); + else + values.describe(frame_no, ret_pc_loc, err_msg("return address for #%d", frame_no)); + values.describe(frame_no, sp() - sender_sp_offset, err_msg("saved fp for #%d", frame_no), 0); } #endif @@ -686,19 +600,6 @@ intptr_t *frame::initial_deoptimization_info() { return NULL; } -intptr_t* frame::real_fp() const { - if (_cb != NULL) { - // use the frame size if valid - int size = _cb->frame_size(); - if (size > 0) { - return unextended_sp() + size; - } - } - // else rely on fp() - assert(! is_compiled_frame(), "unknown compiled frame size"); - return fp(); -} - #undef DESCRIBE_FP_OFFSET #define DESCRIBE_FP_OFFSET(name) \ diff --git a/src/hotspot/cpu/aarch64/frame_aarch64.hpp b/src/hotspot/cpu/aarch64/frame_aarch64.hpp index 95f470befa0..89cc045e43d 100644 --- a/src/hotspot/cpu/aarch64/frame_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/frame_aarch64.hpp @@ -99,8 +99,14 @@ entry_frame_call_wrapper_offset = -8, // we don't need a save area - arg_reg_save_area_bytes = 0 + arg_reg_save_area_bytes = 0, + // size, in words, of frame metadata (e.g. pc and link) + metadata_words = sender_sp_offset, + // in bytes + frame_alignment = 16, + // size, in words, of maximum shift in frame position due to alignment + align_wiggle = 1 }; intptr_t ptr_at(int offset) const { @@ -113,7 +119,10 @@ private: // an additional field beyond _sp and _pc: - intptr_t* _fp; // frame pointer + union { + intptr_t* _fp; // frame pointer + int _offset_fp; // relative frame pointer for use in stack-chunk frames + }; // The interpreter and adapters will extend the frame of the caller. // Since oopMaps are based on the sp of the caller before extension // we need to know that value. However in order to compute the address @@ -121,8 +130,12 @@ // uses sp() to mean "raw" sp and unextended_sp() to mean the caller's // original sp we use that convention. - intptr_t* _unextended_sp; - void adjust_unextended_sp(); + union { + intptr_t* _unextended_sp; + int _offset_unextended_sp; // for use in stack-chunk frames + }; + + void adjust_unextended_sp() NOT_DEBUG_RETURN; // true means _sp value is correct and we can use it to get the sender's sp // of the compiled frame, otherwise, _sp value may be invalid and we can use @@ -138,6 +151,8 @@ static void verify_deopt_original_pc( CompiledMethod* nm, intptr_t* unextended_sp); #endif + const ImmutableOopMap* get_oop_map() const; + public: // Constructors @@ -145,13 +160,21 @@ frame(intptr_t* sp, intptr_t* unextended_sp, intptr_t* fp, address pc); + frame(intptr_t* sp, intptr_t* unextended_sp, intptr_t* fp, address pc, CodeBlob* cb); + // used for fast frame construction by continuations + frame(intptr_t* sp, intptr_t* unextended_sp, intptr_t* fp, address pc, CodeBlob* cb, const ImmutableOopMap* oop_map, bool on_heap); + frame(intptr_t* sp, intptr_t* fp); void init(intptr_t* sp, intptr_t* fp, address pc); + void setup(address pc); // accessors for the instance variables // Note: not necessarily the real 'frame pointer' (see real_fp) - intptr_t* fp() const { return _fp; } + intptr_t* fp() const { assert_absolute(); return _fp; } + void set_fp(intptr_t* newfp) { _fp = newfp; } + int offset_fp() const { assert_offset(); return _offset_fp; } + void set_offset_fp(int value) { assert_on_heap(); _offset_fp = value; } inline address* sender_pc_addr() const; inline address sender_pc_maybe_signed() const; @@ -159,8 +182,8 @@ // expression stack tos if we are nested in a java call intptr_t* interpreter_frame_last_sp() const; - // helper to update a map with callee-saved RBP - static void update_map_with_saved_link(RegisterMap* map, intptr_t** link_addr); + template + static void update_map_with_saved_link(RegisterMapT* map, intptr_t** link_addr); // deoptimization support void interpreter_frame_set_last_sp(intptr_t* sp); @@ -168,7 +191,7 @@ static jint interpreter_frame_expression_stack_direction() { return -1; } // returns the sending frame, without applying any barriers - frame sender_raw(RegisterMap* map) const; + inline frame sender_raw(RegisterMap* map) const; void set_sp_is_trusted() { _sp_is_trusted = true; } diff --git a/src/hotspot/cpu/aarch64/frame_aarch64.inline.hpp b/src/hotspot/cpu/aarch64/frame_aarch64.inline.hpp index 20b5b8e8662..1d9e98d2e51 100644 --- a/src/hotspot/cpu/aarch64/frame_aarch64.inline.hpp +++ b/src/hotspot/cpu/aarch64/frame_aarch64.inline.hpp @@ -26,8 +26,12 @@ #ifndef CPU_AARCH64_FRAME_AARCH64_INLINE_HPP #define CPU_AARCH64_FRAME_AARCH64_INLINE_HPP -#include "code/codeCache.hpp" +#include "code/codeBlob.inline.hpp" +#include "code/codeCache.inline.hpp" #include "code/vmreg.inline.hpp" +#include "interpreter/interpreter.hpp" +#include "interpreter/oopMapCache.hpp" +#include "runtime/sharedRuntime.hpp" #include "pauth_aarch64.hpp" // Inline functions for AArch64 frames: @@ -42,6 +46,8 @@ inline frame::frame() { _cb = NULL; _deopt_state = unknown; _sp_is_trusted = false; + _on_heap = false; + DEBUG_ONLY(_frame_index = -1;) } static int spin; @@ -54,16 +60,30 @@ inline void frame::init(intptr_t* sp, intptr_t* fp, address pc) { _unextended_sp = sp; _fp = fp; _pc = pc; + _oop_map = NULL; + _on_heap = false; + DEBUG_ONLY(_frame_index = -1;) + assert(pc != NULL, "no pc?"); _cb = CodeCache::find_blob(pc); + setup(pc); +} + +inline void frame::setup(address pc) { adjust_unextended_sp(); address original_pc = CompiledMethod::get_deopt_original_pc(this); if (original_pc != NULL) { _pc = original_pc; _deopt_state = is_deoptimized; + assert(_cb == NULL || _cb->as_compiled_method()->insts_contains_inclusive(_pc), + "original PC must be in the main code section of the the compiled method (or must be immediately following it)"); } else { - _deopt_state = not_deoptimized; + if (_cb == SharedRuntime::deopt_blob()) { + _deopt_state = is_deoptimized; + } else { + _deopt_state = not_deoptimized; + } } _sp_is_trusted = false; } @@ -72,7 +92,7 @@ inline frame::frame(intptr_t* sp, intptr_t* fp, address pc) { init(sp, fp, pc); } -inline frame::frame(intptr_t* sp, intptr_t* unextended_sp, intptr_t* fp, address pc) { +inline frame::frame(intptr_t* sp, intptr_t* unextended_sp, intptr_t* fp, address pc, CodeBlob* cb) { assert(pauth_ptr_is_raw(pc), "cannot be signed"); intptr_t a = intptr_t(sp); intptr_t b = intptr_t(fp); @@ -81,21 +101,59 @@ inline frame::frame(intptr_t* sp, intptr_t* unextended_sp, intptr_t* fp, address _fp = fp; _pc = pc; assert(pc != NULL, "no pc?"); - _cb = CodeCache::find_blob(pc); - adjust_unextended_sp(); + _cb = cb; + _oop_map = NULL; + assert(_cb != NULL, "pc: " INTPTR_FORMAT, p2i(pc)); + _on_heap = false; + DEBUG_ONLY(_frame_index = -1;) - address original_pc = CompiledMethod::get_deopt_original_pc(this); - if (original_pc != NULL) { - _pc = original_pc; - assert(_cb->as_compiled_method()->insts_contains_inclusive(_pc), - "original PC must be in the main code section of the the compiled method (or must be immediately following it)"); - _deopt_state = is_deoptimized; - } else { - _deopt_state = not_deoptimized; - } - _sp_is_trusted = false; + setup(pc); } +inline frame::frame(intptr_t* sp, intptr_t* unextended_sp, intptr_t* fp, address pc, CodeBlob* cb, const ImmutableOopMap* oop_map, bool on_heap) { + _sp = sp; + _unextended_sp = unextended_sp; + _fp = fp; + _pc = pc; + _cb = cb; + _oop_map = oop_map; + _deopt_state = not_deoptimized; + _sp_is_trusted = false; + _on_heap = on_heap; + DEBUG_ONLY(_frame_index = -1;) + + // In thaw, non-heap frames use this constructor to pass oop_map. I don't know why. + assert(_on_heap || _cb != nullptr, "these frames are always heap frames"); + if (cb != NULL) { + setup(pc); + } +#ifdef ASSERT + // The following assertion has been disabled because it would sometime trap for Continuation.run, + // which is not *in* a continuation and therefore does not clear the _cont_fastpath flag, but this + // is benign even in fast mode (see Freeze::setup_jump) + // We might freeze deoptimized frame in slow mode + // assert(_pc == pc && _deopt_state == not_deoptimized, ""); +#endif +} + +inline frame::frame(intptr_t* sp, intptr_t* unextended_sp, intptr_t* fp, address pc) { + intptr_t a = intptr_t(sp); + intptr_t b = intptr_t(fp); + _sp = sp; + _unextended_sp = unextended_sp; + _fp = fp; + _pc = pc; + _cb = CodeCache::find_blob_fast(pc); + _oop_map = NULL; + assert(_cb != NULL, "pc: " INTPTR_FORMAT " sp: " INTPTR_FORMAT " unextended_sp: " INTPTR_FORMAT " fp: " INTPTR_FORMAT, p2i(pc), p2i(sp), p2i(unextended_sp), p2i(fp)); + _on_heap = false; + DEBUG_ONLY(_frame_index = -1;) + + setup(pc); +} + +inline frame::frame(intptr_t* sp) : frame(sp, sp, *(intptr_t**)(sp - frame::sender_sp_offset), *(address*)(sp - 1)) {} + inline frame::frame(intptr_t* sp, intptr_t* fp) { intptr_t a = intptr_t(sp); intptr_t b = intptr_t(fp); @@ -103,6 +161,8 @@ inline frame::frame(intptr_t* sp, intptr_t* fp) { _unextended_sp = sp; _fp = fp; _pc = (address)(sp[-1]); + _on_heap = false; + DEBUG_ONLY(_frame_index = -1;) // Here's a sticky one. This constructor can be called via AsyncGetCallTrace // when last_Java_sp is non-null but the pc fetched is junk. If we are truly @@ -155,7 +215,41 @@ inline intptr_t* frame::link_or_null() const { return os::is_readable_pointer(ptr) ? *ptr : NULL; } -inline intptr_t* frame::unextended_sp() const { return _unextended_sp; } +inline intptr_t* frame::unextended_sp() const { assert_absolute(); return _unextended_sp; } +inline void frame::set_unextended_sp(intptr_t* value) { _unextended_sp = value; } +inline int frame::offset_unextended_sp() const { assert_offset(); return _offset_unextended_sp; } +inline void frame::set_offset_unextended_sp(int value) { assert_on_heap(); _offset_unextended_sp = value; } + +inline intptr_t* frame::real_fp() const { + if (_cb != NULL) { + // use the frame size if valid + int size = _cb->frame_size(); + if (size > 0) { + return unextended_sp() + size; + } + } + // else rely on fp() + assert(! is_compiled_frame(), "unknown compiled frame size"); + return fp(); +} + +inline int frame::frame_size() const { + return is_interpreted_frame() + ? sender_sp() - sp() + : cb()->frame_size(); +} + +inline int frame::compiled_frame_stack_argsize() const { + assert(cb()->is_compiled(), ""); + return (cb()->as_compiled_method()->method()->num_stack_arg_slots() * VMRegImpl::stack_slot_size) >> LogBytesPerWord; +} + +inline void frame::interpreted_frame_oop_map(InterpreterOopMap* mask) const { + assert(mask != NULL, ""); + Method* m = interpreter_frame_method(); + int bci = interpreter_frame_bci(); + m->mask_for(bci, mask); // OopMapCache::compute_one_oop_map(m, bci, mask); +} // Return address: @@ -170,7 +264,7 @@ inline intptr_t** frame::interpreter_frame_locals_addr() const { } inline intptr_t* frame::interpreter_frame_last_sp() const { - return *(intptr_t**)addr_at(interpreter_frame_last_sp_offset); + return (intptr_t*)at(interpreter_frame_last_sp_offset); } inline intptr_t* frame::interpreter_frame_bcp_addr() const { @@ -244,21 +338,147 @@ inline JavaCallWrapper** frame::entry_frame_call_wrapper_addr() const { inline oop frame::saved_oop_result(RegisterMap* map) const { PRAGMA_DIAG_PUSH PRAGMA_NONNULL_IGNORED - oop* result_adr = (oop *)map->location(r0->as_VMReg()); + oop* result_adr = (oop *)map->location(r0->as_VMReg(), sp()); PRAGMA_DIAG_POP guarantee(result_adr != NULL, "bad register save location"); - - return (*result_adr); + return *result_adr; } inline void frame::set_saved_oop_result(RegisterMap* map, oop obj) { PRAGMA_DIAG_PUSH PRAGMA_NONNULL_IGNORED - oop* result_adr = (oop *)map->location(r0->as_VMReg()); + oop* result_adr = (oop *)map->location(r0->as_VMReg(), sp()); PRAGMA_DIAG_POP guarantee(result_adr != NULL, "bad register save location"); *result_adr = obj; } +inline bool frame::is_interpreted_frame() const { + return Interpreter::contains(pc()); +} + +inline int frame::sender_sp_ret_address_offset() { + return frame::sender_sp_offset - frame::return_addr_offset; +} + +inline const ImmutableOopMap* frame::get_oop_map() const { + if (_cb == NULL) return NULL; + if (_cb->oop_maps() != NULL) { + NativePostCallNop* nop = nativePostCallNop_at(_pc); + if (nop != NULL && nop->displacement() != 0) { + int slot = ((nop->displacement() >> 24) & 0xff); + return _cb->oop_map_for_slot(slot, _pc); + } + const ImmutableOopMap* oop_map = OopMapSet::find_map(this); + return oop_map; + } + return NULL; +} + +//------------------------------------------------------------------------------ +// frame::sender +inline frame frame::sender(RegisterMap* map) const { + frame result = sender_raw(map); + + if (map->process_frames() && !map->in_cont()) { + StackWatermarkSet::on_iteration(map->thread(), result); + } + + return result; +} + +inline frame frame::sender_raw(RegisterMap* map) const { + // Default is we done have to follow them. The sender_for_xxx will + // update it accordingly + map->set_include_argument_oops(false); + + if (map->in_cont()) { // already in an h-stack + return map->stack_chunk()->sender(*this, map); + } + + if (is_entry_frame()) return sender_for_entry_frame(map); + if (is_optimized_entry_frame()) return sender_for_optimized_entry_frame(map); + if (is_interpreted_frame()) return sender_for_interpreter_frame(map); + + assert(_cb == CodeCache::find_blob(pc()), "Must be the same"); + if (_cb != NULL) return sender_for_compiled_frame(map); + + // Must be native-compiled frame, i.e. the marshaling code for native + // methods that exists in the core system. + + // Native code may or may not have signed the return address, we have no way to be sure or what + // signing methods they used. Instead, just ensure the stripped value is used. + + return frame(sender_sp(), link(), sender_pc()); +} + +inline frame frame::sender_for_compiled_frame(RegisterMap* map) const { + // we cannot rely upon the last fp having been saved to the thread + // in C2 code but it will have been pushed onto the stack. so we + // have to find it relative to the unextended sp + + assert(_cb->frame_size() >= 0, "must have non-zero frame size"); + intptr_t* l_sender_sp = (!PreserveFramePointer || _sp_is_trusted) ? unextended_sp() + _cb->frame_size() + : sender_sp(); + assert(!_sp_is_trusted || l_sender_sp == real_fp(), ""); + + // the return_address is always the word on the stack + // For ROP protection, C1/C2 will have signed the sender_pc, but there is no requirement to authenticate it here. + address sender_pc = pauth_strip_verifiable((address) *(l_sender_sp-1), (address) *(l_sender_sp-2)); + + intptr_t** saved_fp_addr = (intptr_t**) (l_sender_sp - frame::sender_sp_offset); + + if (map->update_map()) { + // Tell GC to use argument oopmaps for some runtime stubs that need it. + // For C1, the runtime stub might not have oop maps, so set this flag + // outside of update_register_map. + if (!_cb->is_compiled()) { // compiled frames do not use callee-saved registers + map->set_include_argument_oops(_cb->caller_must_gc_arguments(map->thread())); + if (oop_map() != NULL) { + _oop_map->update_register_map(this, map); + } + } else { + assert(!_cb->caller_must_gc_arguments(map->thread()), ""); + assert(!map->include_argument_oops(), ""); + assert(oop_map() == NULL || !oop_map()->has_any(OopMapValue::callee_saved_value), "callee-saved value in compiled frame"); + } + + // Since the prolog does the save and restore of EBP there is no oopmap + // for it so we must fill in its location as if there was an oopmap entry + // since if our caller was compiled code there could be live jvm state in it. + update_map_with_saved_link(map, saved_fp_addr); + } + + if (Continuation::is_return_barrier_entry(sender_pc)) { + if (map->walk_cont()) { // about to walk into an h-stack + return Continuation::top_frame(*this, map); + } else { + return Continuation::continuation_bottom_sender(map->thread(), *this, l_sender_sp); + } + } + + intptr_t* unextended_sp = l_sender_sp; + return frame(l_sender_sp, unextended_sp, *saved_fp_addr, sender_pc); +} + +template +void frame::update_map_with_saved_link(RegisterMapT* map, intptr_t** link_addr) { + // The interpreter and compiler(s) always save EBP/RBP in a known + // location on entry. We must record where that location is + // so this if EBP/RBP was live on callout from c2 we can find + // the saved copy no matter what it called. + + // Since the interpreter always saves EBP/RBP if we record where it is then + // we don't have to always save EBP/RBP on entry and exit to c2 compiled + // code, on entry will be enough. + map->set_location(rfp->as_VMReg(), (address) link_addr); + // this is weird "H" ought to be at a higher address however the + // oopMaps seems to have the "H" regs at the same address and the + // vanilla register. + // XXXX make this go away + if (true) { + map->set_location(rfp->as_VMReg()->next(), (address) link_addr); + } +} #endif // CPU_AARCH64_FRAME_AARCH64_INLINE_HPP diff --git a/src/hotspot/cpu/aarch64/gc/g1/g1BarrierSetAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/gc/g1/g1BarrierSetAssembler_aarch64.cpp index 01aff54c96d..de30c7c40ca 100644 --- a/src/hotspot/cpu/aarch64/gc/g1/g1BarrierSetAssembler_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/gc/g1/g1BarrierSetAssembler_aarch64.cpp @@ -154,12 +154,8 @@ void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm, __ b(done); __ bind(runtime); - // save the live input values - RegSet saved = RegSet::of(pre_val); - if (tosca_live) saved += RegSet::of(r0); - if (obj != noreg) saved += RegSet::of(obj); - __ push(saved, sp); + __ push_call_clobbered_registers(); // Calling the runtime using the regular call_VM_leaf mechanism generates // code (generated by InterpreterMacroAssember::call_VM_leaf_base) @@ -180,7 +176,7 @@ void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm, __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), pre_val, thread); } - __ pop(saved, sp); + __ pop_call_clobbered_registers(); __ bind(done); diff --git a/src/hotspot/cpu/aarch64/gc/shared/barrierSetAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/gc/shared/barrierSetAssembler_aarch64.cpp index a4a2b142039..386c5879f89 100644 --- a/src/hotspot/cpu/aarch64/gc/shared/barrierSetAssembler_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/gc/shared/barrierSetAssembler_aarch64.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -232,6 +232,20 @@ void BarrierSetAssembler::incr_allocated_bytes(MacroAssembler* masm, __ str(t1, Address(rthread, in_bytes(JavaThread::allocated_bytes_offset()))); } +static volatile uint32_t _patching_epoch = 0; + +address BarrierSetAssembler::patching_epoch_addr() { + return (address)&_patching_epoch; +} + +void BarrierSetAssembler::increment_patching_epoch() { + Atomic::inc(&_patching_epoch); +} + +void BarrierSetAssembler::clear_patching_epoch() { + _patching_epoch = 0; +} + void BarrierSetAssembler::nmethod_entry_barrier(MacroAssembler* masm) { BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod(); @@ -239,27 +253,51 @@ void BarrierSetAssembler::nmethod_entry_barrier(MacroAssembler* masm) { return; } - Label skip, guard; - Address thread_disarmed_addr(rthread, in_bytes(bs_nm->thread_disarmed_offset())); + Label skip_barrier, guard; __ ldrw(rscratch1, guard); - // Subsequent loads of oops must occur after load of guard value. - // BarrierSetNMethod::disarm sets guard with release semantics. - __ membar(__ LoadLoad); - __ ldrw(rscratch2, thread_disarmed_addr); - __ cmpw(rscratch1, rscratch2); - __ br(Assembler::EQ, skip); + if (nmethod_code_patching()) { + // If we patch code we need both a code patching and a loadload + // fence. It's not super cheap, so we use a global epoch mechanism + // to hide them in a slow path. + // The high level idea of the global epoch mechanism is to detect + // when any thread has performed the required fencing, after the + // last nmethod was disarmed. This implies that the required + // fencing has been performed for all preceding nmethod disarms + // as well. Therefore, we do not need any further fencing. + __ lea(rscratch2, ExternalAddress((address)&_patching_epoch)); + // Embed an artificial data dependency to order the guard load + // before the epoch load. + __ orr(rscratch2, rscratch2, rscratch1, Assembler::LSR, 32); + // Read the global epoch value. + __ ldrw(rscratch2, rscratch2); + // Combine the guard value (low order) with the epoch value (high order). + __ orr(rscratch1, rscratch1, rscratch2, Assembler::LSL, 32); + // Compare the global values with the thread-local values. + Address thread_disarmed_and_epoch_addr(rthread, in_bytes(bs_nm->thread_disarmed_offset())); + __ ldr(rscratch2, thread_disarmed_and_epoch_addr); + __ cmp(rscratch1, rscratch2); + __ br(Assembler::EQ, skip_barrier); + } else { + // Subsequent loads of oops must occur after load of guard value. + // BarrierSetNMethod::disarm sets guard with release semantics. + __ membar(__ LoadLoad); + Address thread_disarmed_addr(rthread, in_bytes(bs_nm->thread_disarmed_offset())); + __ ldrw(rscratch2, thread_disarmed_addr); + __ cmpw(rscratch1, rscratch2); + __ br(Assembler::EQ, skip_barrier); + } __ movptr(rscratch1, (uintptr_t) StubRoutines::aarch64::method_entry_barrier()); __ blr(rscratch1); - __ b(skip); + __ b(skip_barrier); __ bind(guard); __ emit_int32(0); // nmethod guard value. Skipped over in common case. - __ bind(skip); + __ bind(skip_barrier); } void BarrierSetAssembler::c2i_entry_barrier(MacroAssembler* masm) { diff --git a/src/hotspot/cpu/aarch64/gc/shared/barrierSetAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/gc/shared/barrierSetAssembler_aarch64.hpp index 935f8a91564..5e7a5b97ef9 100644 --- a/src/hotspot/cpu/aarch64/gc/shared/barrierSetAssembler_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/gc/shared/barrierSetAssembler_aarch64.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -68,9 +68,14 @@ public: ); virtual void barrier_stubs_init() {} + virtual bool nmethod_code_patching() { return true; } + virtual void nmethod_entry_barrier(MacroAssembler* masm); virtual void c2i_entry_barrier(MacroAssembler* masm); + static address patching_epoch_addr(); + static void clear_patching_epoch(); + static void increment_patching_epoch(); }; #endif // CPU_AARCH64_GC_SHARED_BARRIERSETASSEMBLER_AARCH64_HPP diff --git a/src/hotspot/cpu/aarch64/gc/shared/barrierSetNMethod_aarch64.cpp b/src/hotspot/cpu/aarch64/gc/shared/barrierSetNMethod_aarch64.cpp index 4ce9869a157..e9bc3d85df6 100644 --- a/src/hotspot/cpu/aarch64/gc/shared/barrierSetNMethod_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/gc/shared/barrierSetNMethod_aarch64.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -25,9 +25,12 @@ #include "precompiled.hpp" #include "code/codeCache.hpp" #include "code/nativeInst.hpp" +#include "gc/shared/barrierSet.hpp" +#include "gc/shared/barrierSetAssembler.hpp" #include "gc/shared/barrierSetNMethod.hpp" #include "logging/log.hpp" #include "memory/resourceArea.hpp" +#include "runtime/frame.inline.hpp" #include "runtime/sharedRuntime.hpp" #include "runtime/registerMap.hpp" #include "runtime/thread.hpp" @@ -37,8 +40,17 @@ class NativeNMethodBarrier: public NativeInstruction { address instruction_address() const { return addr_at(0); } + int guard_offset() { + BarrierSetAssembler* bs_asm = BarrierSet::barrier_set()->barrier_set_assembler(); + if (bs_asm->nmethod_code_patching()) { + return 4 * 15; + } else { + return 4 * 10; + } + } + int *guard_addr() { - return reinterpret_cast(instruction_address() + 10 * 4); + return reinterpret_cast(instruction_address() + guard_offset()); } public: @@ -60,31 +72,14 @@ struct CheckInsn { const char *name; }; -static const struct CheckInsn barrierInsn[] = { - { 0xff000000, 0x18000000, "ldr (literal)" }, - { 0xfffff0ff, 0xd50330bf, "dmb" }, - { 0xffc00000, 0xb9400000, "ldr"}, - { 0x7f20001f, 0x6b00001f, "cmp"}, - { 0xff00001f, 0x54000000, "b.eq"}, - { 0xff800000, 0xd2800000, "mov"}, - { 0xff800000, 0xf2800000, "movk"}, - { 0xff800000, 0xf2800000, "movk"}, - { 0xfffffc1f, 0xd63f0000, "blr"}, - { 0xfc000000, 0x14000000, "b"} -}; - -// The encodings must match the instructions emitted by -// BarrierSetAssembler::nmethod_entry_barrier. The matching ignores the specific -// register numbers and immediate values in the encoding. +// The first instruction of the nmethod entry barrier is an ldr (literal) +// instruction. Verify that it's really there, so the offsets are not skewed. void NativeNMethodBarrier::verify() const { - intptr_t addr = (intptr_t) instruction_address(); - for(unsigned int i = 0; i < sizeof(barrierInsn)/sizeof(struct CheckInsn); i++ ) { - uint32_t inst = *((uint32_t*) addr); - if ((inst & barrierInsn[i].mask) != barrierInsn[i].bits) { - tty->print_cr("Addr: " INTPTR_FORMAT " Code: 0x%x", addr, inst); - fatal("not an %s instruction.", barrierInsn[i].name); - } - addr +=4; + uint32_t* addr = (uint32_t*) instruction_address(); + uint32_t inst = *addr; + if ((inst & 0xff000000) != 0x18000000) { + tty->print_cr("Addr: " INTPTR_FORMAT " Code: 0x%x", (intptr_t)addr, inst); + fatal("not an ldr (literal) instruction."); } } @@ -132,10 +127,17 @@ void BarrierSetNMethod::deoptimize(nmethod* nm, address* return_address_ptr) { // not find the expected native instruction at this offset, which needs updating. // Note that this offset is invariant of PreserveFramePointer. -static const int entry_barrier_offset = -4 * 11; +static int entry_barrier_offset() { + BarrierSetAssembler* bs_asm = BarrierSet::barrier_set()->barrier_set_assembler(); + if (bs_asm->nmethod_code_patching()) { + return -4 * 16; + } else { + return -4 * 11; + } +} static NativeNMethodBarrier* native_nmethod_barrier(nmethod* nm) { - address barrier_address = nm->code_begin() + nm->frame_complete_offset() + entry_barrier_offset; + address barrier_address = nm->code_begin() + nm->frame_complete_offset() + entry_barrier_offset(); NativeNMethodBarrier* barrier = reinterpret_cast(barrier_address); debug_only(barrier->verify()); return barrier; @@ -146,13 +148,41 @@ void BarrierSetNMethod::disarm(nmethod* nm) { return; } + // The patching epoch is incremented before the nmethod is disarmed. Disarming + // is performed with a release store. In the nmethod entry barrier, the values + // are read in the opposite order, such that the load of the nmethod guard + // acquires the patching epoch. This way, the guard is guaranteed to block + // entries to the nmethod, until it has safely published the requirement for + // further fencing by mutators, before they are allowed to enter. + BarrierSetAssembler* bs_asm = BarrierSet::barrier_set()->barrier_set_assembler(); + bs_asm->increment_patching_epoch(); + // Disarms the nmethod guard emitted by BarrierSetAssembler::nmethod_entry_barrier. // Symmetric "LDR; DMB ISHLD" is in the nmethod barrier. NativeNMethodBarrier* barrier = native_nmethod_barrier(nm); - barrier->set_value(disarmed_value()); } +void BarrierSetNMethod::arm(nmethod* nm, int arm_value) { + if (!supports_entry_barrier(nm)) { + return; + } + + if (arm_value == disarmed_value()) { + // The patching epoch is incremented before the nmethod is disarmed. Disarming + // is performed with a release store. In the nmethod entry barrier, the values + // are read in the opposite order, such that the load of the nmethod guard + // acquires the patching epoch. This way, the guard is guaranteed to block + // entries to the nmethod, until it has safely published the requirement for + // further fencing by mutators, before they are allowed to enter. + BarrierSetAssembler* bs_asm = BarrierSet::barrier_set()->barrier_set_assembler(); + bs_asm->increment_patching_epoch(); + } + + NativeNMethodBarrier* barrier = native_nmethod_barrier(nm); + barrier->set_value(arm_value); +} + bool BarrierSetNMethod::is_armed(nmethod* nm) { if (!supports_entry_barrier(nm)) { return false; diff --git a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.hpp index 1a20417f080..de3cdf2b236 100644 --- a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.hpp @@ -62,6 +62,8 @@ public: void iu_barrier(MacroAssembler* masm, Register dst, Register tmp); + virtual bool nmethod_code_patching() { return false; } + #ifdef COMPILER1 void gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub); void gen_load_reference_barrier_stub(LIR_Assembler* ce, ShenandoahLoadReferenceBarrierStub* stub); diff --git a/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.hpp index f3d29b44a15..1b9a5cc71c1 100644 --- a/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -76,6 +76,8 @@ public: Register tmp, Label& slowpath); + virtual bool nmethod_code_patching() { return false; } + #ifdef COMPILER1 void generate_c1_load_barrier_test(LIR_Assembler* ce, LIR_Opr ref) const; diff --git a/src/hotspot/cpu/aarch64/interp_masm_aarch64.cpp b/src/hotspot/cpu/aarch64/interp_masm_aarch64.cpp index f1e5758709f..40d427d8176 100644 --- a/src/hotspot/cpu/aarch64/interp_masm_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/interp_masm_aarch64.cpp @@ -606,6 +606,7 @@ void InterpreterMacroAssembler::remove_activation( bind(unlock); unlock_object(c_rarg1); + dec_held_monitor_count(rthread); pop(state); // Check that for block-structured locking (i.e., that all locked @@ -648,6 +649,7 @@ void InterpreterMacroAssembler::remove_activation( push(state); unlock_object(c_rarg1); + dec_held_monitor_count(rthread); pop(state); if (install_monitor_exception) { diff --git a/src/hotspot/cpu/aarch64/javaFrameAnchor_aarch64.hpp b/src/hotspot/cpu/aarch64/javaFrameAnchor_aarch64.hpp index a11bf0dca81..32e8f723609 100644 --- a/src/hotspot/cpu/aarch64/javaFrameAnchor_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/javaFrameAnchor_aarch64.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2002, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2002, 2022, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2014, Red Hat Inc. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * @@ -82,4 +82,6 @@ public: intptr_t* last_Java_fp(void) { return _last_Java_fp; } + void set_last_Java_fp(intptr_t* fp) { _last_Java_fp = fp; } + #endif // CPU_AARCH64_JAVAFRAMEANCHOR_AARCH64_HPP diff --git a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp index 7e8cbd95a6f..cf4cddc0ac3 100644 --- a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp @@ -46,6 +46,7 @@ #include "oops/accessDecorators.hpp" #include "oops/compressedOops.inline.hpp" #include "oops/klass.inline.hpp" +#include "runtime/continuation.hpp" #include "runtime/icache.hpp" #include "runtime/interfaceSupport.inline.hpp" #include "runtime/jniHandles.inline.hpp" @@ -313,6 +314,41 @@ void MacroAssembler::safepoint_poll(Label& slow_path, bool at_return, bool acqui } } +void MacroAssembler::push_cont_fastpath(Register java_thread) { + if (!Continuations::enabled()) return; + Label done; + ldr(rscratch1, Address(java_thread, JavaThread::cont_fastpath_offset())); + cmp(sp, rscratch1); + br(Assembler::LS, done); + mov(rscratch1, sp); // we can't use sp as the source in str + str(rscratch1, Address(java_thread, JavaThread::cont_fastpath_offset())); + bind(done); +} + +void MacroAssembler::pop_cont_fastpath(Register java_thread) { + if (!Continuations::enabled()) return; + Label done; + ldr(rscratch1, Address(java_thread, JavaThread::cont_fastpath_offset())); + cmp(sp, rscratch1); + br(Assembler::LO, done); + str(zr, Address(java_thread, JavaThread::cont_fastpath_offset())); + bind(done); +} + +void MacroAssembler::inc_held_monitor_count(Register java_thread) { + if (!Continuations::enabled()) return; + incrementw(Address(java_thread, JavaThread::held_monitor_count_offset())); +} + +void MacroAssembler::dec_held_monitor_count(Register java_thread) { + if (!Continuations::enabled()) return; + decrementw(Address(java_thread, JavaThread::held_monitor_count_offset())); +} + +void MacroAssembler::reset_held_monitor_count(Register java_thread) { + strw(zr, Address(java_thread, JavaThread::held_monitor_count_offset())); +} + void MacroAssembler::reset_last_Java_frame(bool clear_fp) { // we must set sp to zero to clear frame str(zr, Address(rthread, JavaThread::last_Java_sp_offset())); @@ -566,9 +602,7 @@ void MacroAssembler::call_VM_helper(Register oop_result, address entry_point, in // Maybe emit a call via a trampoline. If the code cache is small // trampolines won't be emitted. - -address MacroAssembler::trampoline_call(Address entry, CodeBuffer* cbuf) { - assert(JavaThread::current()->is_Compiler_thread(), "just checking"); +address MacroAssembler::trampoline_call1(Address entry, CodeBuffer* cbuf, bool check_emit_size) { assert(entry.rspec().type() == relocInfo::runtime_call_type || entry.rspec().type() == relocInfo::opt_virtual_call_type || entry.rspec().type() == relocInfo::static_call_type @@ -578,12 +612,14 @@ address MacroAssembler::trampoline_call(Address entry, CodeBuffer* cbuf) { if (far_branches()) { bool in_scratch_emit_size = false; #ifdef COMPILER2 - // We don't want to emit a trampoline if C2 is generating dummy - // code during its branch shortening phase. - CompileTask* task = ciEnv::current()->task(); - in_scratch_emit_size = - (task != NULL && is_c2_compile(task->comp_level()) && - Compile::current()->output()->in_scratch_emit_size()); + if (check_emit_size) { + // We don't want to emit a trampoline if C2 is generating dummy + // code during its branch shortening phase. + CompileTask* task = ciEnv::current()->task(); + in_scratch_emit_size = + (task != NULL && is_c2_compile(task->comp_level()) && + Compile::current()->output()->in_scratch_emit_size()); + } #endif if (!in_scratch_emit_size) { address stub = emit_trampoline_stub(offset(), entry.target()); @@ -790,6 +826,14 @@ void MacroAssembler::align(int modulus) { while (offset() % modulus != 0) nop(); } +void MacroAssembler::post_call_nop() { + if (!Continuations::enabled()) { + return; + } + relocate(post_call_nop_Relocation::spec()); + nop(); +} + // these are no-ops overridden by InterpreterMacroAssembler void MacroAssembler::check_and_handle_earlyret(Register java_thread) { } @@ -2196,6 +2240,15 @@ void MacroAssembler::unimplemented(const char* what) { stop(buf); } +void MacroAssembler::_assert_asm(Assembler::Condition cc, const char* msg) { +#ifdef ASSERT + Label OK; + br(cc, OK); + stop(msg); + bind(OK); +#endif +} + // If a constant does not fit in an immediate field, generate some // number of MOV instructions and then perform the operation. void MacroAssembler::wrap_add_sub_imm_insn(Register Rd, Register Rn, unsigned imm, @@ -4148,7 +4201,8 @@ void MacroAssembler::movoop(Register dst, jobject obj, bool immediate) { // nmethod entry barrier necessitate using the constant pool. They have to be // ordered with respected to oop accesses. // Using immediate literals would necessitate ISBs. - if (BarrierSet::barrier_set()->barrier_set_nmethod() != NULL || !immediate) { + BarrierSet* bs = BarrierSet::barrier_set(); + if ((bs->barrier_set_nmethod() != NULL && bs->barrier_set_assembler()->nmethod_code_patching()) || !immediate) { address dummy = address(uintptr_t(pc()) & -wordSize); // A nearby aligned address ldr_constant(dst, Address(dummy, rspec)); } else diff --git a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp index b33484111cf..998c438a69c 100644 --- a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp @@ -687,6 +687,9 @@ public: // Alignment void align(int modulus); + // nop + void post_call_nop(); + // Stack frame creation/removal void enter(bool strip_ret_addr = false); void leave(); @@ -874,6 +877,12 @@ public: void pop_CPU_state(bool restore_vectors = false, bool use_sve = false, int sve_vector_size_in_bytes = 0, int total_predicate_in_bytes = 0); + void push_cont_fastpath(Register java_thread); + void pop_cont_fastpath(Register java_thread); + void inc_held_monitor_count(Register java_thread); + void dec_held_monitor_count(Register java_thread); + void reset_held_monitor_count(Register java_thread); + // Round up to a power of two void round_to(Register reg, int modulus); @@ -1004,6 +1013,10 @@ public: void should_not_reach_here() { stop("should not reach here"); } + void _assert_asm(Condition cc, const char* msg); +#define assert_asm0(cc, msg) _assert_asm(cc, FILE_AND_LINE ": " msg) +#define assert_asm(masm, command, cc, msg) DEBUG_ONLY(command; (masm)->_assert_asm(cc, FILE_AND_LINE ": " #command " " #cc ": " msg)) + // Stack overflow checking void bang_stack_with_offset(int offset) { // stack grows down, caller passes positive offset @@ -1084,7 +1097,8 @@ private: public: // Calls - address trampoline_call(Address entry, CodeBuffer* cbuf = NULL); + address trampoline_call(Address entry, CodeBuffer* cbuf = NULL) { return trampoline_call1(entry, cbuf, true); } + address trampoline_call1(Address entry, CodeBuffer* cbuf, bool check_emit_size = true); static bool far_branches() { return ReservedCodeCacheSize > branch_range; diff --git a/src/hotspot/cpu/aarch64/nativeInst_aarch64.cpp b/src/hotspot/cpu/aarch64/nativeInst_aarch64.cpp index 7599c21c2fc..907d4891c71 100644 --- a/src/hotspot/cpu/aarch64/nativeInst_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/nativeInst_aarch64.cpp @@ -544,3 +544,29 @@ address NativeCall::trampoline_jump(CodeBuffer &cbuf, address dest) { return stub; } + +void NativePostCallNop::make_deopt() { + NativeDeoptInstruction::insert(addr_at(0)); +} + +void NativePostCallNop::patch(jint diff) { + // unsupported for now +} + +void NativeDeoptInstruction::verify() { +} + +// Inserts an undefined instruction at a given pc +void NativeDeoptInstruction::insert(address code_pos) { + // 1 1 0 1 | 0 1 0 0 | 1 0 1 imm16 0 0 0 0 1 + // d | 4 | a | de | 0 | 0 | + // 0xd4, 0x20, 0x00, 0x00 + uint32_t insn = 0xd4ade001; + uint32_t *pos = (uint32_t *) code_pos; + *pos = insn; + /**code_pos = 0xd4; + *(code_pos+1) = 0x60; + *(code_pos+2) = 0x00; + *(code_pos+3) = 0x00;*/ + ICache::invalidate_range(code_pos, 4); +} diff --git a/src/hotspot/cpu/aarch64/nativeInst_aarch64.hpp b/src/hotspot/cpu/aarch64/nativeInst_aarch64.hpp index 75f2797c326..1128648b8cb 100644 --- a/src/hotspot/cpu/aarch64/nativeInst_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/nativeInst_aarch64.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2022, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2014, 2108, Red Hat Inc. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * @@ -71,7 +71,7 @@ public: return (encoding() & 0xff000000) == 0x10000000; } - inline bool is_nop(); + inline bool is_nop() const; bool is_jump(); bool is_general_jump(); inline bool is_jump_or_nop(); @@ -543,7 +543,7 @@ class NativeTstRegMem: public NativeInstruction { public: }; -inline bool NativeInstruction::is_nop() { +inline bool NativeInstruction::is_nop() const{ uint32_t insn = *(uint32_t*)addr_at(0); return insn == 0xd503201f; } @@ -666,4 +666,48 @@ inline NativeLdSt* NativeLdSt_at(address addr) { return (NativeLdSt*)addr; } +class NativePostCallNop: public NativeInstruction { +public: + bool check() const { return is_nop(); } + int displacement() const { return 0; } + void patch(jint diff); + void make_deopt(); +}; + +inline NativePostCallNop* nativePostCallNop_at(address address) { + NativePostCallNop* nop = (NativePostCallNop*) address; + if (nop->check()) { + return nop; + } + return NULL; +} + +inline NativePostCallNop* nativePostCallNop_unsafe_at(address address) { + NativePostCallNop* nop = (NativePostCallNop*) address; + assert(nop->check(), ""); + return nop; +} + +class NativeDeoptInstruction: public NativeInstruction { + public: + enum { + instruction_size = 4, + instruction_offset = 0, + }; + + address instruction_address() const { return addr_at(instruction_offset); } + address next_instruction_address() const { return addr_at(instruction_size); } + + void verify(); + + static bool is_deopt_at(address instr) { + assert(instr != NULL, ""); + uint32_t value = *(uint32_t *) instr; + return value == 0xd4ade001; + } + + // MT-safe patching + static void insert(address code_pos); +}; + #endif // CPU_AARCH64_NATIVEINST_AARCH64_HPP diff --git a/src/hotspot/cpu/aarch64/registerMap_aarch64.cpp b/src/hotspot/cpu/aarch64/registerMap_aarch64.cpp index 18e55fa0348..dc5f0a096cb 100644 --- a/src/hotspot/cpu/aarch64/registerMap_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/registerMap_aarch64.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2021, 2022, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2021, Arm Limited. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * @@ -36,13 +36,13 @@ address RegisterMap::pd_location(VMReg base_reg, int slot_idx) const { int base_reg_enc = (base_reg->value() - ConcreteRegisterImpl::max_gpr) / FloatRegisterImpl::max_slots_per_register; intptr_t offset_in_bytes = slot_idx * VMRegImpl::stack_slot_size; - address base_location = location(base_reg); + address base_location = location(base_reg, nullptr); if (base_location != NULL) { return base_location + offset_in_bytes; } else { return NULL; } } else { - return location(base_reg->next(slot_idx)); + return location(base_reg->next(slot_idx), nullptr); } } diff --git a/src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp b/src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp index 35710f020dc..8cc5f9f171d 100644 --- a/src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp @@ -28,6 +28,7 @@ #include "asm/macroAssembler.hpp" #include "asm/macroAssembler.inline.hpp" #include "code/codeCache.hpp" +#include "code/compiledIC.hpp" #include "code/debugInfoRec.hpp" #include "code/icBuffer.hpp" #include "code/vtableStubs.hpp" @@ -41,6 +42,8 @@ #include "oops/compiledICHolder.hpp" #include "oops/klass.inline.hpp" #include "prims/methodHandles.hpp" +#include "runtime/continuation.hpp" +#include "runtime/continuationEntry.inline.hpp" #include "runtime/jniHandles.hpp" #include "runtime/safepointMechanism.hpp" #include "runtime/sharedRuntime.hpp" @@ -267,7 +270,7 @@ void RegisterSaver::restore_live_registers(MacroAssembler* masm) { #endif __ pop_CPU_state(_save_vectors); #endif - __ leave(); + __ ldp(rfp, lr, Address(__ post(sp, 2 * wordSize))); } @@ -720,6 +723,10 @@ void SharedRuntime::gen_i2c_adapter(MacroAssembler *masm, } } + __ mov(rscratch2, rscratch1); + __ push_cont_fastpath(rthread); // Set JavaThread::_cont_fastpath to the sp of the oldest interpreted frame we know about; kills rscratch1 + __ mov(rscratch1, rscratch2); + // 6243940 We might end up in handle_wrong_method if // the callee is deoptimized as we race thru here. If that // happens we don't want to take a safepoint because the @@ -1182,6 +1189,91 @@ static void verify_oop_args(MacroAssembler* masm, } } +// defined in stubGenerator_aarch64.cpp +OopMap* continuation_enter_setup(MacroAssembler* masm, int& stack_slots); +void fill_continuation_entry(MacroAssembler* masm); +void continuation_enter_cleanup(MacroAssembler* masm); + +// enterSpecial(Continuation c, boolean isContinue, boolean isVirtualThread) +// On entry: c_rarg1 -- the continuation object +// c_rarg2 -- isContinue +// c_rarg3 -- isVirtualThread +static void gen_continuation_enter(MacroAssembler* masm, + const methodHandle& method, + const BasicType* sig_bt, + const VMRegPair* regs, + int& exception_offset, + OopMapSet*oop_maps, + int& frame_complete, + int& stack_slots) { + //verify_oop_args(masm, method, sig_bt, regs); + Address resolve(SharedRuntime::get_resolve_static_call_stub(), + relocInfo::static_call_type); + + stack_slots = 2; // will be overwritten + address start = __ pc(); + + Label call_thaw, exit; + + __ enter(); + + OopMap* map = continuation_enter_setup(masm, stack_slots); + + // Frame is now completed as far as size and linkage. + frame_complete =__ pc() - start; + + fill_continuation_entry(masm); + + __ cmp(c_rarg2, (u1)0); + __ br(Assembler::NE, call_thaw); + + address mark = __ pc(); + + __ trampoline_call1(resolve, NULL, false); + + oop_maps->add_gc_map(__ pc() - start, map); + __ post_call_nop(); + + __ b(exit); + + __ bind(call_thaw); + + rt_call(masm, CAST_FROM_FN_PTR(address, StubRoutines::cont_thaw())); + oop_maps->add_gc_map(__ pc() - start, map->deep_copy()); + ContinuationEntry::return_pc_offset = __ pc() - start; + __ post_call_nop(); + + __ bind(exit); + continuation_enter_cleanup(masm); + __ leave(); + __ ret(lr); + + /// exception handling + + exception_offset = __ pc() - start; + { + __ mov(r19, r0); // save return value contaning the exception oop in callee-saved R19 + + continuation_enter_cleanup(masm); + + __ ldr(c_rarg1, Address(rfp, wordSize)); // return address + __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), rthread, c_rarg1); + + // see OptoRuntime::generate_exception_blob: r0 -- exception oop, r3 -- exception pc + + __ mov(r1, r0); // the exception handler + __ mov(r0, r19); // restore return value contaning the exception oop + __ verify_oop(r0); + + __ leave(); + __ mov(r3, lr); + __ br(r1); // the exception handler + } + + CodeBuffer* cbuf = masm->code_section()->outer(); + address stub = CompiledStaticCall::emit_to_interp_stub(*cbuf, mark); +} + static void gen_special_dispatch(MacroAssembler* masm, const methodHandle& method, const BasicType* sig_bt, @@ -1263,6 +1355,37 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm, BasicType* in_sig_bt, VMRegPair* in_regs, BasicType ret_type) { + if (method->is_continuation_enter_intrinsic()) { + vmIntrinsics::ID iid = method->intrinsic_id(); + intptr_t start = (intptr_t)__ pc(); + int vep_offset = ((intptr_t)__ pc()) - start; + int exception_offset = 0; + int frame_complete = 0; + int stack_slots = 0; + OopMapSet* oop_maps = new OopMapSet(); + gen_continuation_enter(masm, + method, + in_sig_bt, + in_regs, + exception_offset, + oop_maps, + frame_complete, + stack_slots); + __ flush(); + nmethod* nm = nmethod::new_native_nmethod(method, + compile_id, + masm->code(), + vep_offset, + frame_complete, + stack_slots, + in_ByteSize(-1), + in_ByteSize(-1), + oop_maps, + exception_offset); + ContinuationEntry::set_enter_nmethod(nm); + return nm; + } + if (method->is_method_handle_intrinsic()) { vmIntrinsics::ID iid = method->intrinsic_id(); intptr_t start = (intptr_t)__ pc(); diff --git a/src/hotspot/cpu/aarch64/smallRegisterMap_aarch64.inline.hpp b/src/hotspot/cpu/aarch64/smallRegisterMap_aarch64.inline.hpp new file mode 100644 index 00000000000..db38b9cf5df --- /dev/null +++ b/src/hotspot/cpu/aarch64/smallRegisterMap_aarch64.inline.hpp @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_AARCH64_SMALLREGISTERMAP_AARCH64_INLINE_HPP +#define CPU_AARCH64_SMALLREGISTERMAP_AARCH64_INLINE_HPP + +#include "runtime/frame.inline.hpp" +#include "runtime/registerMap.hpp" + +// Java frames don't have callee saved registers (except for rfp), so we can use a smaller RegisterMap +class SmallRegisterMap { +public: + static constexpr SmallRegisterMap* instance = nullptr; +private: + static void assert_is_rfp(VMReg r) NOT_DEBUG_RETURN + DEBUG_ONLY({ assert (r == rfp->as_VMReg() || r == rfp->as_VMReg()->next(), "Reg: %s", r->name()); }) +public: + // as_RegisterMap is used when we didn't want to templatize and abstract over RegisterMap type to support SmallRegisterMap + // Consider enhancing SmallRegisterMap to support those cases + const RegisterMap* as_RegisterMap() const { return nullptr; } + RegisterMap* as_RegisterMap() { return nullptr; } + + RegisterMap* copy_to_RegisterMap(RegisterMap* map, intptr_t* sp) const { + map->clear(); + map->set_include_argument_oops(this->include_argument_oops()); + frame::update_map_with_saved_link(map, (intptr_t**)sp - frame::sender_sp_offset); + return map; + } + + SmallRegisterMap() {} + + SmallRegisterMap(const RegisterMap* map) { + #ifdef ASSERT + for(int i = 0; i < RegisterMap::reg_count; i++) { + VMReg r = VMRegImpl::as_VMReg(i); + if (map->location(r, (intptr_t*)nullptr) != nullptr) assert_is_rfp(r); + } + #endif + } + + inline address location(VMReg reg, intptr_t* sp) const { + assert_is_rfp(reg); + return (address)(sp - frame::sender_sp_offset); + } + + inline void set_location(VMReg reg, address loc) { assert_is_rfp(reg); } + + JavaThread* thread() const { + #ifndef ASSERT + guarantee (false, "unreachable"); + #endif + return nullptr; + } + + bool update_map() const { return false; } + bool walk_cont() const { return false; } + bool include_argument_oops() const { return false; } + void set_include_argument_oops(bool f) {} + bool in_cont() const { return false; } + stackChunkHandle stack_chunk() const { return stackChunkHandle(); } + +#ifdef ASSERT + bool should_skip_missing() const { return false; } + VMReg find_register_spilled_here(void* p, intptr_t* sp) { return rfp->as_VMReg(); } + void print() const { print_on(tty); } + void print_on(outputStream* st) const { st->print_cr("Small register map"); } +#endif +}; + +#endif // CPU_AARCH64_SMALLREGISTERMAP_AARCH64_INLINE_HPP diff --git a/src/hotspot/cpu/aarch64/stackChunkFrameStream_aarch64.inline.hpp b/src/hotspot/cpu/aarch64/stackChunkFrameStream_aarch64.inline.hpp new file mode 100644 index 00000000000..58200559181 --- /dev/null +++ b/src/hotspot/cpu/aarch64/stackChunkFrameStream_aarch64.inline.hpp @@ -0,0 +1,149 @@ +/* + * Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_AARCH64_STACKCHUNKFRAMESTREAM_AARCH64_INLINE_HPP +#define CPU_AARCH64_STACKCHUNKFRAMESTREAM_AARCH64_INLINE_HPP + +#include "interpreter/oopMapCache.hpp" +#include "runtime/frame.inline.hpp" +#include "runtime/registerMap.hpp" + +#ifdef ASSERT +template +inline bool StackChunkFrameStream::is_in_frame(void* p0) const { + assert(!is_done(), ""); + intptr_t* p = (intptr_t*)p0; + int argsize = is_compiled() ? (_cb->as_compiled_method()->method()->num_stack_arg_slots() * VMRegImpl::stack_slot_size) >> LogBytesPerWord : 0; + int frame_size = _cb->frame_size() + argsize; + return p == sp() - frame::sender_sp_offset || ((p - unextended_sp()) >= 0 && (p - unextended_sp()) < frame_size); +} +#endif + +template +inline frame StackChunkFrameStream::to_frame() const { + if (is_done()) { + return frame(_sp, _sp, nullptr, nullptr, nullptr, nullptr, true); + } else { + return frame(sp(), unextended_sp(), fp(), pc(), cb(), _oopmap, true); + } +} + +template +inline address StackChunkFrameStream::get_pc() const { + assert(!is_done(), ""); + return *(address*)(_sp - 1); +} + +template +inline intptr_t* StackChunkFrameStream::fp() const { + intptr_t* fp_addr = _sp - frame::sender_sp_offset; + return (frame_kind == ChunkFrames::Mixed && is_interpreted()) + ? fp_addr + *fp_addr // derelativize + : *(intptr_t**)fp_addr; +} + +template +inline intptr_t* StackChunkFrameStream::derelativize(int offset) const { + intptr_t* fp = this->fp(); + assert(fp != nullptr, ""); + return fp + fp[offset]; +} + +template +inline intptr_t* StackChunkFrameStream::unextended_sp_for_interpreter_frame() const { + assert_is_interpreted_and_frame_type_mixed(); + return derelativize(frame::interpreter_frame_last_sp_offset); +} + +template +intptr_t* StackChunkFrameStream::next_sp_for_interpreter_frame() const { + assert_is_interpreted_and_frame_type_mixed(); + return (derelativize(frame::interpreter_frame_locals_offset) + 1 >= _end) ? _end : fp() + frame::sender_sp_offset; +} + +template +inline void StackChunkFrameStream::next_for_interpreter_frame() { + assert_is_interpreted_and_frame_type_mixed(); + if (derelativize(frame::interpreter_frame_locals_offset) + 1 >= _end) { + _unextended_sp = _end; + _sp = _end; + } else { + intptr_t* fp = this->fp(); + _unextended_sp = fp + fp[frame::interpreter_frame_sender_sp_offset]; + _sp = fp + frame::sender_sp_offset; + } +} + +template +inline int StackChunkFrameStream::interpreter_frame_size() const { + assert_is_interpreted_and_frame_type_mixed(); + + intptr_t* top = unextended_sp(); // later subtract argsize if callee is interpreted + intptr_t* bottom = derelativize(frame::interpreter_frame_locals_offset) + 1; // the sender's unextended sp: derelativize(frame::interpreter_frame_sender_sp_offset); + return (int)(bottom - top); +} + +template +inline int StackChunkFrameStream::interpreter_frame_stack_argsize() const { + assert_is_interpreted_and_frame_type_mixed(); + int diff = (int)(derelativize(frame::interpreter_frame_locals_offset) - derelativize(frame::interpreter_frame_sender_sp_offset) + 1); + return diff; +} + +template +inline int StackChunkFrameStream::interpreter_frame_num_oops() const { + assert_is_interpreted_and_frame_type_mixed(); + ResourceMark rm; + InterpreterOopMap mask; + frame f = to_frame(); + f.interpreted_frame_oop_map(&mask); + return mask.num_oops() + + 1 // for the mirror oop + + ((intptr_t*)f.interpreter_frame_monitor_begin() + - (intptr_t*)f.interpreter_frame_monitor_end())/BasicObjectLock::size(); +} + +template<> +template<> +inline void StackChunkFrameStream::update_reg_map_pd(RegisterMap* map) { + if (map->update_map()) { + frame::update_map_with_saved_link(map, map->in_cont() ? (intptr_t**)(intptr_t)frame::sender_sp_offset + : (intptr_t**)(_sp - frame::sender_sp_offset)); + } +} + +template<> +template<> +inline void StackChunkFrameStream::update_reg_map_pd(RegisterMap* map) { + if (map->update_map()) { + frame::update_map_with_saved_link(map, map->in_cont() ? (intptr_t**)(intptr_t)frame::sender_sp_offset + : (intptr_t**)(_sp - frame::sender_sp_offset)); + } +} + +template +template +inline void StackChunkFrameStream::update_reg_map_pd(RegisterMapT* map) {} + +#endif // CPU_AARCH64_STACKCHUNKFRAMESTREAM_AARCH64_INLINE_HPP diff --git a/src/hotspot/cpu/aarch64/stackChunkOop_aarch64.inline.hpp b/src/hotspot/cpu/aarch64/stackChunkOop_aarch64.inline.hpp new file mode 100644 index 00000000000..06625c3e0f2 --- /dev/null +++ b/src/hotspot/cpu/aarch64/stackChunkOop_aarch64.inline.hpp @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_AARCH64_STACKCHUNKOOP_AARCH64_INLINE_HPP +#define CPU_AARCH64_STACKCHUNKOOP_AARCH64_INLINE_HPP + +#include "runtime/frame.inline.hpp" + +inline void stackChunkOopDesc::relativize_frame_pd(frame& fr) const { + if (fr.is_interpreted_frame()) { + fr.set_offset_fp(relativize_address(fr.fp())); + } +} + +inline void stackChunkOopDesc::derelativize_frame_pd(frame& fr) const { + if (fr.is_interpreted_frame()) { + fr.set_fp(derelativize_address(fr.offset_fp())); + } +} + +#endif // CPU_AARCH64_STACKCHUNKOOP_AARCH64_INLINE_HPP diff --git a/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp b/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp index 8a36a08439f..850c09a33bd 100644 --- a/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp @@ -42,6 +42,8 @@ #include "oops/oop.inline.hpp" #include "prims/methodHandles.hpp" #include "runtime/atomic.hpp" +#include "runtime/continuation.hpp" +#include "runtime/continuationEntry.inline.hpp" #include "runtime/frame.inline.hpp" #include "runtime/handles.inline.hpp" #include "runtime/sharedRuntime.hpp" @@ -49,6 +51,7 @@ #include "runtime/stubRoutines.hpp" #include "runtime/thread.inline.hpp" #include "utilities/align.hpp" +#include "utilities/globalDefinitions.hpp" #include "utilities/powerOfTwo.hpp" #ifdef COMPILER2 #include "opto/runtime.hpp" @@ -73,6 +76,10 @@ #define BIND(label) bind(label); BLOCK_COMMENT(#label ":") +OopMap* continuation_enter_setup(MacroAssembler* masm, int& stack_slots); +void fill_continuation_entry(MacroAssembler* masm); +void continuation_enter_cleanup(MacroAssembler* masm); + // Stub Code definitions class StubGenerator: public StubCodeGenerator { @@ -349,6 +356,8 @@ class StubGenerator: public StubCodeGenerator { } #endif + __ pop_cont_fastpath(rthread); + // restore callee-save registers __ ldpd(v15, v14, d15_save); __ ldpd(v13, v12, d13_save); @@ -5144,6 +5153,20 @@ class StubGenerator: public StubCodeGenerator { address start = __ pc(); + BarrierSetAssembler* bs_asm = BarrierSet::barrier_set()->barrier_set_assembler(); + + if (bs_asm->nmethod_code_patching()) { + BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod(); + // We can get here despite the nmethod being good, if we have not + // yet applied our cross modification fence. + Address thread_epoch_addr(rthread, in_bytes(bs_nm->thread_disarmed_offset()) + 4); + __ lea(rscratch2, ExternalAddress(bs_asm->patching_epoch_addr())); + __ ldrw(rscratch2, rscratch2); + __ strw(rscratch2, thread_epoch_addr); + __ isb(); + __ membar(__ LoadLoad); + } + __ set_last_Java_frame(sp, rfp, lr, rscratch1); __ enter(); @@ -6466,6 +6489,268 @@ class StubGenerator: public StubCodeGenerator { } #endif // LINUX + RuntimeStub* generate_cont_doYield() { + if (!Continuations::enabled()) return nullptr; + + const char *name = "cont_doYield"; + + enum layout { + rfp_off1, + rfp_off2, + lr_off, + lr_off2, + framesize // inclusive of return address + }; + // assert(is_even(framesize/2), "sp not 16-byte aligned"); + + int insts_size = 512; + int locs_size = 64; + CodeBuffer code(name, insts_size, locs_size); + OopMapSet* oop_maps = new OopMapSet(); + MacroAssembler* masm = new MacroAssembler(&code); + MacroAssembler* _masm = masm; + + address start = __ pc(); + + __ enter(); + + __ mov(c_rarg1, sp); + + int frame_complete = __ pc() - start; + address the_pc = __ pc(); + + __ post_call_nop(); // this must be exactly after the pc value that is pushed into the frame info, we use this nop for fast CodeBlob lookup + + __ mov(c_rarg0, rthread); + __ set_last_Java_frame(sp, rfp, the_pc, rscratch1); + __ call_VM_leaf(Continuation::freeze_entry(), 2); + __ reset_last_Java_frame(true); + + Label pinned; + + __ cbnz(r0, pinned); + + // We've succeeded, set sp to the ContinuationEntry + __ ldr(rscratch1, Address(rthread, JavaThread::cont_entry_offset())); + __ mov(sp, rscratch1); + continuation_enter_cleanup(masm); + + __ bind(pinned); // pinned -- return to caller + + __ leave(); + __ ret(lr); + + OopMap* map = new OopMap(framesize, 1); + oop_maps->add_gc_map(the_pc - start, map); + + RuntimeStub* stub = // codeBlob framesize is in words (not VMRegImpl::slot_size) + RuntimeStub::new_runtime_stub(name, + &code, + frame_complete, + (framesize >> (LogBytesPerWord - LogBytesPerInt)), + oop_maps, false); + return stub; + } + + address generate_cont_thaw(bool return_barrier, bool exception) { + assert(return_barrier || !exception, "must be"); + + address start = __ pc(); + + if (return_barrier) { + __ ldr(rscratch1, Address(rthread, JavaThread::cont_entry_offset())); + __ mov(sp, rscratch1); + } + assert_asm(_masm, (__ ldr(rscratch1, Address(rthread, JavaThread::cont_entry_offset())), __ cmp(sp, rscratch1)), Assembler::EQ, "incorrect sp"); + + if (return_barrier) { + // preserve possible return value from a method returning to the return barrier + __ fmovd(rscratch1, v0); + __ stp(rscratch1, r0, Address(__ pre(sp, -2 * wordSize))); + } + + __ movw(c_rarg1, (return_barrier ? 1 : 0) + (exception ? 1 : 0)); + __ call_VM_leaf(CAST_FROM_FN_PTR(address, Continuation::prepare_thaw), rthread, c_rarg1); + __ mov(rscratch2, r0); // r0 contains the size of the frames to thaw, 0 if overflow or no more frames + + if (return_barrier) { + // restore return value (no safepoint in the call to thaw, so even an oop return value should be OK) + __ ldp(rscratch1, r0, Address(__ post(sp, 2 * wordSize))); + __ fmovd(v0, rscratch1); + } + assert_asm(_masm, (__ ldr(rscratch1, Address(rthread, JavaThread::cont_entry_offset())), __ cmp(sp, rscratch1)), Assembler::EQ, "incorrect sp"); + + + Label thaw_success; + // rscratch2 contains the size of the frames to thaw, 0 if overflow or no more frames + __ cbnz(rscratch2, thaw_success); + __ lea(rscratch1, ExternalAddress(StubRoutines::throw_StackOverflowError_entry())); + __ br(rscratch1); + __ bind(thaw_success); + + // make room for the thawed frames + __ sub(rscratch1, sp, rscratch2); + __ andr(rscratch1, rscratch1, -16); // align + __ mov(sp, rscratch1); + + if (return_barrier) { + // save original return value -- again + __ fmovd(rscratch1, v0); + __ stp(rscratch1, r0, Address(__ pre(sp, -2 * wordSize))); + } + + // If we want, we can templatize thaw by kind, and have three different entries + if (exception) __ movw(c_rarg1, (uint32_t)Continuation::thaw_exception); + else if (return_barrier) __ movw(c_rarg1, (uint32_t)Continuation::thaw_return_barrier); + else __ movw(c_rarg1, (uint32_t)Continuation::thaw_top); + + __ call_VM_leaf(Continuation::thaw_entry(), rthread, c_rarg1); + __ mov(rscratch2, r0); // r0 is the sp of the yielding frame + + if (return_barrier) { + // restore return value (no safepoint in the call to thaw, so even an oop return value should be OK) + __ ldp(rscratch1, r0, Address(__ post(sp, 2 * wordSize))); + __ fmovd(v0, rscratch1); + } else { + __ mov(r0, zr); // return 0 (success) from doYield + } + + // we're now on the yield frame (which is in an address above us b/c rsp has been pushed down) + __ sub(sp, rscratch2, 2*wordSize); // now pointing to rfp spill + __ mov(rfp, sp); + + if (exception) { + __ ldr(c_rarg1, Address(rfp, wordSize)); // return address + __ verify_oop(r0); + __ mov(r19, r0); // save return value contaning the exception oop in callee-saved R19 + + __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), rthread, c_rarg1); + + // Reinitialize the ptrue predicate register, in case the external runtime call clobbers ptrue reg, as we may return to SVE compiled code. + // __ reinitialize_ptrue(); + + // see OptoRuntime::generate_exception_blob: r0 -- exception oop, r3 -- exception pc + + __ mov(r1, r0); // the exception handler + __ mov(r0, r19); // restore return value contaning the exception oop + __ verify_oop(r0); + + __ leave(); + __ mov(r3, lr); + __ br(r1); // the exception handler + } + + // We're "returning" into the topmost thawed frame; see Thaw::push_return_frame + __ leave(); + __ ret(lr); + + return start; + } + + address generate_cont_thaw() { + if (!Continuations::enabled()) return nullptr; + + StubCodeMark mark(this, "StubRoutines", "Cont thaw"); + address start = __ pc(); + generate_cont_thaw(false, false); + return start; + } + + address generate_cont_returnBarrier() { + if (!Continuations::enabled()) return nullptr; + + // TODO: will probably need multiple return barriers depending on return type + StubCodeMark mark(this, "StubRoutines", "cont return barrier"); + address start = __ pc(); + + generate_cont_thaw(true, false); + + return start; + } + + address generate_cont_returnBarrier_exception() { + if (!Continuations::enabled()) return nullptr; + + StubCodeMark mark(this, "StubRoutines", "cont return barrier exception handler"); + address start = __ pc(); + + generate_cont_thaw(true, true); + + return start; + } + +#if INCLUDE_JFR + + static void jfr_prologue(address the_pc, MacroAssembler* _masm, Register thread) { + __ set_last_Java_frame(sp, rfp, the_pc, rscratch1); + __ mov(c_rarg0, thread); + } + + // The handle is dereferenced through a load barrier. + static void jfr_epilogue(MacroAssembler* _masm, Register thread) { + __ reset_last_Java_frame(true); + Label null_jobject; + __ cbz(r0, null_jobject); + DecoratorSet decorators = ACCESS_READ | IN_NATIVE; + BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler(); + bs->load_at(_masm, decorators, T_OBJECT, r0, Address(r0, 0), c_rarg0, thread); + __ bind(null_jobject); + } + + static RuntimeStub* generate_jfr_stub(const char* name, address entrypoint) { + + enum layout { + rbp_off, + rbpH_off, + return_off, + return_off2, + framesize // inclusive of return address + }; + + int insts_size = 512; + int locs_size = 64; + CodeBuffer code(name, insts_size, locs_size); + OopMapSet* oop_maps = new OopMapSet(); + MacroAssembler* masm = new MacroAssembler(&code); + MacroAssembler* _masm = masm; + + address start = __ pc(); + __ enter(); + int frame_complete = __ pc() - start; + address the_pc = __ pc(); + jfr_prologue(the_pc, _masm, rthread); + __ call_VM_leaf(entrypoint, 1); + jfr_epilogue(_masm, rthread); + __ leave(); + __ ret(lr); + + OopMap* map = new OopMap(framesize, 1); // rfp + oop_maps->add_gc_map(the_pc - start, map); + + RuntimeStub* stub = // codeBlob framesize is in words (not VMRegImpl::slot_size) + RuntimeStub::new_runtime_stub(name, &code, frame_complete, + (framesize >> (LogBytesPerWord - LogBytesPerInt)), + oop_maps, false); + return stub; + } + + // For c2: c_rarg0 is junk, call to runtime to write a checkpoint. + // It returns a jobject handle to the event writer. + // The handle is dereferenced and the return value is the event writer oop. + RuntimeStub* generate_jfr_write_checkpoint() { + return generate_jfr_stub("jfr_write_checkpoint", + CAST_FROM_FN_PTR(address, JfrIntrinsicSupport::write_checkpoint)); + } + + // For c1: call the corresponding runtime routine, it returns a jobject handle to the event writer. + // The handle is dereferenced and the return value is the event writer oop. + RuntimeStub* generate_jfr_get_event_writer() { + return generate_jfr_stub("jfr_get_event_writer", + CAST_FROM_FN_PTR(address, JfrIntrinsicSupport::event_writer)); + } + +#endif // INCLUDE_JFR + // Continuation point for throwing of implicit exceptions that are // not handled in the current activation. Fabricates an exception // oop and initiates normal exception dispatching in this @@ -7450,6 +7735,21 @@ class StubGenerator: public StubCodeGenerator { } } + void generate_phase1() { + // Continuation stubs: + StubRoutines::_cont_thaw = generate_cont_thaw(); + StubRoutines::_cont_returnBarrier = generate_cont_returnBarrier(); + StubRoutines::_cont_returnBarrierExc = generate_cont_returnBarrier_exception(); + StubRoutines::_cont_doYield_stub = generate_cont_doYield(); + StubRoutines::_cont_doYield = StubRoutines::_cont_doYield_stub == nullptr ? nullptr + : StubRoutines::_cont_doYield_stub->entry_point(); + + JFR_ONLY(StubRoutines::_jfr_write_checkpoint_stub = generate_jfr_write_checkpoint();) + JFR_ONLY(StubRoutines::_jfr_write_checkpoint = StubRoutines::_jfr_write_checkpoint_stub->entry_point();) + JFR_ONLY(StubRoutines::_jfr_get_event_writer_stub = generate_jfr_get_event_writer();) + JFR_ONLY(StubRoutines::_jfr_get_event_writer = StubRoutines::_jfr_get_event_writer_stub->entry_point();) + } + void generate_all() { // support for verify_oop (must happen after universe_init) if (VerifyOops) { @@ -7592,21 +7892,23 @@ class StubGenerator: public StubCodeGenerator { } public: - StubGenerator(CodeBuffer* code, bool all) : StubCodeGenerator(code) { - if (all) { - generate_all(); - } else { + StubGenerator(CodeBuffer* code, int phase) : StubCodeGenerator(code) { + if (phase == 0) { generate_initial(); + } else if (phase == 1) { + generate_phase1(); // stubs that must be available for the interpreter + } else { + generate_all(); } } }; // end class declaration #define UCM_TABLE_MAX_ENTRIES 8 -void StubGenerator_generate(CodeBuffer* code, bool all) { +void StubGenerator_generate(CodeBuffer* code, int phase) { if (UnsafeCopyMemory::_table == NULL) { UnsafeCopyMemory::create_table(UCM_TABLE_MAX_ENTRIES); } - StubGenerator g(code, all); + StubGenerator g(code, phase); } @@ -7641,3 +7943,75 @@ DEFAULT_ATOMIC_OP(cmpxchg, 8, _seq_cst) #undef DEFAULT_ATOMIC_OP #endif // LINUX + + +#undef __ +#define __ masm-> + +// on exit, sp points to the ContinuationEntry +OopMap* continuation_enter_setup(MacroAssembler* masm, int& stack_slots) { + assert(ContinuationEntry::size() % VMRegImpl::stack_slot_size == 0, ""); + assert(in_bytes(ContinuationEntry::cont_offset()) % VMRegImpl::stack_slot_size == 0, ""); + assert(in_bytes(ContinuationEntry::chunk_offset()) % VMRegImpl::stack_slot_size == 0, ""); + + stack_slots += (int)ContinuationEntry::size()/wordSize; + __ sub(sp, sp, (int)ContinuationEntry::size()); // place Continuation metadata + + OopMap* map = new OopMap(((int)ContinuationEntry::size() + wordSize)/ VMRegImpl::stack_slot_size, 0 /* arg_slots*/); + ContinuationEntry::setup_oopmap(map); + + __ ldr(rscratch1, Address(rthread, JavaThread::cont_entry_offset())); + __ str(rscratch1, Address(sp, ContinuationEntry::parent_offset())); + __ mov(rscratch1, sp); // we can't use sp as the source in str + __ str(rscratch1, Address(rthread, JavaThread::cont_entry_offset())); + + return map; +} + +// on entry c_rarg1 points to the continuation +// sp points to ContinuationEntry +// c_rarg3 -- isVirtualThread +void fill_continuation_entry(MacroAssembler* masm) { +#ifdef ASSERT + __ movw(rscratch1, 0x1234); + __ strw(rscratch1, Address(sp, ContinuationEntry::cookie_offset())); +#endif + + __ str (c_rarg1, Address(sp, ContinuationEntry::cont_offset())); + __ strw(c_rarg3, Address(sp, ContinuationEntry::flags_offset())); + __ str (zr, Address(sp, ContinuationEntry::chunk_offset())); + __ strw(zr, Address(sp, ContinuationEntry::argsize_offset())); + __ strw(zr, Address(sp, ContinuationEntry::pin_count_offset())); + + __ ldr(rscratch1, Address(rthread, JavaThread::cont_fastpath_offset())); + __ str(rscratch1, Address(sp, ContinuationEntry::parent_cont_fastpath_offset())); + __ ldrw(rscratch1, Address(rthread, JavaThread::held_monitor_count_offset())); + __ strw(rscratch1, Address(sp, ContinuationEntry::parent_held_monitor_count_offset())); + + __ str(zr, Address(rthread, JavaThread::cont_fastpath_offset())); + __ reset_held_monitor_count(rthread); +} + +// on entry, sp points to the ContinuationEntry +// on exit, rfp points to the spilled rfp in the entry frame +void continuation_enter_cleanup(MacroAssembler* masm) { +#ifndef PRODUCT + Label OK; + __ ldr(rscratch1, Address(rthread, JavaThread::cont_entry_offset())); + __ cmp(sp, rscratch1); + __ br(Assembler::EQ, OK); + __ stop("incorrect sp1"); + __ bind(OK); +#endif + + __ ldr(rscratch1, Address(sp, ContinuationEntry::parent_cont_fastpath_offset())); + __ str(rscratch1, Address(rthread, JavaThread::cont_fastpath_offset())); + __ ldrw(rscratch1, Address(sp, ContinuationEntry::parent_held_monitor_count_offset())); + __ strw(rscratch1, Address(rthread, JavaThread::held_monitor_count_offset())); + + __ ldr(rscratch2, Address(sp, ContinuationEntry::parent_offset())); + __ str(rscratch2, Address(rthread, JavaThread::cont_entry_offset())); + __ add(rfp, sp, (int)ContinuationEntry::size()); +} + +#undef __ diff --git a/src/hotspot/cpu/aarch64/templateInterpreterGenerator_aarch64.cpp b/src/hotspot/cpu/aarch64/templateInterpreterGenerator_aarch64.cpp index 2df88c2d069..7ba9b864f56 100644 --- a/src/hotspot/cpu/aarch64/templateInterpreterGenerator_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/templateInterpreterGenerator_aarch64.cpp @@ -573,7 +573,9 @@ address TemplateInterpreterGenerator::generate_safept_entry_for( address runtime_entry) { address entry = __ pc(); __ push(state); + __ push_cont_fastpath(rthread); __ call_VM(noreg, runtime_entry); + __ pop_cont_fastpath(rthread); __ membar(Assembler::AnyAny); __ dispatch_via(vtos, Interpreter::_normal_table.table_for(vtos)); return entry; @@ -786,6 +788,7 @@ void TemplateInterpreterGenerator::lock_method() { __ str(r0, Address(esp, BasicObjectLock::obj_offset_in_bytes())); __ mov(c_rarg1, esp); // object address __ lock_object(c_rarg1); + __ inc_held_monitor_count(rthread); } // Generate a fixed interpreter frame. This is identical setup for @@ -852,6 +855,18 @@ void TemplateInterpreterGenerator::generate_fixed_frame(bool native_call) { // End of helpers +address TemplateInterpreterGenerator::generate_Continuation_doYield_entry(void) { + if (!Continuations::enabled()) return nullptr; + + address entry = __ pc(); + assert(StubRoutines::cont_doYield() != NULL, "stub not yet generated"); + + __ push_cont_fastpath(rthread); + __ far_jump(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::cont_doYield()))); + + return entry; +} + // Various method entries //------------------------------------------------------------------------------------------------------------------------ // @@ -1490,6 +1505,7 @@ address TemplateInterpreterGenerator::generate_native_entry(bool synchronized) { __ bind(unlock); __ unlock_object(c_rarg1); + __ dec_held_monitor_count(rthread); } __ bind(L); } diff --git a/src/hotspot/cpu/aarch64/templateTable_aarch64.cpp b/src/hotspot/cpu/aarch64/templateTable_aarch64.cpp index 981296b5fc2..3ed1c78175a 100644 --- a/src/hotspot/cpu/aarch64/templateTable_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/templateTable_aarch64.cpp @@ -3862,6 +3862,9 @@ void TemplateTable::monitorenter() __ str(r0, Address(c_rarg1, BasicObjectLock::obj_offset_in_bytes())); __ lock_object(c_rarg1); + // The object is stored so counter should be increased even if stackoverflow is generated + __ inc_held_monitor_count(rthread); + // check to make sure this monitor doesn't cause stack overflow after locking __ save_bcp(); // in case of exception __ generate_stack_overflow_check(0); @@ -3920,6 +3923,7 @@ void TemplateTable::monitorexit() __ bind(found); __ push_ptr(r0); // make sure object is on stack (contract with oopMaps) __ unlock_object(c_rarg1); + __ dec_held_monitor_count(rthread); __ pop_ptr(r0); // discard object } diff --git a/src/hotspot/cpu/arm/c1_LIRGenerator_arm.cpp b/src/hotspot/cpu/arm/c1_LIRGenerator_arm.cpp index 4005ec619ea..71dd3e056f2 100644 --- a/src/hotspot/cpu/arm/c1_LIRGenerator_arm.cpp +++ b/src/hotspot/cpu/arm/c1_LIRGenerator_arm.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2008, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -1328,3 +1328,7 @@ void LIRGenerator::volatile_field_load(LIR_Address* address, LIR_Opr result, } __ load(address, result, info, lir_patch_none); } + +void LIRGenerator::do_continuation_doYield(Intrinsic* x) { + fatal("Continuation.doYield intrinsic is not implemented on this platform"); +} diff --git a/src/hotspot/cpu/arm/continuationEntry_arm.inline.hpp b/src/hotspot/cpu/arm/continuationEntry_arm.inline.hpp new file mode 100644 index 00000000000..3917f71656c --- /dev/null +++ b/src/hotspot/cpu/arm/continuationEntry_arm.inline.hpp @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_ARM_CONTINUATIONENTRY_ARM_INLINE_HPP +#define CPU_ARM_CONTINUATIONENTRY_ARM_INLINE_HPP + + +#include "runtime/continuationEntry.hpp" + +// TODO: Implement + +inline frame ContinuationEntry::to_frame() const { + Unimplemented(); + return frame(); +} + +inline intptr_t* ContinuationEntry::entry_fp() const { + Unimplemented(); + return nullptr; +} + +inline void ContinuationEntry::update_register_map(RegisterMap* map) const { + Unimplemented(); +} + +#endif // CPU_ARM_CONTINUATIONENTRY_ARM_INLINE_HPP diff --git a/src/hotspot/cpu/arm/continuationFreezeThaw_arm.inline.hpp b/src/hotspot/cpu/arm/continuationFreezeThaw_arm.inline.hpp new file mode 100644 index 00000000000..1c2f75b9e35 --- /dev/null +++ b/src/hotspot/cpu/arm/continuationFreezeThaw_arm.inline.hpp @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_ARM_CONTINUATION_ARM_INLINE_HPP +#define CPU_ARM_CONTINUATION_ARM_INLINE_HPP + +#include "oops/stackChunkOop.inline.hpp" +#include "runtime/frame.hpp" +#include "runtime/frame.inline.hpp" + +inline void FreezeBase::set_top_frame_metadata_pd(const frame& hf) { + Unimplemented(); +} + +template +inline frame FreezeBase::sender(const frame& f) { + Unimplemented(); + return frame(); +} + +template frame FreezeBase::new_heap_frame(frame& f, frame& caller) { + Unimplemented(); + return frame(); +} + +void FreezeBase::adjust_interpreted_frame_unextended_sp(frame& f) { + Unimplemented(); +} + +inline void FreezeBase::relativize_interpreted_frame_metadata(const frame& f, const frame& hf) { + Unimplemented(); +} + +inline void FreezeBase::patch_pd(frame& hf, const frame& caller) { + Unimplemented(); +} + +inline void FreezeBase::patch_stack_pd(intptr_t* frame_sp, intptr_t* heap_sp) { + Unimplemented(); +} + +inline frame ThawBase::new_entry_frame() { + Unimplemented(); + return frame(); +} + +template frame ThawBase::new_stack_frame(const frame& hf, frame& caller, bool bottom) { + Unimplemented(); + return frame(); +} + +inline void ThawBase::set_interpreter_frame_bottom(const frame& f, intptr_t* bottom) { + Unimplemented(); +} + +inline void ThawBase::derelativize_interpreted_frame_metadata(const frame& hf, const frame& f) { + Unimplemented(); +} + +inline intptr_t* ThawBase::align(const frame& hf, intptr_t* frame_sp, frame& caller, bool bottom) { + Unimplemented(); + return NULL; +} + +inline void ThawBase::patch_pd(frame& f, const frame& caller) { + Unimplemented(); +} + +void ThawBase::patch_chunk_pd(intptr_t* sp) { + Unimplemented(); +} + +inline void ThawBase::prefetch_chunk_pd(void* start, int size) { + Unimplemented(); +} + +#endif // CPU_ARM_CONTINUATION_ARM_INLINE_HPP diff --git a/src/hotspot/cpu/arm/continuationHelper_arm.inline.hpp b/src/hotspot/cpu/arm/continuationHelper_arm.inline.hpp new file mode 100644 index 00000000000..bf194b1e469 --- /dev/null +++ b/src/hotspot/cpu/arm/continuationHelper_arm.inline.hpp @@ -0,0 +1,125 @@ +/* + * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_ARM_CONTINUATIONHELPER_ARM_INLINE_HPP +#define CPU_ARM_CONTINUATIONHELPER_ARM_INLINE_HPP + +#include "runtime/continuationHelper.hpp" + +// TODO: Implement + +template +static inline intptr_t** link_address(const frame& f) { + Unimplemented(); + return NULL; +} + +inline int ContinuationHelper::frame_align_words(int size) { + Unimplemented(); + return 0; +} + +inline intptr_t* ContinuationHelper::frame_align_pointer(intptr_t* sp) { + Unimplemented(); + return NULL; +} + +template +inline void ContinuationHelper::update_register_map(const frame& f, RegisterMap* map) { + Unimplemented(); +} + +inline void ContinuationHelper::update_register_map_with_callee(const frame& f, RegisterMap* map) { + Unimplemented(); +} + +inline void ContinuationHelper::push_pd(const frame& f) { + Unimplemented(); +} + +inline void ContinuationHelper::set_anchor_to_entry_pd(JavaFrameAnchor* anchor, ContinuationEntry* cont) { + Unimplemented(); +} + +#ifdef ASSERT +inline void ContinuationHelper::set_anchor_pd(JavaFrameAnchor* anchor, intptr_t* sp) { + Unimplemented(); +} + +inline bool ContinuationHelper::Frame::assert_frame_laid_out(frame f) { + Unimplemented(); + return false; +} +#endif + +inline intptr_t** ContinuationHelper::Frame::callee_link_address(const frame& f) { + Unimplemented(); + return NULL; +} + +template +static inline intptr_t* real_fp(const frame& f) { + Unimplemented(); + return NULL; +} + +inline address* ContinuationHelper::InterpretedFrame::return_pc_address(const frame& f) { + Unimplemented(); + return NULL; +} + +inline void ContinuationHelper::InterpretedFrame::patch_sender_sp(frame& f, intptr_t* sp) { + Unimplemented(); +} + +inline address* ContinuationHelper::Frame::return_pc_address(const frame& f) { + Unimplemented(); + return NULL; +} + +inline address ContinuationHelper::Frame::real_pc(const frame& f) { + Unimplemented(); + return NULL; +} + +inline void ContinuationHelper::Frame::patch_pc(const frame& f, address pc) { + Unimplemented(); +} + +inline intptr_t* ContinuationHelper::InterpretedFrame::frame_top(const frame& f, InterpreterOopMap* mask) { // inclusive; this will be copied with the frame + Unimplemented(); + return NULL; +} + +inline intptr_t* ContinuationHelper::InterpretedFrame::frame_bottom(const frame& f) { // exclusive; this will not be copied with the frame + Unimplemented(); + return NULL; +} + +inline intptr_t* ContinuationHelper::InterpretedFrame::frame_top(const frame& f, int callee_argsize, bool callee_interpreted) { + Unimplemented(); + return NULL; +} + +#endif // CPU_ARM_CONTINUATIONHELPER_ARM_INLINE_HPP diff --git a/src/hotspot/cpu/arm/frame_arm.cpp b/src/hotspot/cpu/arm/frame_arm.cpp index bd85b1b8954..2b97da814de 100644 --- a/src/hotspot/cpu/arm/frame_arm.cpp +++ b/src/hotspot/cpu/arm/frame_arm.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2008, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -230,10 +230,12 @@ void frame::patch_pc(Thread* thread, address pc) { tty->print_cr("patch_pc at address" INTPTR_FORMAT " [" INTPTR_FORMAT " -> " INTPTR_FORMAT "] ", p2i(pc_addr), p2i(*pc_addr), p2i(pc)); } + DEBUG_ONLY(address old_pc = _pc;) *pc_addr = pc; + _pc = pc; // must be set before call to get_deopt_original_pc address original_pc = CompiledMethod::get_deopt_original_pc(this); if (original_pc != NULL) { - assert(original_pc == _pc, "expected original PC to be stored before patching"); + assert(original_pc == old_pc, "expected original PC to be stored before patching"); _deopt_state = is_deoptimized; // leave _pc as is } else { @@ -246,11 +248,6 @@ bool frame::is_interpreted_frame() const { return Interpreter::contains(pc()); } -int frame::frame_size(RegisterMap* map) const { - frame sender = this->sender(map); - return sender.sp() - sp(); -} - intptr_t* frame::entry_frame_argument_at(int offset) const { assert(is_entry_frame(), "entry frame expected"); // convert offset to index to deal with tsi @@ -277,6 +274,7 @@ BasicObjectLock* frame::interpreter_frame_monitor_begin() const { return (BasicObjectLock*) addr_at(interpreter_frame_monitor_block_bottom_offset); } +// Pointer beyond the "oldest/deepest" BasicObjectLock on stack. BasicObjectLock* frame::interpreter_frame_monitor_end() const { BasicObjectLock* result = (BasicObjectLock*) *addr_at(interpreter_frame_monitor_block_top_offset); // make sure the pointer points inside the frame @@ -397,56 +395,6 @@ frame frame::sender_for_interpreter_frame(RegisterMap* map) const { return frame(sender_sp, unextended_sp, link(), sender_pc()); } -frame frame::sender_for_compiled_frame(RegisterMap* map) const { - assert(map != NULL, "map must be set"); - - // frame owned by optimizing compiler - assert(_cb->frame_size() >= 0, "must have non-zero frame size"); - intptr_t* sender_sp = unextended_sp() + _cb->frame_size(); - intptr_t* unextended_sp = sender_sp; - - address sender_pc = (address) *(sender_sp - sender_sp_offset + return_addr_offset); - - // This is the saved value of FP which may or may not really be an FP. - // It is only an FP if the sender is an interpreter frame (or C1?). - intptr_t** saved_fp_addr = (intptr_t**) (sender_sp - sender_sp_offset + link_offset); - - if (map->update_map()) { - // Tell GC to use argument oopmaps for some runtime stubs that need it. - // For C1, the runtime stub might not have oop maps, so set this flag - // outside of update_register_map. - map->set_include_argument_oops(_cb->caller_must_gc_arguments(map->thread())); - if (_cb->oop_maps() != NULL) { - OopMapSet::update_register_map(this, map); - } - - // Since the prolog does the save and restore of FP there is no oopmap - // for it so we must fill in its location as if there was an oopmap entry - // since if our caller was compiled code there could be live jvm state in it. - update_map_with_saved_link(map, saved_fp_addr); - } - - assert(sender_sp != sp(), "must have changed"); - return frame(sender_sp, unextended_sp, *saved_fp_addr, sender_pc); -} - -frame frame::sender(RegisterMap* map) const { - // Default is we done have to follow them. The sender_for_xxx will - // update it accordingly - map->set_include_argument_oops(false); - - if (is_entry_frame()) return sender_for_entry_frame(map); - if (is_interpreted_frame()) return sender_for_interpreter_frame(map); - assert(_cb == CodeCache::find_blob(pc()),"Must be the same"); - - if (_cb != NULL) { - return sender_for_compiled_frame(map); - } - - assert(false, "should not be called for a C frame"); - return frame(); -} - bool frame::is_interpreted_frame_valid(JavaThread* thread) const { assert(is_interpreted_frame(), "Not an interpreted frame"); // These are reasonable sanity checks @@ -545,7 +493,6 @@ BasicType frame::interpreter_frame_result(oop* oop_result, jvalue* value_result) return type; } - intptr_t* frame::interpreter_frame_tos_at(jint offset) const { int index = (Interpreter::expr_offset_in_bytes(offset)/wordSize); return &interpreter_frame_tos_address()[index]; diff --git a/src/hotspot/cpu/arm/frame_arm.hpp b/src/hotspot/cpu/arm/frame_arm.hpp index d879f9e7cc5..de2e8221382 100644 --- a/src/hotspot/cpu/arm/frame_arm.hpp +++ b/src/hotspot/cpu/arm/frame_arm.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2008, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -54,7 +54,11 @@ interpreter_frame_monitor_block_bottom_offset = interpreter_frame_initial_sp_offset, // Entry frames - entry_frame_call_wrapper_offset = 0 + entry_frame_call_wrapper_offset = 0, + metadata_words = sender_sp_offset, + frame_alignment = 16, + // size, in words, of maximum shift in frame position due to alignment + align_wiggle = 1 }; intptr_t ptr_at(int offset) const { @@ -90,6 +94,8 @@ } #endif + const ImmutableOopMap* get_oop_map() const; + public: // Constructors @@ -110,6 +116,9 @@ // expression stack tos if we are nested in a java call intptr_t* interpreter_frame_last_sp() const; + template + static void update_map_with_saved_link(RegisterMapT* map, intptr_t** link_addr); + // deoptimization support void interpreter_frame_set_last_sp(intptr_t* sp); diff --git a/src/hotspot/cpu/arm/frame_arm.inline.hpp b/src/hotspot/cpu/arm/frame_arm.inline.hpp index 773b6d06f7b..972318bd062 100644 --- a/src/hotspot/cpu/arm/frame_arm.inline.hpp +++ b/src/hotspot/cpu/arm/frame_arm.inline.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2008, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -39,6 +39,12 @@ inline frame::frame() { _fp = NULL; _cb = NULL; _deopt_state = unknown; + _on_heap = false; + DEBUG_ONLY(_frame_index = -1;) +} + +inline frame::frame(intptr_t* sp) { + Unimplemented(); } inline void frame::init(intptr_t* sp, intptr_t* fp, address pc) { @@ -49,6 +55,7 @@ inline void frame::init(intptr_t* sp, intptr_t* fp, address pc) { assert(pc != NULL, "no pc?"); _cb = CodeCache::find_blob(pc); adjust_unextended_sp(); + DEBUG_ONLY(_frame_index = -1;) address original_pc = CompiledMethod::get_deopt_original_pc(this); if (original_pc != NULL) { @@ -57,6 +64,7 @@ inline void frame::init(intptr_t* sp, intptr_t* fp, address pc) { } else { _deopt_state = not_deoptimized; } + _on_heap = false; } inline frame::frame(intptr_t* sp, intptr_t* fp, address pc) { @@ -71,6 +79,7 @@ inline frame::frame(intptr_t* sp, intptr_t* unextended_sp, intptr_t* fp, address assert(pc != NULL, "no pc?"); _cb = CodeCache::find_blob(pc); adjust_unextended_sp(); + DEBUG_ONLY(_frame_index = -1;) address original_pc = CompiledMethod::get_deopt_original_pc(this); if (original_pc != NULL) { @@ -81,6 +90,7 @@ inline frame::frame(intptr_t* sp, intptr_t* unextended_sp, intptr_t* fp, address } else { _deopt_state = not_deoptimized; } + _on_heap = false; } @@ -93,6 +103,7 @@ inline frame::frame(intptr_t* sp, intptr_t* fp) { // assert(_pc != NULL, "no pc?"); // see comments in x86 _cb = CodeCache::find_blob(_pc); adjust_unextended_sp(); + DEBUG_ONLY(_frame_index = -1;) address original_pc = CompiledMethod::get_deopt_original_pc(this); if (original_pc != NULL) { @@ -101,6 +112,7 @@ inline frame::frame(intptr_t* sp, intptr_t* fp) { } else { _deopt_state = not_deoptimized; } + _on_heap = false; } @@ -215,15 +227,107 @@ inline JavaCallWrapper** frame::entry_frame_call_wrapper_addr() const { // Compiled frames inline oop frame::saved_oop_result(RegisterMap* map) const { - oop* result_adr = (oop*) map->location(R0->as_VMReg()); + oop* result_adr = (oop*) map->location(R0->as_VMReg(), nullptr); guarantee(result_adr != NULL, "bad register save location"); return (*result_adr); } inline void frame::set_saved_oop_result(RegisterMap* map, oop obj) { - oop* result_adr = (oop*) map->location(R0->as_VMReg()); + oop* result_adr = (oop*) map->location(R0->as_VMReg(), nullptr); guarantee(result_adr != NULL, "bad register save location"); *result_adr = obj; } +inline int frame::frame_size() const { + return sender_sp() - sp(); +} + +inline const ImmutableOopMap* frame::get_oop_map() const { + Unimplemented(); + return NULL; +} + +inline int frame::compiled_frame_stack_argsize() const { + Unimplemented(); + return 0; +} + +inline void frame::interpreted_frame_oop_map(InterpreterOopMap* mask) const { + Unimplemented(); +} + +inline int frame::sender_sp_ret_address_offset() { + Unimplemented(); + return 0; +} + +inline void frame::set_unextended_sp(intptr_t* value) { + Unimplemented(); +} + +inline int frame::offset_unextended_sp() const { + Unimplemented(); + return 0; +} + +inline void frame::set_offset_unextended_sp(int value) { + Unimplemented(); +} + +//------------------------------------------------------------------------------ +// frame::sender + +inline frame frame::sender(RegisterMap* map) const { + // Default is we done have to follow them. The sender_for_xxx will + // update it accordingly + map->set_include_argument_oops(false); + + if (is_entry_frame()) return sender_for_entry_frame(map); + if (is_interpreted_frame()) return sender_for_interpreter_frame(map); + assert(_cb == CodeCache::find_blob(pc()),"Must be the same"); + + if (_cb != NULL) return sender_for_compiled_frame(map); + + assert(false, "should not be called for a C frame"); + return frame(); +} + +inline frame frame::sender_for_compiled_frame(RegisterMap* map) const { + assert(map != NULL, "map must be set"); + + // frame owned by optimizing compiler + assert(_cb->frame_size() >= 0, "must have non-zero frame size"); + intptr_t* sender_sp = unextended_sp() + _cb->frame_size(); + intptr_t* unextended_sp = sender_sp; + + address sender_pc = (address) *(sender_sp - sender_sp_offset + return_addr_offset); + + // This is the saved value of FP which may or may not really be an FP. + // It is only an FP if the sender is an interpreter frame (or C1?). + intptr_t** saved_fp_addr = (intptr_t**) (sender_sp - sender_sp_offset + link_offset); + + if (map->update_map()) { + // Tell GC to use argument oopmaps for some runtime stubs that need it. + // For C1, the runtime stub might not have oop maps, so set this flag + // outside of update_register_map. + map->set_include_argument_oops(_cb->caller_must_gc_arguments(map->thread())); + if (_cb->oop_maps() != NULL) { + OopMapSet::update_register_map(this, map); + } + + // Since the prolog does the save and restore of FP there is no oopmap + // for it so we must fill in its location as if there was an oopmap entry + // since if our caller was compiled code there could be live jvm state in it. + update_map_with_saved_link(map, saved_fp_addr); + } + + assert(sender_sp != sp(), "must have changed"); + return frame(sender_sp, unextended_sp, *saved_fp_addr, sender_pc); +} + +template +void frame::update_map_with_saved_link(RegisterMapT* map, intptr_t** link_addr) { + Unimplemented(); +} + #endif // CPU_ARM_FRAME_ARM_INLINE_HPP diff --git a/src/hotspot/cpu/arm/nativeInst_arm_32.hpp b/src/hotspot/cpu/arm/nativeInst_arm_32.hpp index b20c279a624..a81cac359f6 100644 --- a/src/hotspot/cpu/arm/nativeInst_arm_32.hpp +++ b/src/hotspot/cpu/arm/nativeInst_arm_32.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2008, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -432,4 +432,35 @@ inline NativeCall* nativeCall_before(address return_address) { return (NativeCall *) rawNativeCall_before(return_address); } +class NativePostCallNop: public NativeInstruction { +public: + bool check() const { Unimplemented(); return false; } + int displacement() const { Unimplemented(); return 0; } + void patch(jint diff) { Unimplemented(); } + void make_deopt() { Unimplemented(); } +}; + +inline NativePostCallNop* nativePostCallNop_at(address address) { + Unimplemented(); + return NULL; +} + +class NativeDeoptInstruction: public NativeInstruction { +public: + address instruction_address() const { Unimplemented(); return NULL; } + address next_instruction_address() const { Unimplemented(); return NULL; } + + void verify() { Unimplemented(); } + + static bool is_deopt_at(address instr) { + Unimplemented(); + return false; + } + + // MT-safe patching + static void insert(address code_pos) { + Unimplemented(); + } +}; + #endif // CPU_ARM_NATIVEINST_ARM_32_HPP diff --git a/src/hotspot/cpu/arm/registerMap_arm.hpp b/src/hotspot/cpu/arm/registerMap_arm.hpp index cf81f8d0e1f..7abea94cca7 100644 --- a/src/hotspot/cpu/arm/registerMap_arm.hpp +++ b/src/hotspot/cpu/arm/registerMap_arm.hpp @@ -37,7 +37,7 @@ address pd_location(VMReg reg) const {return NULL;} address pd_location(VMReg base_reg, int slot_idx) const { - return location(base_reg->next(slot_idx)); + return location(base_reg->next(slot_idx), nullptr); } // no PD state to clear or copy: diff --git a/src/hotspot/cpu/arm/smallRegisterMap_arm.inline.hpp b/src/hotspot/cpu/arm/smallRegisterMap_arm.inline.hpp new file mode 100644 index 00000000000..a298da70ef0 --- /dev/null +++ b/src/hotspot/cpu/arm/smallRegisterMap_arm.inline.hpp @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_ARM_SMALLREGISTERMAP_ARM_INLINE_HPP +#define CPU_ARM_SMALLREGISTERMAP_ARM_INLINE_HPP + +#include "runtime/frame.inline.hpp" +#include "runtime/registerMap.hpp" + +// Java frames don't have callee saved registers (except for rfp), so we can use a smaller RegisterMap +class SmallRegisterMap { +public: + static constexpr SmallRegisterMap* instance = nullptr; +private: + static void assert_is_rfp(VMReg r) PRODUCT_RETURN + DEBUG_ONLY({ Unimplemented(); }) +public: + // as_RegisterMap is used when we didn't want to templatize and abstract over RegisterMap type to support SmallRegisterMap + // Consider enhancing SmallRegisterMap to support those cases + const RegisterMap* as_RegisterMap() const { return nullptr; } + RegisterMap* as_RegisterMap() { return nullptr; } + + RegisterMap* copy_to_RegisterMap(RegisterMap* map, intptr_t* sp) const { + Unimplemented(); + return map; + } + + SmallRegisterMap() {} + + SmallRegisterMap(const RegisterMap* map) { + Unimplemented(); + } + + inline address location(VMReg reg, intptr_t* sp) const { + Unimplemented(); + return NULL; + } + + inline void set_location(VMReg reg, address loc) { assert_is_rfp(reg); } + + JavaThread* thread() const { + #ifndef ASSERT + guarantee (false, ""); + #endif + return nullptr; + } + + bool update_map() const { return false; } + bool walk_cont() const { return false; } + bool include_argument_oops() const { return false; } + void set_include_argument_oops(bool f) {} + bool in_cont() const { return false; } + stackChunkHandle stack_chunk() const { return stackChunkHandle(); } + +#ifdef ASSERT + bool should_skip_missing() const { return false; } + VMReg find_register_spilled_here(void* p, intptr_t* sp) { + Unimplemented(); + return NULL; + } + void print() const { print_on(tty); } + void print_on(outputStream* st) const { st->print_cr("Small register map"); } +#endif +}; + +#endif // CPU_ARM_SMALLREGISTERMAP_ARM_INLINE_HPP diff --git a/src/hotspot/cpu/arm/stackChunkFrameStream_arm.inline.hpp b/src/hotspot/cpu/arm/stackChunkFrameStream_arm.inline.hpp new file mode 100644 index 00000000000..b3c2714037c --- /dev/null +++ b/src/hotspot/cpu/arm/stackChunkFrameStream_arm.inline.hpp @@ -0,0 +1,115 @@ +/* + * Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_ARM_STACKCHUNKFRAMESTREAM_ARM_INLINE_HPP +#define CPU_ARM_STACKCHUNKFRAMESTREAM_ARM_INLINE_HPP + +#include "interpreter/oopMapCache.hpp" +#include "runtime/frame.inline.hpp" +#include "runtime/registerMap.hpp" + +#ifdef ASSERT +template +inline bool StackChunkFrameStream::is_in_frame(void* p0) const { + Unimplemented(); + return true; +} +#endif + +template +inline frame StackChunkFrameStream::to_frame() const { + Unimplemented(); + return frame(); +} + +template +inline address StackChunkFrameStream::get_pc() const { + Unimplemented(); + return NULL; +} + +template +inline intptr_t* StackChunkFrameStream::fp() const { + Unimplemented(); + return NULL; +} + +template +inline intptr_t* StackChunkFrameStream::derelativize(int offset) const { + Unimplemented(); + return NULL; +} + +template +inline intptr_t* StackChunkFrameStream::unextended_sp_for_interpreter_frame() const { + Unimplemented(); + return NULL; +} + +template +intptr_t* StackChunkFrameStream::next_sp_for_interpreter_frame() const { + Unimplemented(); + return NULL; +} + +template +inline void StackChunkFrameStream::next_for_interpreter_frame() { + Unimplemented(); +} + +template +inline int StackChunkFrameStream::interpreter_frame_size() const { + Unimplemented(); + return 0; +} + +template +inline int StackChunkFrameStream::interpreter_frame_stack_argsize() const { + Unimplemented(); + return 0; +} + +template +inline int StackChunkFrameStream::interpreter_frame_num_oops() const { + Unimplemented(); + return 0; +} + +template<> +template<> +inline void StackChunkFrameStream::update_reg_map_pd(RegisterMap* map) { + Unimplemented(); +} + +template<> +template<> +inline void StackChunkFrameStream::update_reg_map_pd(RegisterMap* map) { + Unimplemented(); +} + +template +template +inline void StackChunkFrameStream::update_reg_map_pd(RegisterMapT* map) {} + +#endif // CPU_ARM_STACKCHUNKFRAMESTREAM_ARM_INLINE_HPP diff --git a/src/hotspot/cpu/arm/stackChunkOop_arm.inline.hpp b/src/hotspot/cpu/arm/stackChunkOop_arm.inline.hpp new file mode 100644 index 00000000000..2fa75870fe1 --- /dev/null +++ b/src/hotspot/cpu/arm/stackChunkOop_arm.inline.hpp @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_ARM_STACKCHUNKOOP_ARM_INLINE_HPP +#define CPU_ARM_STACKCHUNKOOP_ARM_INLINE_HPP + +inline void stackChunkOopDesc::relativize_frame_pd(frame& fr) const { + Unimplemented(); +} + +inline void stackChunkOopDesc::derelativize_frame_pd(frame& fr) const { + Unimplemented(); +} + +#endif // CPU_ARM_STACKCHUNKOOP_ARM_INLINE_HPP diff --git a/src/hotspot/cpu/arm/stubGenerator_arm.cpp b/src/hotspot/cpu/arm/stubGenerator_arm.cpp index a0b4bf620b6..6137141aed5 100644 --- a/src/hotspot/cpu/arm/stubGenerator_arm.cpp +++ b/src/hotspot/cpu/arm/stubGenerator_arm.cpp @@ -3039,9 +3039,9 @@ class StubGenerator: public StubCodeGenerator { }; // end class declaration #define UCM_TABLE_MAX_ENTRIES 32 -void StubGenerator_generate(CodeBuffer* code, bool all) { +void StubGenerator_generate(CodeBuffer* code, int phase) { if (UnsafeCopyMemory::_table == NULL) { UnsafeCopyMemory::create_table(UCM_TABLE_MAX_ENTRIES); } - StubGenerator g(code, all); + StubGenerator g(code, phase); } diff --git a/src/hotspot/cpu/arm/templateInterpreterGenerator_arm.cpp b/src/hotspot/cpu/arm/templateInterpreterGenerator_arm.cpp index f95ea39d3b2..093187776d5 100644 --- a/src/hotspot/cpu/arm/templateInterpreterGenerator_arm.cpp +++ b/src/hotspot/cpu/arm/templateInterpreterGenerator_arm.cpp @@ -728,6 +728,11 @@ void TemplateInterpreterGenerator::generate_fixed_frame(bool native_call) { // [ parameter 1 ] <--- Rlocals // +address TemplateInterpreterGenerator::generate_Continuation_doYield_entry(void) { + Unimplemented(); + return NULL; +} + address TemplateInterpreterGenerator::generate_Reference_get_entry(void) { // Code: _aload_0, _getfield, _areturn // parameter size = 1 diff --git a/src/hotspot/cpu/ppc/assembler_ppc.hpp b/src/hotspot/cpu/ppc/assembler_ppc.hpp index 073a7d9adf9..21682513abe 100644 --- a/src/hotspot/cpu/ppc/assembler_ppc.hpp +++ b/src/hotspot/cpu/ppc/assembler_ppc.hpp @@ -140,6 +140,9 @@ class Argument { // shows that xlC places all float args after argument 8 on the stack AND // in a register. This is not documented, but we follow this convention, too. n_regs_not_on_stack_c = 8, + + n_int_register_parameters_j = 8, + n_float_register_parameters_j = 13 }; // creation Argument(int number) : _number(number) {} diff --git a/src/hotspot/cpu/ppc/c1_LIRGenerator_ppc.cpp b/src/hotspot/cpu/ppc/c1_LIRGenerator_ppc.cpp index 39383f4c20d..d42e3552fd4 100644 --- a/src/hotspot/cpu/ppc/c1_LIRGenerator_ppc.cpp +++ b/src/hotspot/cpu/ppc/c1_LIRGenerator_ppc.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2005, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2005, 2022, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2012, 2019 SAP SE. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * @@ -1381,3 +1381,7 @@ void LIRGenerator::do_FmaIntrinsic(Intrinsic* x) { void LIRGenerator::do_vectorizedMismatch(Intrinsic* x) { fatal("vectorizedMismatch intrinsic is not implemented on this platform"); } + +void LIRGenerator::do_continuation_doYield(Intrinsic* x) { + fatal("Continuation.doYield intrinsic is not implemented on this platform"); +} diff --git a/src/hotspot/cpu/ppc/continuationEntry_ppc.inline.hpp b/src/hotspot/cpu/ppc/continuationEntry_ppc.inline.hpp new file mode 100644 index 00000000000..c38ccd842ac --- /dev/null +++ b/src/hotspot/cpu/ppc/continuationEntry_ppc.inline.hpp @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_PPC_CONTINUATIONENTRY_PPC_INLINE_HPP +#define CPU_PPC_CONTINUATIONENTRY_PPC_INLINE_HPP + +#include "runtime/continuationEntry.hpp" + +// TODO: Implement + +inline frame ContinuationEntry::to_frame() const { + Unimplemented(); + return frame(); +} + +inline intptr_t* ContinuationEntry::entry_fp() const { + Unimplemented(); + return nullptr; +} + +inline void ContinuationEntry::update_register_map(RegisterMap* map) const { + Unimplemented(); +} + +#endif // CPU_PPC_CONTINUATIONENTRY_PPC_INLINE_HPP diff --git a/src/hotspot/cpu/ppc/continuationFreezeThaw_ppc.inline.hpp b/src/hotspot/cpu/ppc/continuationFreezeThaw_ppc.inline.hpp new file mode 100644 index 00000000000..8753c4c8e5e --- /dev/null +++ b/src/hotspot/cpu/ppc/continuationFreezeThaw_ppc.inline.hpp @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_PPC_CONTINUATION_PPC_INLINE_HPP +#define CPU_PPC_CONTINUATION_PPC_INLINE_HPP + +#include "oops/stackChunkOop.inline.hpp" +#include "runtime/frame.hpp" +#include "runtime/frame.inline.hpp" + + +inline void FreezeBase::set_top_frame_metadata_pd(const frame& hf) { + Unimplemented(); +} + +template +inline frame FreezeBase::sender(const frame& f) { + Unimplemented(); + return frame(); +} + +template frame FreezeBase::new_heap_frame(frame& f, frame& caller) { + Unimplemented(); + return frame(); +} + +void FreezeBase::adjust_interpreted_frame_unextended_sp(frame& f) { + Unimplemented(); +} + +inline void FreezeBase::relativize_interpreted_frame_metadata(const frame& f, const frame& hf) { + Unimplemented(); +} + +inline void FreezeBase::patch_pd(frame& hf, const frame& caller) { + Unimplemented(); +} + +inline void FreezeBase::patch_stack_pd(intptr_t* frame_sp, intptr_t* heap_sp) { + Unimplemented(); +} + +inline frame ThawBase::new_entry_frame() { + Unimplemented(); + return frame(); +} + +template frame ThawBase::new_stack_frame(const frame& hf, frame& caller, bool bottom) { + Unimplemented(); + return frame(); +} + +inline void ThawBase::set_interpreter_frame_bottom(const frame& f, intptr_t* bottom) { + Unimplemented(); +} + +inline void ThawBase::derelativize_interpreted_frame_metadata(const frame& hf, const frame& f) { + Unimplemented(); +} + +inline intptr_t* ThawBase::align(const frame& hf, intptr_t* frame_sp, frame& caller, bool bottom) { + Unimplemented(); + return NULL; +} + +inline void ThawBase::patch_pd(frame& f, const frame& caller) { + Unimplemented(); +} + +void ThawBase::patch_chunk_pd(intptr_t* sp) { + Unimplemented(); +} + +inline void ThawBase::prefetch_chunk_pd(void* start, int size) { + Unimplemented(); +} + +#endif // CPU_PPC_CONTINUATION_PPC_INLINE_HPP diff --git a/src/hotspot/cpu/ppc/continuationHelper_ppc.inline.hpp b/src/hotspot/cpu/ppc/continuationHelper_ppc.inline.hpp new file mode 100644 index 00000000000..88bbc44de84 --- /dev/null +++ b/src/hotspot/cpu/ppc/continuationHelper_ppc.inline.hpp @@ -0,0 +1,124 @@ +/* + * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_PPC_CONTINUATIONHELPER_PPC_INLINE_HPP +#define CPU_PPC_CONTINUATIONHELPER_PPC_INLINE_HPP + +#include "runtime/continuationHelper.hpp" + +template +static inline intptr_t** link_address(const frame& f) { + Unimplemented(); + return NULL; +} + +inline int ContinuationHelper::frame_align_words(int size) { + Unimplemented(); + return 0; +} + +inline intptr_t* ContinuationHelper::frame_align_pointer(intptr_t* sp) { + Unimplemented(); + return NULL; +} + +template +inline void ContinuationHelper::update_register_map(const frame& f, RegisterMap* map) { + Unimplemented(); +} + +inline void ContinuationHelper::update_register_map_with_callee(const frame& f, RegisterMap* map) { + Unimplemented(); +} + +inline void ContinuationHelper::push_pd(const frame& f) { + Unimplemented(); +} + + +inline void ContinuationHelper::set_anchor_to_entry_pd(JavaFrameAnchor* anchor, ContinuationEntry* cont) { + Unimplemented(); +} + +#ifdef ASSERT +inline void ContinuationHelper::set_anchor_pd(JavaFrameAnchor* anchor, intptr_t* sp) { + Unimplemented(); +} + +inline bool ContinuationHelper::Frame::assert_frame_laid_out(frame f) { + Unimplemented(); + return false; +} +#endif + +inline intptr_t** ContinuationHelper::Frame::callee_link_address(const frame& f) { + Unimplemented(); + return NULL; +} + +template +static inline intptr_t* real_fp(const frame& f) { + Unimplemented(); + return NULL; +} + +inline address* ContinuationHelper::InterpretedFrame::return_pc_address(const frame& f) { + Unimplemented(); + return NULL; +} + +inline void ContinuationHelper::InterpretedFrame::patch_sender_sp(frame& f, intptr_t* sp) { + Unimplemented(); +} + +inline address* ContinuationHelper::Frame::return_pc_address(const frame& f) { + Unimplemented(); + return NULL; +} + +inline address ContinuationHelper::Frame::real_pc(const frame& f) { + Unimplemented(); + return NULL; +} + +inline void ContinuationHelper::Frame::patch_pc(const frame& f, address pc) { + Unimplemented(); +} + +inline intptr_t* ContinuationHelper::InterpretedFrame::frame_top(const frame& f, InterpreterOopMap* mask) { // inclusive; this will be copied with the frame + Unimplemented(); + return NULL; +} + +inline intptr_t* ContinuationHelper::InterpretedFrame::frame_bottom(const frame& f) { // exclusive; this will not be copied with the frame + Unimplemented(); + return NULL; +} + +inline intptr_t* ContinuationHelper::InterpretedFrame::frame_top(const frame& f, int callee_argsize, bool callee_interpreted) { + Unimplemented(); + return NULL; +} + +#endif // CPU_PPC_CONTINUATIONFRAMEHELPERS_PPC_INLINE_HPP diff --git a/src/hotspot/cpu/ppc/frame_ppc.cpp b/src/hotspot/cpu/ppc/frame_ppc.cpp index b8e6433913b..744508414f3 100644 --- a/src/hotspot/cpu/ppc/frame_ppc.cpp +++ b/src/hotspot/cpu/ppc/frame_ppc.cpp @@ -219,27 +219,6 @@ frame frame::sender_for_interpreter_frame(RegisterMap *map) const { return frame(sender_sp(), sender_pc(), (intptr_t*)get_ijava_state()->sender_sp); } -frame frame::sender_for_compiled_frame(RegisterMap *map) const { - assert(map != NULL, "map must be set"); - - // Frame owned by compiler. - address pc = *compiled_sender_pc_addr(_cb); - frame caller(compiled_sender_sp(_cb), pc); - - // Now adjust the map. - - // Get the rest. - if (map->update_map()) { - // Tell GC to use argument oopmaps for some runtime stubs that need it. - map->set_include_argument_oops(_cb->caller_must_gc_arguments(map->thread())); - if (_cb->oop_maps() != NULL) { - OopMapSet::update_register_map(this, map); - } - } - - return caller; -} - intptr_t* frame::compiled_sender_sp(CodeBlob* cb) const { return sender_sp(); } @@ -248,33 +227,6 @@ address* frame::compiled_sender_pc_addr(CodeBlob* cb) const { return sender_pc_addr(); } -frame frame::sender_raw(RegisterMap* map) const { - // Default is we do have to follow them. The sender_for_xxx will - // update it accordingly. - map->set_include_argument_oops(false); - - if (is_entry_frame()) return sender_for_entry_frame(map); - if (is_interpreted_frame()) return sender_for_interpreter_frame(map); - assert(_cb == CodeCache::find_blob(pc()),"Must be the same"); - - if (_cb != NULL) { - return sender_for_compiled_frame(map); - } - // Must be native-compiled frame, i.e. the marshaling code for native - // methods that exists in the core system. - return frame(sender_sp(), sender_pc()); -} - -frame frame::sender(RegisterMap* map) const { - frame result = sender_raw(map); - - if (map->process_frames()) { - StackWatermarkSet::on_iteration(map->thread(), result); - } - - return result; -} - void frame::patch_pc(Thread* thread, address pc) { assert(_cb == CodeCache::find_blob(pc), "unexpected pc"); if (TracePcPatching) { @@ -431,8 +383,19 @@ intptr_t *frame::initial_deoptimization_info() { #ifndef PRODUCT // This is a generic constructor which is only used by pns() in debug.cpp. -frame::frame(void* sp, void* fp, void* pc) : _sp((intptr_t*)sp), _unextended_sp((intptr_t*)sp) { +frame::frame(void* sp, void* fp, void* pc) : _sp((intptr_t*)sp), + _on_heap(false), + _unextended_sp((intptr_t*)sp) { find_codeblob_and_set_pc_and_deopt_state((address)pc); // also sets _fp and adjusts _unextended_sp } #endif + +// Pointer beyond the "oldest/deepest" BasicObjectLock on stack. +BasicObjectLock* frame::interpreter_frame_monitor_end() const { + return (BasicObjectLock*) get_ijava_state()->monitors; +} + +intptr_t* frame::interpreter_frame_tos_at(jint offset) const { + return &interpreter_frame_tos_address()[offset]; +} diff --git a/src/hotspot/cpu/ppc/frame_ppc.hpp b/src/hotspot/cpu/ppc/frame_ppc.hpp index ac717bd4734..cc0ebe10115 100644 --- a/src/hotspot/cpu/ppc/frame_ppc.hpp +++ b/src/hotspot/cpu/ppc/frame_ppc.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2000, 2022, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2012, 2021 SAP SE. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * @@ -379,8 +379,9 @@ public: + const ImmutableOopMap* get_oop_map() const; + // Constructors - inline frame(intptr_t* sp); inline frame(intptr_t* sp, address pc); inline frame(intptr_t* sp, address pc, intptr_t* unextended_sp); @@ -400,6 +401,11 @@ inline void interpreter_frame_set_top_frame_sp(intptr_t* top_frame_sp); inline void interpreter_frame_set_sender_sp(intptr_t* sender_sp); + inline intptr_t* interpreter_frame_last_sp() const; + + template + static void update_map_with_saved_link(RegisterMapT* map, intptr_t** link_addr); + // Size of a monitor in bytes. static int interpreter_frame_monitor_size_in_bytes(); @@ -413,12 +419,16 @@ enum { // normal return address is 1 bundle past PC - pc_return_offset = 0 + pc_return_offset = 0, + metadata_words = 0, + frame_alignment = 16, + // size, in words, of maximum shift in frame position due to alignment + align_wiggle = 1 }; static jint interpreter_frame_expression_stack_direction() { return -1; } // returns the sending frame, without applying any barriers - frame sender_raw(RegisterMap* map) const; + inline frame sender_raw(RegisterMap* map) const; #endif // CPU_PPC_FRAME_PPC_HPP diff --git a/src/hotspot/cpu/ppc/frame_ppc.inline.hpp b/src/hotspot/cpu/ppc/frame_ppc.inline.hpp index 239db8224c0..f44abaaa2eb 100644 --- a/src/hotspot/cpu/ppc/frame_ppc.inline.hpp +++ b/src/hotspot/cpu/ppc/frame_ppc.inline.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2000, 2022, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2012, 2015 SAP SE. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * @@ -55,17 +55,33 @@ inline void frame::find_codeblob_and_set_pc_and_deopt_state(address pc) { // Constructors // Initialize all fields, _unextended_sp will be adjusted in find_codeblob_and_set_pc_and_deopt_state. -inline frame::frame() : _sp(NULL), _pc(NULL), _cb(NULL), _deopt_state(unknown), _unextended_sp(NULL), _fp(NULL) {} +inline frame::frame() : _sp(NULL), _pc(NULL), _cb(NULL), _deopt_state(unknown), _on_heap(false), +#ifdef ASSERT + _frame_index(-1), +#endif + _unextended_sp(NULL), _fp(NULL) {} -inline frame::frame(intptr_t* sp) : _sp(sp), _unextended_sp(sp) { +inline frame::frame(intptr_t* sp) : _sp(sp), _on_heap(false), +#ifdef ASSERT + _frame_index(-1), +#endif + _unextended_sp(sp) { find_codeblob_and_set_pc_and_deopt_state((address)own_abi()->lr); // also sets _fp and adjusts _unextended_sp } -inline frame::frame(intptr_t* sp, address pc) : _sp(sp), _unextended_sp(sp) { +inline frame::frame(intptr_t* sp, address pc) : _sp(sp), _on_heap(false), +#ifdef ASSERT + _frame_index(-1), +#endif + _unextended_sp(sp) { find_codeblob_and_set_pc_and_deopt_state(pc); // also sets _fp and adjusts _unextended_sp } -inline frame::frame(intptr_t* sp, address pc, intptr_t* unextended_sp) : _sp(sp), _unextended_sp(unextended_sp) { +inline frame::frame(intptr_t* sp, address pc, intptr_t* unextended_sp) : _sp(sp), _on_heap(false), +#ifdef ASSERT + _frame_index(-1), +#endif + _unextended_sp(unextended_sp) { find_codeblob_and_set_pc_and_deopt_state(pc); // also sets _fp and adjusts _unextended_sp } @@ -87,7 +103,7 @@ inline bool frame::is_older(intptr_t* id) const { return this->id() > id; } -inline int frame::frame_size(RegisterMap* map) const { +inline int frame::frame_size() const { // Stack grows towards smaller addresses on PPC64: sender is at a higher address. return sender_sp() - sp(); } @@ -143,11 +159,6 @@ inline intptr_t* frame::interpreter_frame_mdp_addr() const { return (intptr_t*) &(get_ijava_state()->mdx); } -// Pointer beyond the "oldest/deepest" BasicObjectLock on stack. -inline BasicObjectLock* frame::interpreter_frame_monitor_end() const { - return (BasicObjectLock*) get_ijava_state()->monitors; -} - inline BasicObjectLock* frame::interpreter_frame_monitor_begin() const { return (BasicObjectLock*) get_ijava_state(); } @@ -189,10 +200,6 @@ inline intptr_t* frame::interpreter_frame_tos_address() const { return ((intptr_t*) get_ijava_state()->esp) + Interpreter::stackElementWords; } -inline intptr_t* frame::interpreter_frame_tos_at(jint offset) const { - return &interpreter_frame_tos_address()[offset]; -} - inline int frame::interpreter_frame_monitor_size() { // Number of stack slots for a monitor. return align_up(BasicObjectLock::size(), // number of stack slots @@ -218,11 +225,103 @@ inline JavaCallWrapper** frame::entry_frame_call_wrapper_addr() const { } inline oop frame::saved_oop_result(RegisterMap* map) const { - return *((oop*)map->location(R3->as_VMReg())); + return *((oop*)map->location(R3->as_VMReg(), nullptr)); } inline void frame::set_saved_oop_result(RegisterMap* map, oop obj) { - *((oop*)map->location(R3->as_VMReg())) = obj; + *((oop*)map->location(R3->as_VMReg(), nullptr)) = obj; +} + +inline const ImmutableOopMap* frame::get_oop_map() const { + Unimplemented(); + return NULL; +} + +inline int frame::compiled_frame_stack_argsize() const { + Unimplemented(); + return 0; +} + +inline void frame::interpreted_frame_oop_map(InterpreterOopMap* mask) const { + Unimplemented(); +} + +inline intptr_t* frame::interpreter_frame_last_sp() const { + Unimplemented(); + return NULL; +} + +inline int frame::sender_sp_ret_address_offset() { + Unimplemented(); + return 0; +} + +inline void frame::set_unextended_sp(intptr_t* value) { + Unimplemented(); +} + +inline int frame::offset_unextended_sp() const { + Unimplemented(); + return 0; +} + +inline void frame::set_offset_unextended_sp(int value) { + Unimplemented(); +} + +//------------------------------------------------------------------------------ +// frame::sender + +frame frame::sender(RegisterMap* map) const { + frame result = sender_raw(map); + + if (map->process_frames()) { + StackWatermarkSet::on_iteration(map->thread(), result); + } + + return result; +} + +inline frame frame::sender_raw(RegisterMap* map) const { + // Default is we do have to follow them. The sender_for_xxx will + // update it accordingly. + map->set_include_argument_oops(false); + + if (is_entry_frame()) return sender_for_entry_frame(map); + if (is_interpreted_frame()) return sender_for_interpreter_frame(map); + assert(_cb == CodeCache::find_blob(pc()),"Must be the same"); + + if (_cb != NULL) return sender_for_compiled_frame(map); + + // Must be native-compiled frame, i.e. the marshaling code for native + // methods that exists in the core system. + return frame(sender_sp(), sender_pc()); +} + +inline frame frame::sender_for_compiled_frame(RegisterMap *map) const { + assert(map != NULL, "map must be set"); + + // Frame owned by compiler. + address pc = *compiled_sender_pc_addr(_cb); + frame caller(compiled_sender_sp(_cb), pc); + + // Now adjust the map. + + // Get the rest. + if (map->update_map()) { + // Tell GC to use argument oopmaps for some runtime stubs that need it. + map->set_include_argument_oops(_cb->caller_must_gc_arguments(map->thread())); + if (_cb->oop_maps() != NULL) { + OopMapSet::update_register_map(this, map); + } + } + + return caller; +} + +template +void frame::update_map_with_saved_link(RegisterMapT* map, intptr_t** link_addr) { + Unimplemented(); } #endif // CPU_PPC_FRAME_PPC_INLINE_HPP diff --git a/src/hotspot/cpu/ppc/gc/shared/barrierSetNMethod_ppc.cpp b/src/hotspot/cpu/ppc/gc/shared/barrierSetNMethod_ppc.cpp index 6efeb94b381..ae655bb2b2b 100644 --- a/src/hotspot/cpu/ppc/gc/shared/barrierSetNMethod_ppc.cpp +++ b/src/hotspot/cpu/ppc/gc/shared/barrierSetNMethod_ppc.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -121,6 +121,10 @@ void BarrierSetNMethod::disarm(nmethod* nm) { barrier->release_set_guard_value(disarmed_value()); } +void BarrierSetNMethod::arm(nmethod* nm, int arm_value) { + Unimplemented(); +} + bool BarrierSetNMethod::is_armed(nmethod* nm) { if (!supports_entry_barrier(nm)) { return false; diff --git a/src/hotspot/cpu/ppc/nativeInst_ppc.hpp b/src/hotspot/cpu/ppc/nativeInst_ppc.hpp index e9cc46e8681..6976e55f774 100644 --- a/src/hotspot/cpu/ppc/nativeInst_ppc.hpp +++ b/src/hotspot/cpu/ppc/nativeInst_ppc.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2002, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2002, 2022, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2012, 2021 SAP SE. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * @@ -503,4 +503,35 @@ class NativeMovRegMem: public NativeInstruction { } }; +class NativePostCallNop: public NativeInstruction { +public: + bool check() const { Unimplemented(); return false; } + int displacement() const { Unimplemented(); return 0; } + void patch(jint diff) { Unimplemented(); } + void make_deopt() { Unimplemented(); } +}; + +inline NativePostCallNop* nativePostCallNop_at(address address) { + Unimplemented(); + return NULL; +} + +class NativeDeoptInstruction: public NativeInstruction { +public: + address instruction_address() const { Unimplemented(); return NULL; } + address next_instruction_address() const { Unimplemented(); return NULL; } + + void verify() { Unimplemented(); } + + static bool is_deopt_at(address instr) { + Unimplemented(); + return false; + } + + // MT-safe patching + static void insert(address code_pos) { + Unimplemented(); + } +}; + #endif // CPU_PPC_NATIVEINST_PPC_HPP diff --git a/src/hotspot/cpu/ppc/registerMap_ppc.hpp b/src/hotspot/cpu/ppc/registerMap_ppc.hpp index 40b026e926f..cbbed577d5f 100644 --- a/src/hotspot/cpu/ppc/registerMap_ppc.hpp +++ b/src/hotspot/cpu/ppc/registerMap_ppc.hpp @@ -36,7 +36,7 @@ address pd_location(VMReg reg) const { return NULL; } address pd_location(VMReg base_reg, int slot_idx) const { - return location(base_reg->next(slot_idx)); + return location(base_reg->next(slot_idx), nullptr); } // no PD state to clear or copy: diff --git a/src/hotspot/cpu/ppc/smallRegisterMap_ppc.inline.hpp b/src/hotspot/cpu/ppc/smallRegisterMap_ppc.inline.hpp new file mode 100644 index 00000000000..64c44cd655c --- /dev/null +++ b/src/hotspot/cpu/ppc/smallRegisterMap_ppc.inline.hpp @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_PPC_SMALLREGISTERMAP_PPC_INLINE_HPP +#define CPU_PPC_SMALLREGISTERMAP_PPC_INLINE_HPP + +#include "runtime/frame.inline.hpp" +#include "runtime/registerMap.hpp" + +// Java frames don't have callee saved registers (except for rfp), so we can use a smaller RegisterMap +class SmallRegisterMap { +public: + static constexpr SmallRegisterMap* instance = nullptr; +private: + static void assert_is_rfp(VMReg r) PRODUCT_RETURN + DEBUG_ONLY({ Unimplemented(); }) +public: + // as_RegisterMap is used when we didn't want to templatize and abstract over RegisterMap type to support SmallRegisterMap + // Consider enhancing SmallRegisterMap to support those cases + const RegisterMap* as_RegisterMap() const { return nullptr; } + RegisterMap* as_RegisterMap() { return nullptr; } + + RegisterMap* copy_to_RegisterMap(RegisterMap* map, intptr_t* sp) const { + Unimplemented(); + return map; + } + + SmallRegisterMap() {} + + SmallRegisterMap(const RegisterMap* map) { + Unimplemented(); + } + + inline address location(VMReg reg, intptr_t* sp) const { + Unimplemented(); + return NULL; + } + + inline void set_location(VMReg reg, address loc) { assert_is_rfp(reg); } + + JavaThread* thread() const { + #ifndef ASSERT + guarantee (false, ""); + #endif + return nullptr; + } + + bool update_map() const { return false; } + bool walk_cont() const { return false; } + bool include_argument_oops() const { return false; } + void set_include_argument_oops(bool f) {} + bool in_cont() const { return false; } + stackChunkHandle stack_chunk() const { return stackChunkHandle(); } + +#ifdef ASSERT + bool should_skip_missing() const { return false; } + VMReg find_register_spilled_here(void* p, intptr_t* sp) { + Unimplemented(); + return NULL; + } + void print() const { print_on(tty); } + void print_on(outputStream* st) const { st->print_cr("Small register map"); } +#endif +}; + +#endif // CPU_PPC_SMALLREGISTERMAP_PPC_INLINE_HPP diff --git a/src/hotspot/cpu/ppc/stackChunkFrameStream_ppc.inline.hpp b/src/hotspot/cpu/ppc/stackChunkFrameStream_ppc.inline.hpp new file mode 100644 index 00000000000..f3b28dac0da --- /dev/null +++ b/src/hotspot/cpu/ppc/stackChunkFrameStream_ppc.inline.hpp @@ -0,0 +1,115 @@ +/* + * Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_PPC_STACKCHUNKFRAMESTREAM_PPC_INLINE_HPP +#define CPU_PPC_STACKCHUNKFRAMESTREAM_PPC_INLINE_HPP + +#include "interpreter/oopMapCache.hpp" +#include "runtime/frame.inline.hpp" +#include "runtime/registerMap.hpp" + +#ifdef ASSERT +template +inline bool StackChunkFrameStream::is_in_frame(void* p0) const { + Unimplemented(); + return true; +} +#endif + +template +inline frame StackChunkFrameStream::to_frame() const { + Unimplemented(); + return frame(); +} + +template +inline address StackChunkFrameStream::get_pc() const { + Unimplemented(); + return NULL; +} + +template +inline intptr_t* StackChunkFrameStream::fp() const { + Unimplemented(); + return NULL; +} + +template +inline intptr_t* StackChunkFrameStream::derelativize(int offset) const { + Unimplemented(); + return NULL; +} + +template +inline intptr_t* StackChunkFrameStream::unextended_sp_for_interpreter_frame() const { + Unimplemented(); + return NULL; +} + +template +intptr_t* StackChunkFrameStream::next_sp_for_interpreter_frame() const { + Unimplemented(); + return NULL; +} + +template +inline void StackChunkFrameStream::next_for_interpreter_frame() { + Unimplemented(); +} + +template +inline int StackChunkFrameStream::interpreter_frame_size() const { + Unimplemented(); + return 0; +} + +template +inline int StackChunkFrameStream::interpreter_frame_stack_argsize() const { + Unimplemented(); + return 0; +} + +template +inline int StackChunkFrameStream::interpreter_frame_num_oops() const { + Unimplemented(); + return 0; +} + +template<> +template<> +inline void StackChunkFrameStream::update_reg_map_pd(RegisterMap* map) { + Unimplemented(); +} + +template<> +template<> +inline void StackChunkFrameStream::update_reg_map_pd(RegisterMap* map) { + Unimplemented(); +} + +template +template +inline void StackChunkFrameStream::update_reg_map_pd(RegisterMapT* map) {} + +#endif // CPU_PPC_STACKCHUNKFRAMESTREAM_PPC_INLINE_HPP diff --git a/src/hotspot/cpu/ppc/stackChunkOop_ppc.inline.hpp b/src/hotspot/cpu/ppc/stackChunkOop_ppc.inline.hpp new file mode 100644 index 00000000000..ad8af047518 --- /dev/null +++ b/src/hotspot/cpu/ppc/stackChunkOop_ppc.inline.hpp @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_PPC_STACKCHUNKOOP_PPC_INLINE_HPP +#define CPU_PPC_STACKCHUNKOOP_PPC_INLINE_HPP + +inline void stackChunkOopDesc::relativize_frame_pd(frame& fr) const { + Unimplemented(); +} + +inline void stackChunkOopDesc::derelativize_frame_pd(frame& fr) const { + Unimplemented(); +} + +#endif // CPU_PPC_STACKCHUNKOOP_PPC_INLINE_HPP diff --git a/src/hotspot/cpu/ppc/stubGenerator_ppc.cpp b/src/hotspot/cpu/ppc/stubGenerator_ppc.cpp index e942ec11408..ffb2216bbca 100644 --- a/src/hotspot/cpu/ppc/stubGenerator_ppc.cpp +++ b/src/hotspot/cpu/ppc/stubGenerator_ppc.cpp @@ -4618,9 +4618,9 @@ class StubGenerator: public StubCodeGenerator { }; #define UCM_TABLE_MAX_ENTRIES 8 -void StubGenerator_generate(CodeBuffer* code, bool all) { +void StubGenerator_generate(CodeBuffer* code, int phase) { if (UnsafeCopyMemory::_table == NULL) { UnsafeCopyMemory::create_table(UCM_TABLE_MAX_ENTRIES); } - StubGenerator g(code, all); + StubGenerator g(code, phase); } diff --git a/src/hotspot/cpu/ppc/templateInterpreterGenerator_ppc.cpp b/src/hotspot/cpu/ppc/templateInterpreterGenerator_ppc.cpp index ca91e78dd30..7fef01546ca 100644 --- a/src/hotspot/cpu/ppc/templateInterpreterGenerator_ppc.cpp +++ b/src/hotspot/cpu/ppc/templateInterpreterGenerator_ppc.cpp @@ -476,6 +476,11 @@ address TemplateInterpreterGenerator::generate_abstract_entry(void) { return entry; } +address TemplateInterpreterGenerator::generate_Continuation_doYield_entry(void) { + Unimplemented(); + return NULL; +} + // Interpreter intrinsic for WeakReference.get(). // 1. Don't push a full blown frame and go on dispatching, but fetch the value // into R8 and return quickly diff --git a/src/hotspot/cpu/riscv/c1_LIRGenerator_riscv.cpp b/src/hotspot/cpu/riscv/c1_LIRGenerator_riscv.cpp index dcab5b49e62..82d1b30cea0 100644 --- a/src/hotspot/cpu/riscv/c1_LIRGenerator_riscv.cpp +++ b/src/hotspot/cpu/riscv/c1_LIRGenerator_riscv.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2005, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2005, 2022, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2014, Red Hat Inc. All rights reserved. * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. @@ -301,6 +301,10 @@ void LIRGenerator::do_MonitorExit(MonitorExit* x) { monitor_exit(obj_temp, lock, syncTempOpr(), LIR_OprFact::illegalOpr, x->monitor_no()); } +void LIRGenerator::do_continuation_doYield(Intrinsic* x) { + fatal("Continuation.doYield intrinsic is not implemented on this platform"); +} + // neg void LIRGenerator::do_NegateOp(NegateOp* x) { LIRItem from(x->x(), this); diff --git a/src/hotspot/cpu/riscv/continuationEntry_riscv.inline.hpp b/src/hotspot/cpu/riscv/continuationEntry_riscv.inline.hpp new file mode 100644 index 00000000000..85cd03d00d1 --- /dev/null +++ b/src/hotspot/cpu/riscv/continuationEntry_riscv.inline.hpp @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_RISCV_CONTINUATIONENTRY_RISCV_INLINE_HPP +#define CPU_RISCV_CONTINUATIONENTRY_RISCV_INLINE_HPP + +#include "runtime/continuationEntry.hpp" + +// TODO: Implement + +inline frame ContinuationEntry::to_frame() const { + Unimplemented(); + return frame(); +} + +inline intptr_t* ContinuationEntry::entry_fp() const { + Unimplemented(); + return nullptr; +} + +inline void ContinuationEntry::update_register_map(RegisterMap* map) const { + Unimplemented(); +} + +#endif // CPU_RISCV_CONTINUATIONENTRY_RISCV_INLINE_HPP diff --git a/src/hotspot/cpu/riscv/continuationFreezeThaw_riscv.inline.hpp b/src/hotspot/cpu/riscv/continuationFreezeThaw_riscv.inline.hpp new file mode 100644 index 00000000000..74d94687df9 --- /dev/null +++ b/src/hotspot/cpu/riscv/continuationFreezeThaw_riscv.inline.hpp @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_RISCV_CONTINUATIONFREEZETHAW_RISCV_INLINE_HPP +#define CPU_RISCV_CONTINUATIONFREEZETHAW_RISCV_INLINE_HPP + +#include "oops/stackChunkOop.inline.hpp" +#include "runtime/frame.hpp" +#include "runtime/frame.inline.hpp" + + +inline void FreezeBase::set_top_frame_metadata_pd(const frame& hf) { + Unimplemented(); +} + +template +inline frame FreezeBase::sender(const frame& f) { + Unimplemented(); + return frame(); +} + +template frame FreezeBase::new_heap_frame(frame& f, frame& caller) { + Unimplemented(); + return frame(); +} + +void FreezeBase::adjust_interpreted_frame_unextended_sp(frame& f) { + Unimplemented(); +} + +inline void FreezeBase::relativize_interpreted_frame_metadata(const frame& f, const frame& hf) { + Unimplemented(); +} + +inline void FreezeBase::patch_pd(frame& hf, const frame& caller) { + Unimplemented(); +} + +inline void FreezeBase::patch_stack_pd(intptr_t* frame_sp, intptr_t* heap_sp) { + Unimplemented(); +} + +inline frame ThawBase::new_entry_frame() { + Unimplemented(); + return frame(); +} + +template frame ThawBase::new_stack_frame(const frame& hf, frame& caller, bool bottom) { + Unimplemented(); + return frame(); +} + +inline void ThawBase::set_interpreter_frame_bottom(const frame& f, intptr_t* bottom) { + Unimplemented(); +} + +inline void ThawBase::derelativize_interpreted_frame_metadata(const frame& hf, const frame& f) { + Unimplemented(); +} + +inline intptr_t* ThawBase::align(const frame& hf, intptr_t* frame_sp, frame& caller, bool bottom) { + Unimplemented(); + return NULL; +} + +inline void ThawBase::patch_pd(frame& f, const frame& caller) { + Unimplemented(); +} + +void ThawBase::patch_chunk_pd(intptr_t* sp) { + Unimplemented(); +} + +inline void ThawBase::prefetch_chunk_pd(void* start, int size) { + Unimplemented(); +} + +#endif // CPU_RISCV_CONTINUATIONFREEZETHAW_RISCV_INLINE_HPP diff --git a/src/hotspot/cpu/riscv/continuationHelper_riscv.inline.hpp b/src/hotspot/cpu/riscv/continuationHelper_riscv.inline.hpp new file mode 100644 index 00000000000..3adfbc8119f --- /dev/null +++ b/src/hotspot/cpu/riscv/continuationHelper_riscv.inline.hpp @@ -0,0 +1,124 @@ +/* + * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_RISCV_CONTINUATIONHELPER_RISCV_INLINE_HPP +#define CPU_RISCV_CONTINUATIONHELPER_RISCV_INLINE_HPP + +#include "runtime/continuationHelper.hpp" + +template +static inline intptr_t** link_address(const frame& f) { + Unimplemented(); + return NULL; +} + +inline int ContinuationHelper::frame_align_words(int size) { + Unimplemented(); + return 0; +} + +inline intptr_t* ContinuationHelper::frame_align_pointer(intptr_t* sp) { + Unimplemented(); + return NULL; +} + +template +inline void ContinuationHelper::update_register_map(const frame& f, RegisterMap* map) { + Unimplemented(); +} + +inline void ContinuationHelper::update_register_map_with_callee(const frame& f, RegisterMap* map) { + Unimplemented(); +} + +inline void ContinuationHelper::push_pd(const frame& f) { + Unimplemented(); +} + + +inline void ContinuationHelper::set_anchor_to_entry_pd(JavaFrameAnchor* anchor, ContinuationEntry* cont) { + Unimplemented(); +} + +#ifdef ASSERT +inline void ContinuationHelper::set_anchor_pd(JavaFrameAnchor* anchor, intptr_t* sp) { + Unimplemented(); +} + +inline bool ContinuationHelper::Frame::assert_frame_laid_out(frame f) { + Unimplemented(); + return false; +} +#endif + +inline intptr_t** ContinuationHelper::Frame::callee_link_address(const frame& f) { + Unimplemented(); + return NULL; +} + +template +static inline intptr_t* real_fp(const frame& f) { + Unimplemented(); + return NULL; +} + +inline address* ContinuationHelper::InterpretedFrame::return_pc_address(const frame& f) { + Unimplemented(); + return NULL; +} + +inline void ContinuationHelper::InterpretedFrame::patch_sender_sp(frame& f, intptr_t* sp) { + Unimplemented(); +} + +inline address* ContinuationHelper::Frame::return_pc_address(const frame& f) { + Unimplemented(); + return NULL; +} + +inline address ContinuationHelper::Frame::real_pc(const frame& f) { + Unimplemented(); + return NULL; +} + +inline void ContinuationHelper::Frame::patch_pc(const frame& f, address pc) { + Unimplemented(); +} + +inline intptr_t* ContinuationHelper::InterpretedFrame::frame_top(const frame& f, InterpreterOopMap* mask) { // inclusive; this will be copied with the frame + Unimplemented(); + return NULL; +} + +inline intptr_t* ContinuationHelper::InterpretedFrame::frame_bottom(const frame& f) { // exclusive; this will not be copied with the frame + Unimplemented(); + return NULL; +} + +inline intptr_t* ContinuationHelper::InterpretedFrame::frame_top(const frame& f, int callee_argsize, bool callee_interpreted) { + Unimplemented(); + return NULL; +} + +#endif // CPU_RISCV_CONTINUATIONFRAMEHELPERS_RISCV_INLINE_HPP diff --git a/src/hotspot/cpu/riscv/frame_riscv.cpp b/src/hotspot/cpu/riscv/frame_riscv.cpp index e4c39981475..aeac8257f25 100644 --- a/src/hotspot/cpu/riscv/frame_riscv.cpp +++ b/src/hotspot/cpu/riscv/frame_riscv.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2022, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved. * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. @@ -282,11 +282,6 @@ bool frame::is_interpreted_frame() const { return Interpreter::contains(pc()); } -int frame::frame_size(RegisterMap* map) const { - frame sender = this->sender(map); - return sender.sp() - sp(); -} - intptr_t* frame::entry_frame_argument_at(int offset) const { // convert offset to index to deal with tsi int index = (Interpreter::expr_offset_in_bytes(offset)/wordSize); @@ -394,32 +389,12 @@ void frame::adjust_unextended_sp() { // If the sender PC is a deoptimization point, get the original PC. if (sender_cm->is_deopt_entry(_pc) || sender_cm->is_deopt_mh_entry(_pc)) { - DEBUG_ONLY(verify_deopt_original_pc(sender_cm, _unextended_sp)); + verify_deopt_original_pc(sender_cm, _unextended_sp); } } } } -//------------------------------------------------------------------------------ -// frame::update_map_with_saved_link -void frame::update_map_with_saved_link(RegisterMap* map, intptr_t** link_addr) { - // The interpreter and compiler(s) always save fp in a known - // location on entry. We must record where that location is - // so that if fp was live on callout from c2 we can find - // the saved copy no matter what it called. - - // Since the interpreter always saves fp if we record where it is then - // we don't have to always save fp on entry and exit to c2 compiled - // code, on entry will be enough. - assert(map != NULL, "map must be set"); - map->set_location(::fp->as_VMReg(), (address) link_addr); - // this is weird "H" ought to be at a higher address however the - // oopMaps seems to have the "H" regs at the same address and the - // vanilla register. - map->set_location(::fp->as_VMReg()->next(), (address) link_addr); -} - - //------------------------------------------------------------------------------ // frame::sender_for_interpreter_frame frame frame::sender_for_interpreter_frame(RegisterMap* map) const { @@ -440,80 +415,6 @@ frame frame::sender_for_interpreter_frame(RegisterMap* map) const { return frame(sender_sp, unextended_sp, link(), sender_pc()); } - -//------------------------------------------------------------------------------ -// frame::sender_for_compiled_frame -frame frame::sender_for_compiled_frame(RegisterMap* map) const { - // we cannot rely upon the last fp having been saved to the thread - // in C2 code but it will have been pushed onto the stack. so we - // have to find it relative to the unextended sp - - assert(_cb->frame_size() >= 0, "must have non-zero frame size"); - intptr_t* l_sender_sp = unextended_sp() + _cb->frame_size(); - intptr_t* unextended_sp = l_sender_sp; - - // the return_address is always the word on the stack - address sender_pc = (address) *(l_sender_sp + frame::return_addr_offset); - - intptr_t** saved_fp_addr = (intptr_t**) (l_sender_sp + frame::link_offset); - - assert(map != NULL, "map must be set"); - if (map->update_map()) { - // Tell GC to use argument oopmaps for some runtime stubs that need it. - // For C1, the runtime stub might not have oop maps, so set this flag - // outside of update_register_map. - map->set_include_argument_oops(_cb->caller_must_gc_arguments(map->thread())); - if (_cb->oop_maps() != NULL) { - OopMapSet::update_register_map(this, map); - } - - // Since the prolog does the save and restore of FP there is no - // oopmap for it so we must fill in its location as if there was - // an oopmap entry since if our caller was compiled code there - // could be live jvm state in it. - update_map_with_saved_link(map, saved_fp_addr); - } - - return frame(l_sender_sp, unextended_sp, *saved_fp_addr, sender_pc); -} - -//------------------------------------------------------------------------------ -// frame::sender_raw -frame frame::sender_raw(RegisterMap* map) const { - // Default is we done have to follow them. The sender_for_xxx will - // update it accordingly - assert(map != NULL, "map must be set"); - map->set_include_argument_oops(false); - - if (is_entry_frame()) { - return sender_for_entry_frame(map); - } - if (is_interpreted_frame()) { - return sender_for_interpreter_frame(map); - } - assert(_cb == CodeCache::find_blob(pc()),"Must be the same"); - - // This test looks odd: why is it not is_compiled_frame() ? That's - // because stubs also have OOP maps. - if (_cb != NULL) { - return sender_for_compiled_frame(map); - } - - // Must be native-compiled frame, i.e. the marshaling code for native - // methods that exists in the core system. - return frame(sender_sp(), link(), sender_pc()); -} - -frame frame::sender(RegisterMap* map) const { - frame result = sender_raw(map); - - if (map->process_frames()) { - StackWatermarkSet::on_iteration(map->thread(), result); - } - - return result; -} - bool frame::is_interpreted_frame_valid(JavaThread* thread) const { assert(is_interpreted_frame(), "Not an interpreted frame"); // These are reasonable sanity checks @@ -651,24 +552,11 @@ intptr_t *frame::initial_deoptimization_info() { return NULL; } -intptr_t* frame::real_fp() const { - if (_cb != NULL) { - // use the frame size if valid - int size = _cb->frame_size(); - if (size > 0) { - return unextended_sp() + size; - } - } - // else rely on fp() - assert(!is_compiled_frame(), "unknown compiled frame size"); - return fp(); -} - #undef DESCRIBE_FP_OFFSET #ifndef PRODUCT // This is a generic constructor which is only used by pns() in debug.cpp. -frame::frame(void* ptr_sp, void* ptr_fp, void* pc) { +frame::frame(void* ptr_sp, void* ptr_fp, void* pc) : _on_heap(false) { init((intptr_t*)ptr_sp, (intptr_t*)ptr_fp, (address)pc); } diff --git a/src/hotspot/cpu/riscv/frame_riscv.hpp b/src/hotspot/cpu/riscv/frame_riscv.hpp index a0c3e7c036a..7bf1540447f 100644 --- a/src/hotspot/cpu/riscv/frame_riscv.hpp +++ b/src/hotspot/cpu/riscv/frame_riscv.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2022, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * @@ -137,7 +137,14 @@ entry_frame_call_wrapper_offset = -10, // we don't need a save area - arg_reg_save_area_bytes = 0 + arg_reg_save_area_bytes = 0, + + // size, in words, of frame metadata (e.g. pc and link) + metadata_words = sender_sp_offset, + // in bytes + frame_alignment = 16, + // size, in words, of maximum shift in frame position due to alignment + align_wiggle = 1 }; intptr_t ptr_at(int offset) const { @@ -170,6 +177,8 @@ static void verify_deopt_original_pc( CompiledMethod* nm, intptr_t* unextended_sp); #endif + const ImmutableOopMap* get_oop_map() const; + public: // Constructors @@ -183,15 +192,15 @@ // accessors for the instance variables // Note: not necessarily the real 'frame pointer' (see real_fp) - intptr_t* fp() const { return _fp; } + intptr_t* fp() const { return _fp; } inline address* sender_pc_addr() const; // expression stack tos if we are nested in a java call intptr_t* interpreter_frame_last_sp() const; - // helper to update a map with callee-saved RBP - static void update_map_with_saved_link(RegisterMap* map, intptr_t** link_addr); + template + static void update_map_with_saved_link(RegisterMapT* map, intptr_t** link_addr); // deoptimization support void interpreter_frame_set_last_sp(intptr_t* last_sp); @@ -199,6 +208,6 @@ static jint interpreter_frame_expression_stack_direction() { return -1; } // returns the sending frame, without applying any barriers - frame sender_raw(RegisterMap* map) const; + inline frame sender_raw(RegisterMap* map) const; #endif // CPU_RISCV_FRAME_RISCV_HPP diff --git a/src/hotspot/cpu/riscv/frame_riscv.inline.hpp b/src/hotspot/cpu/riscv/frame_riscv.inline.hpp index 5ac1bf57f57..df161301719 100644 --- a/src/hotspot/cpu/riscv/frame_riscv.inline.hpp +++ b/src/hotspot/cpu/riscv/frame_riscv.inline.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2022, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2014, Red Hat Inc. All rights reserved. * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. @@ -41,6 +41,8 @@ inline frame::frame() { _fp = NULL; _cb = NULL; _deopt_state = unknown; + _on_heap = false; + DEBUG_ONLY(_frame_index = -1;) } static int spin; @@ -63,6 +65,9 @@ inline void frame::init(intptr_t* ptr_sp, intptr_t* ptr_fp, address pc) { } else { _deopt_state = not_deoptimized; } + + _on_heap = false; + DEBUG_ONLY(_frame_index = -1;) } inline frame::frame(intptr_t* ptr_sp, intptr_t* ptr_fp, address pc) { @@ -89,6 +94,13 @@ inline frame::frame(intptr_t* ptr_sp, intptr_t* unextended_sp, intptr_t* ptr_fp, } else { _deopt_state = not_deoptimized; } + + _on_heap = false; + DEBUG_ONLY(_frame_index = -1;) +} + +inline frame::frame(intptr_t* ptr_sp) { + Unimplemented(); } inline frame::frame(intptr_t* ptr_sp, intptr_t* ptr_fp) { @@ -119,6 +131,9 @@ inline frame::frame(intptr_t* ptr_sp, intptr_t* ptr_fp) { } else { _deopt_state = not_deoptimized; } + + _on_heap = false; + DEBUG_ONLY(_frame_index = -1;) } // Accessors @@ -150,6 +165,38 @@ inline intptr_t* frame::link_or_null() const { inline intptr_t* frame::unextended_sp() const { return _unextended_sp; } +inline void frame::set_unextended_sp(intptr_t* value) { + Unimplemented(); +} + +inline int frame::offset_unextended_sp() const { + Unimplemented(); + return 0; +} + +inline void frame::set_offset_unextended_sp(int value) { + Unimplemented(); +} + +inline intptr_t* frame::real_fp() const { + if (_cb != NULL) { + // use the frame size if valid + int size = _cb->frame_size(); + if (size > 0) { + return unextended_sp() + size; + } + } + // else rely on fp() + assert(!is_compiled_frame(), "unknown compiled frame size"); + return fp(); +} + +inline int frame::frame_size() const { + return is_interpreted_frame() + ? sender_sp() - sp() + : cb()->frame_size(); +} + // Return address inline address* frame::sender_pc_addr() const { return (address*) addr_at(return_addr_offset); } inline address frame::sender_pc() const { return *sender_pc_addr(); } @@ -160,7 +207,7 @@ inline intptr_t** frame::interpreter_frame_locals_addr() const { } inline intptr_t* frame::interpreter_frame_last_sp() const { - return *(intptr_t**)addr_at(interpreter_frame_last_sp_offset); + return (intptr_t*)at(interpreter_frame_last_sp_offset); } inline intptr_t* frame::interpreter_frame_bcp_addr() const { @@ -233,16 +280,130 @@ inline JavaCallWrapper** frame::entry_frame_call_wrapper_addr() const { PRAGMA_DIAG_PUSH PRAGMA_NONNULL_IGNORED inline oop frame::saved_oop_result(RegisterMap* map) const { - oop* result_adr = (oop *)map->location(x10->as_VMReg()); + oop* result_adr = (oop *)map->location(x10->as_VMReg(), nullptr); guarantee(result_adr != NULL, "bad register save location"); return (*result_adr); } inline void frame::set_saved_oop_result(RegisterMap* map, oop obj) { - oop* result_adr = (oop *)map->location(x10->as_VMReg()); + oop* result_adr = (oop *)map->location(x10->as_VMReg(), nullptr); guarantee(result_adr != NULL, "bad register save location"); *result_adr = obj; } PRAGMA_DIAG_POP +inline const ImmutableOopMap* frame::get_oop_map() const { + Unimplemented(); + return NULL; +} + +inline int frame::compiled_frame_stack_argsize() const { + Unimplemented(); + return 0; +} + +inline void frame::interpreted_frame_oop_map(InterpreterOopMap* mask) const { + Unimplemented(); +} + +inline int frame::sender_sp_ret_address_offset() { + Unimplemented(); + return 0; +} + +//------------------------------------------------------------------------------ +// frame::sender +frame frame::sender(RegisterMap* map) const { + frame result = sender_raw(map); + + if (map->process_frames()) { + StackWatermarkSet::on_iteration(map->thread(), result); + } + + return result; +} + +//------------------------------------------------------------------------------ +// frame::sender_raw +frame frame::sender_raw(RegisterMap* map) const { + // Default is we done have to follow them. The sender_for_xxx will + // update it accordingly + assert(map != NULL, "map must be set"); + map->set_include_argument_oops(false); + + if (is_entry_frame()) { + return sender_for_entry_frame(map); + } + if (is_interpreted_frame()) { + return sender_for_interpreter_frame(map); + } + assert(_cb == CodeCache::find_blob(pc()),"Must be the same"); + + // This test looks odd: why is it not is_compiled_frame() ? That's + // because stubs also have OOP maps. + if (_cb != NULL) { + return sender_for_compiled_frame(map); + } + + // Must be native-compiled frame, i.e. the marshaling code for native + // methods that exists in the core system. + return frame(sender_sp(), link(), sender_pc()); +} + +//------------------------------------------------------------------------------ +// frame::sender_for_compiled_frame +frame frame::sender_for_compiled_frame(RegisterMap* map) const { + // we cannot rely upon the last fp having been saved to the thread + // in C2 code but it will have been pushed onto the stack. so we + // have to find it relative to the unextended sp + + assert(_cb->frame_size() >= 0, "must have non-zero frame size"); + intptr_t* l_sender_sp = unextended_sp() + _cb->frame_size(); + intptr_t* unextended_sp = l_sender_sp; + + // the return_address is always the word on the stack + address sender_pc = (address) *(l_sender_sp + frame::return_addr_offset); + + intptr_t** saved_fp_addr = (intptr_t**) (l_sender_sp + frame::link_offset); + + assert(map != NULL, "map must be set"); + if (map->update_map()) { + // Tell GC to use argument oopmaps for some runtime stubs that need it. + // For C1, the runtime stub might not have oop maps, so set this flag + // outside of update_register_map. + map->set_include_argument_oops(_cb->caller_must_gc_arguments(map->thread())); + if (_cb->oop_maps() != NULL) { + OopMapSet::update_register_map(this, map); + } + + // Since the prolog does the save and restore of FP there is no + // oopmap for it so we must fill in its location as if there was + // an oopmap entry since if our caller was compiled code there + // could be live jvm state in it. + update_map_with_saved_link(map, saved_fp_addr); + } + + return frame(l_sender_sp, unextended_sp, *saved_fp_addr, sender_pc); +} + +//------------------------------------------------------------------------------ +// frame::update_map_with_saved_link +template +void frame::update_map_with_saved_link(RegisterMapT* map, intptr_t** link_addr) { + // The interpreter and compiler(s) always save fp in a known + // location on entry. We must record where that location is + // so that if fp was live on callout from c2 we can find + // the saved copy no matter what it called. + + // Since the interpreter always saves fp if we record where it is then + // we don't have to always save fp on entry and exit to c2 compiled + // code, on entry will be enough. + assert(map != NULL, "map must be set"); + map->set_location(::fp->as_VMReg(), (address) link_addr); + // this is weird "H" ought to be at a higher address however the + // oopMaps seems to have the "H" regs at the same address and the + // vanilla register. + map->set_location(::fp->as_VMReg()->next(), (address) link_addr); +} + #endif // CPU_RISCV_FRAME_RISCV_INLINE_HPP diff --git a/src/hotspot/cpu/riscv/gc/shared/barrierSetNMethod_riscv.cpp b/src/hotspot/cpu/riscv/gc/shared/barrierSetNMethod_riscv.cpp index ae7ee4c5a44..07a2f233a28 100644 --- a/src/hotspot/cpu/riscv/gc/shared/barrierSetNMethod_riscv.cpp +++ b/src/hotspot/cpu/riscv/gc/shared/barrierSetNMethod_riscv.cpp @@ -1,6 +1,6 @@ /* - * Copyright (c) 2018, 2020, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved. + * Copyright (c) 2018, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -29,6 +29,7 @@ #include "gc/shared/barrierSetNMethod.hpp" #include "logging/log.hpp" #include "memory/resourceArea.hpp" +#include "runtime/frame.inline.hpp" #include "runtime/sharedRuntime.hpp" #include "runtime/registerMap.hpp" #include "runtime/thread.hpp" @@ -161,6 +162,10 @@ void BarrierSetNMethod::disarm(nmethod* nm) { barrier->set_value(disarmed_value()); } +void BarrierSetNMethod::arm(nmethod* nm, int arm_value) { + Unimplemented(); +} + bool BarrierSetNMethod::is_armed(nmethod* nm) { if (!supports_entry_barrier(nm)) { return false; diff --git a/src/hotspot/cpu/riscv/nativeInst_riscv.hpp b/src/hotspot/cpu/riscv/nativeInst_riscv.hpp index 718b2e3de6c..d6e0bac1bac 100644 --- a/src/hotspot/cpu/riscv/nativeInst_riscv.hpp +++ b/src/hotspot/cpu/riscv/nativeInst_riscv.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2022, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2014, 2018, Red Hat Inc. All rights reserved. * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. @@ -569,4 +569,35 @@ public: } }; +class NativePostCallNop: public NativeInstruction { +public: + bool check() const { Unimplemented(); return false; } + int displacement() const { Unimplemented(); return 0; } + void patch(jint diff) { Unimplemented(); } + void make_deopt() { Unimplemented(); } +}; + +inline NativePostCallNop* nativePostCallNop_at(address address) { + Unimplemented(); + return NULL; +} + +class NativeDeoptInstruction: public NativeInstruction { +public: + address instruction_address() const { Unimplemented(); return NULL; } + address next_instruction_address() const { Unimplemented(); return NULL; } + + void verify() { Unimplemented(); } + + static bool is_deopt_at(address instr) { + Unimplemented(); + return false; + } + + // MT-safe patching + static void insert(address code_pos) { + Unimplemented(); + } +}; + #endif // CPU_RISCV_NATIVEINST_RISCV_HPP diff --git a/src/hotspot/cpu/riscv/registerMap_riscv.cpp b/src/hotspot/cpu/riscv/registerMap_riscv.cpp index 26c1edc36ff..41041eef4c0 100644 --- a/src/hotspot/cpu/riscv/registerMap_riscv.cpp +++ b/src/hotspot/cpu/riscv/registerMap_riscv.cpp @@ -1,6 +1,6 @@ /* - * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2021, Huawei Technologies Co., Ltd. All rights reserved. + * Copyright (c) 2021, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2021, 2022, Huawei Technologies Co., Ltd. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -33,13 +33,13 @@ address RegisterMap::pd_location(VMReg base_reg, int slot_idx) const { int base_reg_enc = (base_reg->value() - ConcreteRegisterImpl::max_fpr) / VectorRegisterImpl::max_slots_per_register; intptr_t offset_in_bytes = slot_idx * VMRegImpl::stack_slot_size; - address base_location = location(base_reg); + address base_location = location(base_reg, nullptr); if (base_location != NULL) { return base_location + offset_in_bytes; } else { return NULL; } } else { - return location(base_reg->next(slot_idx)); + return location(base_reg->next(slot_idx), nullptr); } } diff --git a/src/hotspot/cpu/riscv/smallRegisterMap_riscv.inline.hpp b/src/hotspot/cpu/riscv/smallRegisterMap_riscv.inline.hpp new file mode 100644 index 00000000000..33ce0bdfbc3 --- /dev/null +++ b/src/hotspot/cpu/riscv/smallRegisterMap_riscv.inline.hpp @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_RISCV_SMALLREGISTERMAP_RISCV_INLINE_HPP +#define CPU_RISCV_SMALLREGISTERMAP_RISCV_INLINE_HPP + +#include "runtime/frame.inline.hpp" +#include "runtime/registerMap.hpp" + +// Java frames don't have callee saved registers (except for rfp), so we can use a smaller RegisterMap +class SmallRegisterMap { +public: + static constexpr SmallRegisterMap* instance = nullptr; +private: + static void assert_is_rfp(VMReg r) PRODUCT_RETURN + DEBUG_ONLY({ Unimplemented(); }) +public: + // as_RegisterMap is used when we didn't want to templatize and abstract over RegisterMap type to support SmallRegisterMap + // Consider enhancing SmallRegisterMap to support those cases + const RegisterMap* as_RegisterMap() const { return nullptr; } + RegisterMap* as_RegisterMap() { return nullptr; } + + RegisterMap* copy_to_RegisterMap(RegisterMap* map, intptr_t* sp) const { + Unimplemented(); + return map; + } + + SmallRegisterMap() {} + + SmallRegisterMap(const RegisterMap* map) { + Unimplemented(); + } + + inline address location(VMReg reg, intptr_t* sp) const { + Unimplemented(); + return NULL; + } + + inline void set_location(VMReg reg, address loc) { assert_is_rfp(reg); } + + JavaThread* thread() const { + #ifndef ASSERT + guarantee (false, ""); + #endif + return nullptr; + } + + bool update_map() const { return false; } + bool walk_cont() const { return false; } + bool include_argument_oops() const { return false; } + void set_include_argument_oops(bool f) {} + bool in_cont() const { return false; } + stackChunkHandle stack_chunk() const { return stackChunkHandle(); } + +#ifdef ASSERT + bool should_skip_missing() const { return false; } + VMReg find_register_spilled_here(void* p, intptr_t* sp) { + Unimplemented(); + return NULL; + } + void print() const { print_on(tty); } + void print_on(outputStream* st) const { st->print_cr("Small register map"); } +#endif +}; + +#endif // CPU_RISCV_SMALLREGISTERMAP_RISCV_INLINE_HPP diff --git a/src/hotspot/cpu/riscv/stackChunkFrameStream_riscv.inline.hpp b/src/hotspot/cpu/riscv/stackChunkFrameStream_riscv.inline.hpp new file mode 100644 index 00000000000..3a421bd5374 --- /dev/null +++ b/src/hotspot/cpu/riscv/stackChunkFrameStream_riscv.inline.hpp @@ -0,0 +1,115 @@ +/* + * Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_RISCV_STACKCHUNKFRAMESTREAM_RISCV_INLINE_HPP +#define CPU_RISCV_STACKCHUNKFRAMESTREAM_RISCV_INLINE_HPP + +#include "interpreter/oopMapCache.hpp" +#include "runtime/frame.inline.hpp" +#include "runtime/registerMap.hpp" + +#ifdef ASSERT +template +inline bool StackChunkFrameStream::is_in_frame(void* p0) const { + Unimplemented(); + return true; +} +#endif + +template +inline frame StackChunkFrameStream::to_frame() const { + Unimplemented(); + return frame(); +} + +template +inline address StackChunkFrameStream::get_pc() const { + Unimplemented(); + return NULL; +} + +template +inline intptr_t* StackChunkFrameStream::fp() const { + Unimplemented(); + return NULL; +} + +template +inline intptr_t* StackChunkFrameStream::derelativize(int offset) const { + Unimplemented(); + return NULL; +} + +template +inline intptr_t* StackChunkFrameStream::unextended_sp_for_interpreter_frame() const { + Unimplemented(); + return NULL; +} + +template +intptr_t* StackChunkFrameStream::next_sp_for_interpreter_frame() const { + Unimplemented(); + return NULL; +} + +template +inline void StackChunkFrameStream::next_for_interpreter_frame() { + Unimplemented(); +} + +template +inline int StackChunkFrameStream::interpreter_frame_size() const { + Unimplemented(); + return 0; +} + +template +inline int StackChunkFrameStream::interpreter_frame_stack_argsize() const { + Unimplemented(); + return 0; +} + +template +inline int StackChunkFrameStream::interpreter_frame_num_oops() const { + Unimplemented(); + return 0; +} + +template<> +template<> +inline void StackChunkFrameStream::update_reg_map_pd(RegisterMap* map) { + Unimplemented(); +} + +template<> +template<> +inline void StackChunkFrameStream::update_reg_map_pd(RegisterMap* map) { + Unimplemented(); +} + +template +template +inline void StackChunkFrameStream::update_reg_map_pd(RegisterMapT* map) {} + +#endif // CPU_RISCV_STACKCHUNKFRAMESTREAM_RISCV_INLINE_HPP diff --git a/src/hotspot/cpu/riscv/stackChunkOop_riscv.inline.hpp b/src/hotspot/cpu/riscv/stackChunkOop_riscv.inline.hpp new file mode 100644 index 00000000000..0d1f406fb4e --- /dev/null +++ b/src/hotspot/cpu/riscv/stackChunkOop_riscv.inline.hpp @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_RISCV_STACKCHUNKOOP_RISCV_INLINE_HPP +#define CPU_RISCV_STACKCHUNKOOP_RISCV_INLINE_HPP + +inline void stackChunkOopDesc::relativize_frame_pd(frame& fr) const { + Unimplemented(); +} + +inline void stackChunkOopDesc::derelativize_frame_pd(frame& fr) const { + Unimplemented(); +} + +#endif // CPU_RISCV_STACKCHUNKOOP_RISCV_INLINE_HPP diff --git a/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp b/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp index 602b01cfc4a..abd50c354bb 100644 --- a/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp +++ b/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp @@ -3810,10 +3810,10 @@ class StubGenerator: public StubCodeGenerator { }; // end class declaration #define UCM_TABLE_MAX_ENTRIES 8 -void StubGenerator_generate(CodeBuffer* code, bool all) { +void StubGenerator_generate(CodeBuffer* code, int phase) { if (UnsafeCopyMemory::_table == NULL) { UnsafeCopyMemory::create_table(UCM_TABLE_MAX_ENTRIES); } - StubGenerator g(code, all); + StubGenerator g(code, phase); } diff --git a/src/hotspot/cpu/riscv/templateInterpreterGenerator_riscv.cpp b/src/hotspot/cpu/riscv/templateInterpreterGenerator_riscv.cpp index 5e4bcf18ab1..57156c8a0d8 100644 --- a/src/hotspot/cpu/riscv/templateInterpreterGenerator_riscv.cpp +++ b/src/hotspot/cpu/riscv/templateInterpreterGenerator_riscv.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2003, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2003, 2022, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved. * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. @@ -797,6 +797,11 @@ void TemplateInterpreterGenerator::generate_fixed_frame(bool native_call) { // End of helpers +address TemplateInterpreterGenerator::generate_Continuation_doYield_entry(void) { + Unimplemented(); + return NULL; +} + // Various method entries //------------------------------------------------------------------------------------------------------------------------ // diff --git a/src/hotspot/cpu/s390/assembler_s390.hpp b/src/hotspot/cpu/s390/assembler_s390.hpp index bad9f393d98..df003ef4bcd 100644 --- a/src/hotspot/cpu/s390/assembler_s390.hpp +++ b/src/hotspot/cpu/s390/assembler_s390.hpp @@ -392,7 +392,10 @@ class Argument { // Only 5 registers may contain integer parameters. n_register_parameters = 5, // Can have up to 4 floating registers. - n_float_register_parameters = 4 + n_float_register_parameters = 4, + + n_int_register_parameters_j = n_register_parameters, + n_float_register_parameters_j = n_float_register_parameters }; // creation diff --git a/src/hotspot/cpu/s390/c1_LIRGenerator_s390.cpp b/src/hotspot/cpu/s390/c1_LIRGenerator_s390.cpp index d5e5c01ebde..eb13148fb96 100644 --- a/src/hotspot/cpu/s390/c1_LIRGenerator_s390.cpp +++ b/src/hotspot/cpu/s390/c1_LIRGenerator_s390.cpp @@ -1183,3 +1183,7 @@ void LIRGenerator::do_FmaIntrinsic(Intrinsic* x) { void LIRGenerator::do_vectorizedMismatch(Intrinsic* x) { fatal("vectorizedMismatch intrinsic is not implemented on this platform"); } + +void LIRGenerator::do_continuation_doYield(Intrinsic* x) { + fatal("Continuation.doYield intrinsic is not implemented on this platform"); +} diff --git a/src/hotspot/cpu/s390/continuationEntry_s390.inline.hpp b/src/hotspot/cpu/s390/continuationEntry_s390.inline.hpp new file mode 100644 index 00000000000..1d4e3c2439d --- /dev/null +++ b/src/hotspot/cpu/s390/continuationEntry_s390.inline.hpp @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_S390_CONTINUATIONENTRY_S390_INLINE_HPP +#define CPU_S390_CONTINUATIONENTRY_S390_INLINE_HPP + +#include "runtime/continuationEntry.hpp" + +// TODO: Implement + +inline frame ContinuationEntry::to_frame() const { + Unimplemented(); + return frame(); +} + +inline intptr_t* ContinuationEntry::entry_fp() const { + Unimplemented(); + return nullptr; +} + +inline void ContinuationEntry::update_register_map(RegisterMap* map) const { + Unimplemented(); +} + +#endif // CPU_S390_CONTINUATIONENTRY_S390_INLINE_HPP diff --git a/src/hotspot/cpu/s390/continuationFreezeThaw_s390.inline.hpp b/src/hotspot/cpu/s390/continuationFreezeThaw_s390.inline.hpp new file mode 100644 index 00000000000..de6ace6823c --- /dev/null +++ b/src/hotspot/cpu/s390/continuationFreezeThaw_s390.inline.hpp @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_S390_CONTINUATION_S390_INLINE_HPP +#define CPU_S390_CONTINUATION_S390_INLINE_HPP + +#include "oops/stackChunkOop.inline.hpp" +#include "runtime/frame.hpp" +#include "runtime/frame.inline.hpp" + +inline void FreezeBase::set_top_frame_metadata_pd(const frame& hf) { + Unimplemented(); +} + +template +inline frame FreezeBase::sender(const frame& f) { + Unimplemented(); + return frame(); +} + +template frame FreezeBase::new_heap_frame(frame& f, frame& caller) { + Unimplemented(); + return frame(); +} + +void FreezeBase::adjust_interpreted_frame_unextended_sp(frame& f) { + Unimplemented(); +} + +inline void FreezeBase::relativize_interpreted_frame_metadata(const frame& f, const frame& hf) { + Unimplemented(); +} + +inline void FreezeBase::patch_pd(frame& hf, const frame& caller) { + Unimplemented(); +} + +inline void FreezeBase::patch_stack_pd(intptr_t* frame_sp, intptr_t* heap_sp) { + Unimplemented(); +} + +inline frame ThawBase::new_entry_frame() { + Unimplemented(); + return frame(); +} + +template frame ThawBase::new_stack_frame(const frame& hf, frame& caller, bool bottom) { + Unimplemented(); + return frame(); +} + +inline void ThawBase::set_interpreter_frame_bottom(const frame& f, intptr_t* bottom) { + Unimplemented(); +} + +inline void ThawBase::derelativize_interpreted_frame_metadata(const frame& hf, const frame& f) { + Unimplemented(); +} + +inline intptr_t* ThawBase::align(const frame& hf, intptr_t* frame_sp, frame& caller, bool bottom) { + Unimplemented(); + return NULL; +} + +inline void ThawBase::patch_pd(frame& f, const frame& caller) { + Unimplemented(); +} + +void ThawBase::patch_chunk_pd(intptr_t* sp) { + Unimplemented(); +} + +inline void ThawBase::prefetch_chunk_pd(void* start, int size) { + Unimplemented(); +} + +#endif // CPU_S390_CONTINUATION_S390_INLINE_HPP diff --git a/src/hotspot/cpu/s390/continuationHelper_s390.inline.hpp b/src/hotspot/cpu/s390/continuationHelper_s390.inline.hpp new file mode 100644 index 00000000000..93568916520 --- /dev/null +++ b/src/hotspot/cpu/s390/continuationHelper_s390.inline.hpp @@ -0,0 +1,125 @@ +/* + * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_S390_CONTINUATIONHELPER_S390_INLINE_HPP +#define CPU_S390_CONTINUATIONHELPER_S390_INLINE_HPP + +#include "runtime/continuationHelper.hpp" + +// TODO: Implement + +template +static inline intptr_t** link_address(const frame& f) { + Unimplemented(); + return NULL; +} + +inline int ContinuationHelper::frame_align_words(int size) { + Unimplemented(); + return 0; +} + +inline intptr_t* ContinuationHelper::frame_align_pointer(intptr_t* sp) { + Unimplemented(); + return NULL; +} + +template +inline void ContinuationHelper::update_register_map(const frame& f, RegisterMap* map) { + Unimplemented(); +} + +inline void ContinuationHelper::update_register_map_with_callee(const frame& f, RegisterMap* map) { + Unimplemented(); +} + +inline void ContinuationHelper::push_pd(const frame& f) { + Unimplemented(); +} + +inline void ContinuationHelper::set_anchor_to_entry_pd(JavaFrameAnchor* anchor, ContinuationEntry* cont) { + Unimplemented(); +} + +#ifdef ASSERT +inline void ContinuationHelper::set_anchor_pd(JavaFrameAnchor* anchor, intptr_t* sp) { + Unimplemented(); +} + +inline bool ContinuationHelper::Frame::assert_frame_laid_out(frame f) { + Unimplemented(); + return false; +} +#endif + +inline intptr_t** ContinuationHelper::Frame::callee_link_address(const frame& f) { + Unimplemented(); + return NULL; +} + +template +static inline intptr_t* real_fp(const frame& f) { + Unimplemented(); + return NULL; +} + +inline address* ContinuationHelper::InterpretedFrame::return_pc_address(const frame& f) { + Unimplemented(); + return NULL; +} + +inline void ContinuationHelper::InterpretedFrame::patch_sender_sp(frame& f, intptr_t* sp) { + Unimplemented(); +} + +inline address* ContinuationHelper::Frame::return_pc_address(const frame& f) { + Unimplemented(); + return NULL; +} + +inline address ContinuationHelper::Frame::real_pc(const frame& f) { + Unimplemented(); + return NULL; +} + +inline void ContinuationHelper::Frame::patch_pc(const frame& f, address pc) { + Unimplemented(); +} + +inline intptr_t* ContinuationHelper::InterpretedFrame::frame_top(const frame& f, InterpreterOopMap* mask) { // inclusive; this will be copied with the frame + Unimplemented(); + return NULL; +} + +inline intptr_t* ContinuationHelper::InterpretedFrame::frame_bottom(const frame& f) { // exclusive; this will not be copied with the frame + Unimplemented(); + return NULL; +} + +inline intptr_t* ContinuationHelper::InterpretedFrame::frame_top(const frame& f, int callee_argsize, bool callee_interpreted) { + Unimplemented(); + return NULL; +} + +#endif // CPU_S390_CONTINUATIONHELPER_S390_INLINE_HPP diff --git a/src/hotspot/cpu/s390/frame_s390.cpp b/src/hotspot/cpu/s390/frame_s390.cpp index 3588915f79d..dc964f8dc78 100644 --- a/src/hotspot/cpu/s390/frame_s390.cpp +++ b/src/hotspot/cpu/s390/frame_s390.cpp @@ -230,27 +230,6 @@ frame frame::sender_for_interpreter_frame(RegisterMap *map) const { return frame(sender_sp(), sender_pc(), (intptr_t*)(ijava_state()->sender_sp)); } -frame frame::sender_for_compiled_frame(RegisterMap *map) const { - assert(map != NULL, "map must be set"); - // Frame owned by compiler. - - address pc = *compiled_sender_pc_addr(_cb); - frame caller(compiled_sender_sp(_cb), pc); - - // Now adjust the map. - - // Get the rest. - if (map->update_map()) { - // Tell GC to use argument oopmaps for some runtime stubs that need it. - map->set_include_argument_oops(_cb->caller_must_gc_arguments(map->thread())); - if (_cb->oop_maps() != NULL) { - OopMapSet::update_register_map(this, map); - } - } - - return caller; -} - intptr_t* frame::compiled_sender_sp(CodeBlob* cb) const { return sender_sp(); } @@ -259,26 +238,6 @@ address* frame::compiled_sender_pc_addr(CodeBlob* cb) const { return sender_pc_addr(); } -frame frame::sender(RegisterMap* map) const { - // Default is we don't have to follow them. The sender_for_xxx will - // update it accordingly. - map->set_include_argument_oops(false); - - if (is_entry_frame()) { - return sender_for_entry_frame(map); - } - if (is_interpreted_frame()) { - return sender_for_interpreter_frame(map); - } - assert(_cb == CodeCache::find_blob(pc()),"Must be the same"); - if (_cb != NULL) { - return sender_for_compiled_frame(map); - } - // Must be native-compiled frame, i.e. the marshaling code for native - // methods that exists in the core system. - return frame(sender_sp(), sender_pc()); -} - void frame::patch_pc(Thread* thread, address pc) { assert(_cb == CodeCache::find_blob(pc), "unexpected pc"); if (TracePcPatching) { @@ -680,3 +639,13 @@ intptr_t *frame::initial_deoptimization_info() { // Used to reset the saved FP. return fp(); } + +// Pointer beyond the "oldest/deepest" BasicObjectLock on stack. +BasicObjectLock* frame::interpreter_frame_monitor_end() const { + return interpreter_frame_monitors(); +} + +intptr_t* frame::interpreter_frame_tos_at(jint offset) const { + return &interpreter_frame_tos_address()[offset]; +} + diff --git a/src/hotspot/cpu/s390/frame_s390.hpp b/src/hotspot/cpu/s390/frame_s390.hpp index 86622a96d4a..7e592001eff 100644 --- a/src/hotspot/cpu/s390/frame_s390.hpp +++ b/src/hotspot/cpu/s390/frame_s390.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2016, 2022, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2016 SAP SE. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * @@ -461,11 +461,11 @@ private: inline void find_codeblob_and_set_pc_and_deopt_state(address pc); + const ImmutableOopMap* get_oop_map() const; // Constructors public: - inline frame(intptr_t* sp); // To be used, if sp was not extended to match callee's calling convention. inline frame(intptr_t* sp, address pc); inline frame(intptr_t* sp, address pc, intptr_t* unextended_sp); @@ -486,6 +486,10 @@ address* sender_pc_addr(void) const; public: + inline intptr_t* interpreter_frame_last_sp() const; + + template + static void update_map_with_saved_link(RegisterMapT* map, intptr_t** link_addr); // Additional interface for interpreter frames: static int interpreter_frame_interpreterstate_size_in_bytes(); @@ -550,6 +554,10 @@ // // Normal return address is the instruction following the branch. pc_return_offset = 0, + metadata_words = 0, + frame_alignment = 16, + // size, in words, of maximum shift in frame position due to alignment + align_wiggle = 1 }; static jint interpreter_frame_expression_stack_direction() { return -1; } diff --git a/src/hotspot/cpu/s390/frame_s390.inline.hpp b/src/hotspot/cpu/s390/frame_s390.inline.hpp index ca36b4730ea..ba6b882fe6c 100644 --- a/src/hotspot/cpu/s390/frame_s390.inline.hpp +++ b/src/hotspot/cpu/s390/frame_s390.inline.hpp @@ -54,24 +54,44 @@ inline void frame::find_codeblob_and_set_pc_and_deopt_state(address pc) { // Constructors // Initialize all fields, _unextended_sp will be adjusted in find_codeblob_and_set_pc_and_deopt_state. -inline frame::frame() : _sp(NULL), _pc(NULL), _cb(NULL), _deopt_state(unknown), _unextended_sp(NULL), _fp(NULL) {} +inline frame::frame() : _sp(NULL), _pc(NULL), _cb(NULL), _deopt_state(unknown), _on_heap(false), +#ifdef ASSERT + _frame_index(-1), +#endif + _unextended_sp(NULL), _fp(NULL) {} -inline frame::frame(intptr_t* sp) : _sp(sp), _unextended_sp(sp) { +inline frame::frame(intptr_t* sp) : _sp(sp), _on_heap(false), +#ifdef ASSERT + _frame_index(-1), +#endif + _unextended_sp(sp) { find_codeblob_and_set_pc_and_deopt_state((address)own_abi()->return_pc); } -inline frame::frame(intptr_t* sp, address pc) : _sp(sp), _unextended_sp(sp) { +inline frame::frame(intptr_t* sp, address pc) : _sp(sp), _on_heap(false), +#ifdef ASSERT + _frame_index(-1), +#endif + _unextended_sp(sp) { find_codeblob_and_set_pc_and_deopt_state(pc); // Also sets _fp and adjusts _unextended_sp. } -inline frame::frame(intptr_t* sp, address pc, intptr_t* unextended_sp) : _sp(sp), _unextended_sp(unextended_sp) { +inline frame::frame(intptr_t* sp, address pc, intptr_t* unextended_sp) : _sp(sp), _on_heap(false), +#ifdef ASSERT + _frame_index(-1), +#endif + _unextended_sp(unextended_sp) { find_codeblob_and_set_pc_and_deopt_state(pc); // Also sets _fp and adjusts _unextended_sp. } // Generic constructor. Used by pns() in debug.cpp only #ifndef PRODUCT inline frame::frame(void* sp, void* pc, void* unextended_sp) : - _sp((intptr_t*)sp), _pc(NULL), _cb(NULL), _unextended_sp((intptr_t*)unextended_sp) { + _sp((intptr_t*)sp), _pc(NULL), _cb(NULL), _on_heap(false), +#ifdef ASSERT + _frame_index(-1), +#endif + _unextended_sp((intptr_t*)unextended_sp) { find_codeblob_and_set_pc_and_deopt_state((address)pc); // Also sets _fp and adjusts _unextended_sp. } #endif @@ -119,7 +139,7 @@ inline bool frame::is_older(intptr_t* id) const { return this->id() > id; } -inline int frame::frame_size(RegisterMap* map) const { +inline int frame::frame_size() const { // Stack grows towards smaller addresses on z/Linux: sender is at a higher address. return sender_sp() - sp(); } @@ -176,11 +196,6 @@ inline intptr_t* frame::interpreter_frame_expression_stack() const { return (intptr_t*)interpreter_frame_monitor_end() - 1; } -inline intptr_t* frame::interpreter_frame_tos_at(jint offset) const { - return &interpreter_frame_tos_address()[offset]; -} - - // monitor elements // End is lower in memory than begin, and beginning element is oldest element. @@ -229,10 +244,6 @@ inline BasicObjectLock * frame::interpreter_frame_monitor_begin() const { return (BasicObjectLock*)ijava_state(); } -inline BasicObjectLock * frame::interpreter_frame_monitor_end() const { - return interpreter_frame_monitors(); -} - inline void frame::interpreter_frame_set_monitor_end(BasicObjectLock* monitors) { interpreter_frame_set_monitors((BasicObjectLock *)monitors); } @@ -281,15 +292,100 @@ inline JavaCallWrapper** frame::entry_frame_call_wrapper_addr() const { } inline oop frame::saved_oop_result(RegisterMap* map) const { - return *((oop*) map->location(Z_R2->as_VMReg())); // R2 is return register. + return *((oop*) map->location(Z_R2->as_VMReg(), nullptr)); // R2 is return register. } inline void frame::set_saved_oop_result(RegisterMap* map, oop obj) { - *((oop*) map->location(Z_R2->as_VMReg())) = obj; // R2 is return register. + *((oop*) map->location(Z_R2->as_VMReg(), nullptr)) = obj; // R2 is return register. } inline intptr_t* frame::real_fp() const { return fp(); } +inline const ImmutableOopMap* frame::get_oop_map() const { + Unimplemented(); + return NULL; +} + +inline int frame::compiled_frame_stack_argsize() const { + Unimplemented(); + return 0; +} + +inline void frame::interpreted_frame_oop_map(InterpreterOopMap* mask) const { + Unimplemented(); +} + +inline intptr_t* frame::interpreter_frame_last_sp() const { + Unimplemented(); + return NULL; +} + +inline int frame::sender_sp_ret_address_offset() { + Unimplemented(); + return 0; +} + +inline void frame::set_unextended_sp(intptr_t* value) { + Unimplemented(); +} + +inline int frame::offset_unextended_sp() const { + Unimplemented(); + return 0; +} + +inline void frame::set_offset_unextended_sp(int value) { + Unimplemented(); +} + +//------------------------------------------------------------------------------ +// frame::sender + +inline frame frame::sender(RegisterMap* map) const { + // Default is we don't have to follow them. The sender_for_xxx will + // update it accordingly. + map->set_include_argument_oops(false); + + if (is_entry_frame()) { + return sender_for_entry_frame(map); + } + if (is_interpreted_frame()) { + return sender_for_interpreter_frame(map); + } + assert(_cb == CodeCache::find_blob(pc()),"Must be the same"); + if (_cb != NULL) return sender_for_compiled_frame(map); + + // Must be native-compiled frame, i.e. the marshaling code for native + // methods that exists in the core system. + return frame(sender_sp(), sender_pc()); +} + +inline frame frame::sender_for_compiled_frame(RegisterMap *map) const { + assert(map != NULL, "map must be set"); + // Frame owned by compiler. + + address pc = *compiled_sender_pc_addr(_cb); + frame caller(compiled_sender_sp(_cb), pc); + + // Now adjust the map. + + // Get the rest. + if (map->update_map()) { + // Tell GC to use argument oopmaps for some runtime stubs that need it. + map->set_include_argument_oops(_cb->caller_must_gc_arguments(map->thread())); + if (_cb->oop_maps() != NULL) { + OopMapSet::update_register_map(this, map); + } + } + + return caller; +} + +template +void frame::update_map_with_saved_link(RegisterMapT* map, intptr_t** link_addr) { + Unimplemented(); +} + #endif // CPU_S390_FRAME_S390_INLINE_HPP diff --git a/src/hotspot/cpu/s390/nativeInst_s390.hpp b/src/hotspot/cpu/s390/nativeInst_s390.hpp index 8ed919686c2..29cf5f1b03e 100644 --- a/src/hotspot/cpu/s390/nativeInst_s390.hpp +++ b/src/hotspot/cpu/s390/nativeInst_s390.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2016, 2022, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2016 SAP SE. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * @@ -654,4 +654,35 @@ class NativeGeneralJump: public NativeInstruction { void verify() PRODUCT_RETURN; }; +class NativePostCallNop: public NativeInstruction { +public: + bool check() const { Unimplemented(); return false; } + int displacement() const { Unimplemented(); return 0; } + void patch(jint diff) { Unimplemented(); } + void make_deopt() { Unimplemented(); } +}; + +inline NativePostCallNop* nativePostCallNop_at(address address) { + Unimplemented(); + return NULL; +} + +class NativeDeoptInstruction: public NativeInstruction { +public: + address instruction_address() const { Unimplemented(); return NULL; } + address next_instruction_address() const { Unimplemented(); return NULL; } + + void verify() { Unimplemented(); } + + static bool is_deopt_at(address instr) { + Unimplemented(); + return false; + } + + // MT-safe patching + static void insert(address code_pos) { + Unimplemented(); + } +}; + #endif // CPU_S390_NATIVEINST_S390_HPP diff --git a/src/hotspot/cpu/s390/registerMap_s390.hpp b/src/hotspot/cpu/s390/registerMap_s390.hpp index 6fa7a5a12da..74cf3855fa6 100644 --- a/src/hotspot/cpu/s390/registerMap_s390.hpp +++ b/src/hotspot/cpu/s390/registerMap_s390.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2016, 2022, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2016 SAP SE. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * @@ -37,7 +37,7 @@ address pd_location(VMReg reg) const {return NULL;} address pd_location(VMReg base_reg, int slot_idx) const { - return location(base_reg->next(slot_idx)); + return location(base_reg->next(slot_idx), nullptr); } // No PD state to clear or copy. diff --git a/src/hotspot/cpu/s390/smallRegisterMap_s390.inline.hpp b/src/hotspot/cpu/s390/smallRegisterMap_s390.inline.hpp new file mode 100644 index 00000000000..594cc0a42ea --- /dev/null +++ b/src/hotspot/cpu/s390/smallRegisterMap_s390.inline.hpp @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_S390_SMALLREGISTERMAP_S390_INLINE_HPP +#define CPU_S390_SMALLREGISTERMAP_S390_INLINE_HPP + +#include "runtime/frame.inline.hpp" +#include "runtime/registerMap.hpp" + +// Java frames don't have callee saved registers (except for rfp), so we can use a smaller RegisterMap +class SmallRegisterMap { +public: + static constexpr SmallRegisterMap* instance = nullptr; +private: + static void assert_is_rfp(VMReg r) PRODUCT_RETURN + DEBUG_ONLY({ Unimplemented(); }) +public: + // as_RegisterMap is used when we didn't want to templatize and abstract over RegisterMap type to support SmallRegisterMap + // Consider enhancing SmallRegisterMap to support those cases + const RegisterMap* as_RegisterMap() const { return nullptr; } + RegisterMap* as_RegisterMap() { return nullptr; } + + RegisterMap* copy_to_RegisterMap(RegisterMap* map, intptr_t* sp) const { + Unimplemented(); + return map; + } + + SmallRegisterMap() {} + + SmallRegisterMap(const RegisterMap* map) { + Unimplemented(); + } + + inline address location(VMReg reg, intptr_t* sp) const { + Unimplemented(); + return NULL; + } + + inline void set_location(VMReg reg, address loc) { assert_is_rfp(reg); } + + JavaThread* thread() const { + #ifndef ASSERT + guarantee (false, ""); + #endif + return nullptr; + } + + bool update_map() const { return false; } + bool walk_cont() const { return false; } + bool include_argument_oops() const { return false; } + void set_include_argument_oops(bool f) {} + bool in_cont() const { return false; } + stackChunkHandle stack_chunk() const { return stackChunkHandle(); } + +#ifdef ASSERT + bool should_skip_missing() const { return false; } + VMReg find_register_spilled_here(void* p, intptr_t* sp) { + Unimplemented(); + return NULL; + } + void print() const { print_on(tty); } + void print_on(outputStream* st) const { st->print_cr("Small register map"); } +#endif +}; + +#endif // CPU_S390_SMALLREGISTERMAP_S390_INLINE_HPP diff --git a/src/hotspot/cpu/s390/stackChunkFrameStream_s390.inline.hpp b/src/hotspot/cpu/s390/stackChunkFrameStream_s390.inline.hpp new file mode 100644 index 00000000000..2df1e38a9de --- /dev/null +++ b/src/hotspot/cpu/s390/stackChunkFrameStream_s390.inline.hpp @@ -0,0 +1,115 @@ +/* + * Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_S390_STACKCHUNKFRAMESTREAM_S390_INLINE_HPP +#define CPU_S390_STACKCHUNKFRAMESTREAM_S390_INLINE_HPP + +#include "interpreter/oopMapCache.hpp" +#include "runtime/frame.inline.hpp" +#include "runtime/registerMap.hpp" + +#ifdef ASSERT +template +inline bool StackChunkFrameStream::is_in_frame(void* p0) const { + Unimplemented(); + return true; +} +#endif + +template +inline frame StackChunkFrameStream::to_frame() const { + Unimplemented(); + return frame(); +} + +template +inline address StackChunkFrameStream::get_pc() const { + Unimplemented(); + return NULL; +} + +template +inline intptr_t* StackChunkFrameStream::fp() const { + Unimplemented(); + return NULL; +} + +template +inline intptr_t* StackChunkFrameStream::derelativize(int offset) const { + Unimplemented(); + return NULL; +} + +template +inline intptr_t* StackChunkFrameStream::unextended_sp_for_interpreter_frame() const { + Unimplemented(); + return NULL; +} + +template +intptr_t* StackChunkFrameStream::next_sp_for_interpreter_frame() const { + Unimplemented(); + return NULL; +} + +template +inline void StackChunkFrameStream::next_for_interpreter_frame() { + Unimplemented(); +} + +template +inline int StackChunkFrameStream::interpreter_frame_size() const { + Unimplemented(); + return 0; +} + +template +inline int StackChunkFrameStream::interpreter_frame_stack_argsize() const { + Unimplemented(); + return 0; +} + +template +inline int StackChunkFrameStream::interpreter_frame_num_oops() const { + Unimplemented(); + return 0; +} + +template<> +template<> +inline void StackChunkFrameStream::update_reg_map_pd(RegisterMap* map) { + Unimplemented(); +} + +template<> +template<> +inline void StackChunkFrameStream::update_reg_map_pd(RegisterMap* map) { + Unimplemented(); +} + +template +template +inline void StackChunkFrameStream::update_reg_map_pd(RegisterMapT* map) {} + +#endif // CPU_S390_STACKCHUNKFRAMESTREAM_S390_INLINE_HPP diff --git a/src/hotspot/cpu/s390/stackChunkOop_s390.inline.hpp b/src/hotspot/cpu/s390/stackChunkOop_s390.inline.hpp new file mode 100644 index 00000000000..dfd3562c9d9 --- /dev/null +++ b/src/hotspot/cpu/s390/stackChunkOop_s390.inline.hpp @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_S390_STACKCHUNKOOP_S390_INLINE_HPP +#define CPU_S390_STACKCHUNKOOP_S390_INLINE_HPP + +inline void stackChunkOopDesc::relativize_frame_pd(frame& fr) const { + Unimplemented(); +} + +inline void stackChunkOopDesc::derelativize_frame_pd(frame& fr) const { + Unimplemented(); +} + +#endif // CPU_S390_STACKCHUNKOOP_S390_INLINE_HPP diff --git a/src/hotspot/cpu/s390/stubGenerator_s390.cpp b/src/hotspot/cpu/s390/stubGenerator_s390.cpp index 4ce8924d8bf..fbe8e0c09b4 100644 --- a/src/hotspot/cpu/s390/stubGenerator_s390.cpp +++ b/src/hotspot/cpu/s390/stubGenerator_s390.cpp @@ -3014,6 +3014,6 @@ class StubGenerator: public StubCodeGenerator { }; -void StubGenerator_generate(CodeBuffer* code, bool all) { - StubGenerator g(code, all); +void StubGenerator_generate(CodeBuffer* code, int phase) { + StubGenerator g(code, phase); } diff --git a/src/hotspot/cpu/s390/templateInterpreterGenerator_s390.cpp b/src/hotspot/cpu/s390/templateInterpreterGenerator_s390.cpp index f8a2d30c364..d74fafd3822 100644 --- a/src/hotspot/cpu/s390/templateInterpreterGenerator_s390.cpp +++ b/src/hotspot/cpu/s390/templateInterpreterGenerator_s390.cpp @@ -482,6 +482,11 @@ address TemplateInterpreterGenerator::generate_abstract_entry(void) { return __ addr_at(entry_offset); } +address TemplateInterpreterGenerator::generate_Continuation_doYield_entry(void) { + Unimplemented(); + return NULL; +} + address TemplateInterpreterGenerator::generate_Reference_get_entry(void) { // Inputs: // Z_ARG1 - receiver diff --git a/src/hotspot/cpu/x86/assembler_x86.hpp b/src/hotspot/cpu/x86/assembler_x86.hpp index 1898ac9e6e4..68174533229 100644 --- a/src/hotspot/cpu/x86/assembler_x86.hpp +++ b/src/hotspot/cpu/x86/assembler_x86.hpp @@ -49,7 +49,9 @@ class Argument { n_int_register_parameters_j = 6, // j_rarg0, j_rarg1, ... n_float_register_parameters_j = 8 // j_farg0, j_farg1, ... #else - n_register_parameters = 0 // 0 registers used to pass arguments + n_register_parameters = 0, // 0 registers used to pass arguments + n_int_register_parameters_j = 0, + n_float_register_parameters_j = 0 #endif // _LP64 }; }; diff --git a/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp b/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp index 1452fae4dc0..501dbfadd6e 100644 --- a/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp +++ b/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp @@ -438,7 +438,7 @@ int LIR_Assembler::emit_unwind_handler() { // Fetch the exception from TLS and clear out exception related thread state Register thread = NOT_LP64(rsi) LP64_ONLY(r15_thread); - NOT_LP64(__ get_thread(rsi)); + NOT_LP64(__ get_thread(thread)); __ movptr(rax, Address(thread, JavaThread::exception_oop_offset())); __ movptr(Address(thread, JavaThread::exception_oop_offset()), (intptr_t)NULL_WORD); __ movptr(Address(thread, JavaThread::exception_pc_offset()), (intptr_t)NULL_WORD); @@ -460,6 +460,8 @@ int LIR_Assembler::emit_unwind_handler() { __ unlock_object(rdi, rsi, rax, *stub->entry()); } __ bind(*stub->continuation()); + NOT_LP64(__ get_thread(thread);) + __ dec_held_monitor_count(thread); } if (compilation()->env()->dtrace_method_probes()) { @@ -2864,6 +2866,7 @@ void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) { "must be aligned"); __ call(AddressLiteral(op->addr(), rtype)); add_call_info(code_offset(), op->info()); + __ post_call_nop(); } @@ -2872,6 +2875,7 @@ void LIR_Assembler::ic_call(LIR_OpJavaCall* op) { add_call_info(code_offset(), op->info()); assert((__ offset() - NativeCall::instruction_size + NativeCall::displacement_offset) % BytesPerWord == 0, "must be aligned"); + __ post_call_nop(); } @@ -3507,7 +3511,38 @@ void LIR_Assembler::emit_lock(LIR_OpLock* op) { } else { Unimplemented(); } + if (op->code() == lir_lock) { + // If deoptimization happens in Runtime1::monitorenter, inc_held_monitor_count after backing from slowpath + // will be skipped. Solution is + // 1. Increase only in fastpath + // 2. Runtime1::monitorenter increase count after locking +#ifndef _LP64 + Register thread = rsi; + __ push(thread); + __ get_thread(thread); +#else + Register thread = r15_thread; +#endif + __ inc_held_monitor_count(thread); +#ifndef _LP64 + __ pop(thread); +#endif + } __ bind(*op->stub()->continuation()); + if (op->code() == lir_unlock) { + // unlock in slowpath is JRT_Leaf stub, no deoptimization can happen +#ifndef _LP64 + Register thread = rsi; + __ push(thread); + __ get_thread(thread); +#else + Register thread = r15_thread; +#endif + __ dec_held_monitor_count(thread); +#ifndef _LP64 + __ pop(thread); +#endif + } } void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) { @@ -3868,6 +3903,7 @@ void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* arg if (info != NULL) { add_call_info_here(info); } + __ post_call_nop(); } diff --git a/src/hotspot/cpu/x86/c1_LIRGenerator_x86.cpp b/src/hotspot/cpu/x86/c1_LIRGenerator_x86.cpp index a525f484075..6c0b03cfba6 100644 --- a/src/hotspot/cpu/x86/c1_LIRGenerator_x86.cpp +++ b/src/hotspot/cpu/x86/c1_LIRGenerator_x86.cpp @@ -336,6 +336,17 @@ void LIRGenerator::do_MonitorExit(MonitorExit* x) { monitor_exit(obj_temp, lock, syncTempOpr(), LIR_OprFact::illegalOpr, x->monitor_no()); } +void LIRGenerator::do_continuation_doYield(Intrinsic* x) { + BasicTypeList signature(0); + CallingConvention* cc = frame_map()->java_calling_convention(&signature, true); + + const LIR_Opr result_reg = result_register_for(x->type()); + address entry = StubRoutines::cont_doYield(); + LIR_Opr result = rlock_result(x); + CodeEmitInfo* info = state_for(x, x->state()); + __ call_runtime(entry, LIR_OprFact::illegalOpr, result_reg, cc->args(), info); + __ move(result_reg, result); +} // _ineg, _lneg, _fneg, _dneg void LIRGenerator::do_NegateOp(NegateOp* x) { @@ -362,7 +373,6 @@ void LIRGenerator::do_NegateOp(NegateOp* x) { set_result(x, round_item(reg)); } - // for _fadd, _fmul, _fsub, _fdiv, _frem // _dadd, _dmul, _dsub, _ddiv, _drem void LIRGenerator::do_ArithmeticOp_FPU(ArithmeticOp* x) { diff --git a/src/hotspot/cpu/x86/continuationEntry_x86.inline.hpp b/src/hotspot/cpu/x86/continuationEntry_x86.inline.hpp new file mode 100644 index 00000000000..92b27e8e1ba --- /dev/null +++ b/src/hotspot/cpu/x86/continuationEntry_x86.inline.hpp @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_X86_CONTINUATIONENTRY_X86_INLINE_HPP +#define CPU_X86_CONTINUATIONENTRY_X86_INLINE_HPP + +#include "runtime/continuationEntry.hpp" + +#include "runtime/frame.inline.hpp" +#include "runtime/registerMap.hpp" +#include "utilities/macros.hpp" + +inline frame ContinuationEntry::to_frame() const { + static CodeBlob* cb = CodeCache::find_blob_fast(entry_pc()); + assert(cb != nullptr, ""); + assert(cb->as_compiled_method()->method()->is_continuation_enter_intrinsic(), ""); + return frame(entry_sp(), entry_sp(), entry_fp(), entry_pc(), cb); +} + +inline intptr_t* ContinuationEntry::entry_fp() const { + return (intptr_t*)((address)this + size()); +} + +inline void ContinuationEntry::update_register_map(RegisterMap* map) const { + intptr_t** fp = (intptr_t**)(bottom_sender_sp() - frame::sender_sp_offset); + frame::update_map_with_saved_link(map, fp); +} + +#endif // CPU_X86_CONTINUATIONENTRY_X86_INLINE_HPP diff --git a/src/hotspot/cpu/x86/continuationFreezeThaw_x86.inline.hpp b/src/hotspot/cpu/x86/continuationFreezeThaw_x86.inline.hpp new file mode 100644 index 00000000000..06e2e6fc0fb --- /dev/null +++ b/src/hotspot/cpu/x86/continuationFreezeThaw_x86.inline.hpp @@ -0,0 +1,289 @@ +/* + * Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_X86_CONTINUATIONFREEZETHAW_X86_INLINE_HPP +#define CPU_X86_CONTINUATIONFREEZETHAW_X86_INLINE_HPP + +#include "code/codeBlob.inline.hpp" +#include "oops/stackChunkOop.inline.hpp" +#include "runtime/frame.hpp" +#include "runtime/frame.inline.hpp" + +inline void patch_callee_link(const frame& f, intptr_t* fp) { + *ContinuationHelper::Frame::callee_link_address(f) = fp; +} + +inline void patch_callee_link_relative(const frame& f, intptr_t* fp) { + intptr_t* la = (intptr_t*)ContinuationHelper::Frame::callee_link_address(f); + intptr_t new_value = fp - la; + *la = new_value; +} + +////// Freeze + +// Fast path + +inline void FreezeBase::patch_stack_pd(intptr_t* frame_sp, intptr_t* heap_sp) { + // copy the spilled rbp from the heap to the stack + *(frame_sp - frame::sender_sp_offset) = *(heap_sp - frame::sender_sp_offset); +} + +// Slow path + +template +inline frame FreezeBase::sender(const frame& f) { + assert(FKind::is_instance(f), ""); + if (FKind::interpreted) { + return frame(f.sender_sp(), f.interpreter_frame_sender_sp(), f.link(), f.sender_pc()); + } + intptr_t** link_addr = link_address(f); + + intptr_t* sender_sp = (intptr_t*)(link_addr + frame::sender_sp_offset); // f.unextended_sp() + (fsize/wordSize); // + address sender_pc = (address) *(sender_sp-1); + assert(sender_sp != f.sp(), "must have changed"); + + int slot = 0; + CodeBlob* sender_cb = CodeCache::find_blob_and_oopmap(sender_pc, slot); + return sender_cb != nullptr + ? frame(sender_sp, sender_sp, *link_addr, sender_pc, sender_cb, + slot == -1 ? nullptr : sender_cb->oop_map_for_slot(slot, sender_pc), false) + : frame(sender_sp, sender_sp, *link_addr, sender_pc); +} + +template +frame FreezeBase::new_heap_frame(frame& f, frame& caller) { + assert(FKind::is_instance(f), ""); + assert(!caller.is_interpreted_frame() + || caller.unextended_sp() == (intptr_t*)caller.at(frame::interpreter_frame_last_sp_offset), ""); + + intptr_t *sp, *fp; // sp is really our unextended_sp + if (FKind::interpreted) { + assert((intptr_t*)f.at(frame::interpreter_frame_last_sp_offset) == nullptr + || f.unextended_sp() == (intptr_t*)f.at(frame::interpreter_frame_last_sp_offset), ""); + int locals = f.interpreter_frame_method()->max_locals(); + // If the caller.is_empty(), i.e. we're freezing into an empty chunk, then we set + // the chunk's argsize in finalize_freeze and make room for it above the unextended_sp + bool overlap_caller = caller.is_interpreted_frame() || caller.is_empty(); + fp = caller.unextended_sp() - (locals + frame::sender_sp_offset) + (overlap_caller ? ContinuationHelper::InterpretedFrame::stack_argsize(f) : 0); + sp = fp - (f.fp() - f.unextended_sp()); + assert(sp <= fp, ""); + assert(fp <= caller.unextended_sp(), ""); + caller.set_sp(fp + frame::sender_sp_offset); + + assert(_cont.tail()->is_in_chunk(sp), ""); + + frame hf(sp, sp, fp, f.pc(), nullptr, nullptr, true /* on_heap */); + *hf.addr_at(frame::interpreter_frame_locals_offset) = frame::sender_sp_offset + locals - 1; + return hf; + } else { + // We need to re-read fp out of the frame because it may be an oop and we might have + // had a safepoint in finalize_freeze, after constructing f. + fp = *(intptr_t**)(f.sp() - frame::sender_sp_offset); + + int fsize = FKind::size(f); + sp = caller.unextended_sp() - fsize; + if (caller.is_interpreted_frame()) { + // If the caller is interpreted, our stackargs are not supposed to overlap with it + // so we make more room by moving sp down by argsize + int argsize = FKind::stack_argsize(f); + sp -= argsize; + } + caller.set_sp(sp + fsize); + + assert(_cont.tail()->is_in_chunk(sp), ""); + + return frame(sp, sp, fp, f.pc(), nullptr, nullptr, true /* on_heap */); + } +} + +void FreezeBase::adjust_interpreted_frame_unextended_sp(frame& f) { + assert((f.at(frame::interpreter_frame_last_sp_offset) != 0) || (f.unextended_sp() == f.sp()), ""); + intptr_t* real_unextended_sp = (intptr_t*)f.at(frame::interpreter_frame_last_sp_offset); + if (real_unextended_sp != nullptr) { + f.set_unextended_sp(real_unextended_sp); // can be null at a safepoint + } +} + +static inline void relativize_one(intptr_t* const vfp, intptr_t* const hfp, int offset) { + assert(*(hfp + offset) == *(vfp + offset), ""); + intptr_t* addr = hfp + offset; + intptr_t value = *(intptr_t**)addr - vfp; + *addr = value; +} + +inline void FreezeBase::relativize_interpreted_frame_metadata(const frame& f, const frame& hf) { + intptr_t* vfp = f.fp(); + intptr_t* hfp = hf.fp(); + assert(hfp == hf.unextended_sp() + (f.fp() - f.unextended_sp()), ""); + assert((f.at(frame::interpreter_frame_last_sp_offset) != 0) + || (f.unextended_sp() == f.sp()), ""); + assert(f.fp() > (intptr_t*)f.at(frame::interpreter_frame_initial_sp_offset), ""); + + // We compute the locals as below rather than relativize the value in the frame because then we can use the same + // code on AArch64, which has an added complication (see this method in continuation_aarch64.inline.hpp) + + // at(frame::interpreter_frame_last_sp_offset) can be NULL at safepoint preempts + *hf.addr_at(frame::interpreter_frame_last_sp_offset) = hf.unextended_sp() - hf.fp(); + *hf.addr_at(frame::interpreter_frame_locals_offset) = frame::sender_sp_offset + f.interpreter_frame_method()->max_locals() - 1; + + relativize_one(vfp, hfp, frame::interpreter_frame_initial_sp_offset); // == block_top == block_bottom + + assert((hf.fp() - hf.unextended_sp()) == (f.fp() - f.unextended_sp()), ""); + assert(hf.unextended_sp() == (intptr_t*)hf.at(frame::interpreter_frame_last_sp_offset), ""); + assert(hf.unextended_sp() <= (intptr_t*)hf.at(frame::interpreter_frame_initial_sp_offset), ""); + assert(hf.fp() > (intptr_t*)hf.at(frame::interpreter_frame_initial_sp_offset), ""); + assert(hf.fp() <= (intptr_t*)hf.at(frame::interpreter_frame_locals_offset), ""); +} + +inline void FreezeBase::set_top_frame_metadata_pd(const frame& hf) { + stackChunkOop chunk = _cont.tail(); + assert(chunk->is_in_chunk(hf.sp() - 1), ""); + assert(chunk->is_in_chunk(hf.sp() - frame::sender_sp_offset), ""); + + address frame_pc = hf.pc(); + + *(hf.sp() - 1) = (intptr_t)hf.pc(); + + intptr_t* fp_addr = hf.sp() - frame::sender_sp_offset; + *fp_addr = hf.is_interpreted_frame() ? (intptr_t)(hf.fp() - fp_addr) + : (intptr_t)hf.fp(); + assert(frame_pc == ContinuationHelper::Frame::real_pc(hf), ""); +} + +inline void FreezeBase::patch_pd(frame& hf, const frame& caller) { + if (caller.is_interpreted_frame()) { + assert(!caller.is_empty(), ""); + patch_callee_link_relative(caller, caller.fp()); + } else { + // If we're the bottom-most frame frozen in this freeze, the caller might have stayed frozen in the chunk, + // and its oop-containing fp fixed. We've now just overwritten it, so we must patch it back to its value + // as read from the chunk. + patch_callee_link(caller, caller.fp()); + } +} + +//////// Thaw + +// Fast path + +inline void ThawBase::prefetch_chunk_pd(void* start, int size) { + size <<= LogBytesPerWord; + Prefetch::read(start, size); + Prefetch::read(start, size - 64); +} + +void ThawBase::patch_chunk_pd(intptr_t* sp) { + intptr_t* fp = _cont.entryFP(); + *(intptr_t**)(sp - frame::sender_sp_offset) = fp; +} + +// Slow path + +inline frame ThawBase::new_entry_frame() { + intptr_t* sp = _cont.entrySP(); + return frame(sp, sp, _cont.entryFP(), _cont.entryPC()); // TODO PERF: This finds code blob and computes deopt state +} + +template frame ThawBase::new_stack_frame(const frame& hf, frame& caller, bool bottom) { + assert(FKind::is_instance(hf), ""); + // The values in the returned frame object will be written into the callee's stack in patch. + + if (FKind::interpreted) { + intptr_t* heap_sp = hf.unextended_sp(); + const int fsize = ContinuationHelper::InterpretedFrame::frame_bottom(hf) - hf.unextended_sp(); + const int locals = hf.interpreter_frame_method()->max_locals(); + intptr_t* frame_sp = caller.unextended_sp() - fsize; + intptr_t* fp = frame_sp + (hf.fp() - heap_sp); + DEBUG_ONLY(intptr_t* unextended_sp = fp + *hf.addr_at(frame::interpreter_frame_last_sp_offset);) + assert(frame_sp == unextended_sp, ""); + caller.set_sp(fp + frame::sender_sp_offset); + frame f(frame_sp, frame_sp, fp, hf.pc()); + // it's set again later in set_interpreter_frame_bottom, but we need to set the locals now so that + // we could call ContinuationHelper::InterpretedFrame::frame_bottom + intptr_t offset = *hf.addr_at(frame::interpreter_frame_locals_offset); + assert((int)offset == frame::sender_sp_offset + locals - 1, ""); + // derelativize locals + *(intptr_t**)f.addr_at(frame::interpreter_frame_locals_offset) = fp + offset; + return f; + } else { + int fsize = FKind::size(hf); + intptr_t* frame_sp = caller.unextended_sp() - fsize; + if (bottom || caller.is_interpreted_frame()) { + int argsize = hf.compiled_frame_stack_argsize(); + + fsize += argsize; + frame_sp -= argsize; + caller.set_sp(caller.sp() - argsize); + assert(caller.sp() == frame_sp + (fsize-argsize), ""); + + frame_sp = align(hf, frame_sp, caller, bottom); + } + + assert(hf.cb() != nullptr, ""); + assert(hf.oop_map() != nullptr, ""); + intptr_t* fp; + if (PreserveFramePointer) { + // we need to recreate a "real" frame pointer, pointing into the stack + fp = frame_sp + FKind::size(hf) - frame::sender_sp_offset; + } else { + // we need to re-read fp because it may be an oop and we might have fixed the frame. + fp = *(intptr_t**)(hf.sp() - frame::sender_sp_offset); + } + return frame(frame_sp, frame_sp, fp, hf.pc(), hf.cb(), hf.oop_map(), false); // TODO PERF : this computes deopt state; is it necessary? + } +} + +inline intptr_t* ThawBase::align(const frame& hf, intptr_t* frame_sp, frame& caller, bool bottom) { +#ifdef _LP64 + if (((intptr_t)frame_sp & 0xf) != 0) { + assert(caller.is_interpreted_frame() || (bottom && hf.compiled_frame_stack_argsize() % 2 != 0), ""); + frame_sp--; + caller.set_sp(caller.sp() - 1); + } + assert(is_aligned(frame_sp, frame::frame_alignment), ""); +#endif + + return frame_sp; +} + +inline void ThawBase::patch_pd(frame& f, const frame& caller) { + patch_callee_link(caller, caller.fp()); +} + +static inline void derelativize_one(intptr_t* const fp, int offset) { + intptr_t* addr = fp + offset; + *addr = (intptr_t)(fp + *addr); +} + +inline void ThawBase::derelativize_interpreted_frame_metadata(const frame& hf, const frame& f) { + intptr_t* vfp = f.fp(); + + derelativize_one(vfp, frame::interpreter_frame_last_sp_offset); + derelativize_one(vfp, frame::interpreter_frame_initial_sp_offset); +} + +inline void ThawBase::set_interpreter_frame_bottom(const frame& f, intptr_t* bottom) { + *(intptr_t**)f.addr_at(frame::interpreter_frame_locals_offset) = bottom - 1; +} +#endif // CPU_X86_CONTINUATIONFREEZE_THAW_X86_INLINE_HPP diff --git a/src/hotspot/cpu/x86/continuationHelper_x86.inline.hpp b/src/hotspot/cpu/x86/continuationHelper_x86.inline.hpp new file mode 100644 index 00000000000..15c031b95e2 --- /dev/null +++ b/src/hotspot/cpu/x86/continuationHelper_x86.inline.hpp @@ -0,0 +1,139 @@ +/* + * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_X86_CONTINUATIONHELPER_X86_INLINE_HPP +#define CPU_X86_CONTINUATIONHELPER_X86_INLINE_HPP + +#include "runtime/continuationHelper.hpp" + +#include "runtime/continuationEntry.inline.hpp" +#include "runtime/frame.inline.hpp" +#include "runtime/registerMap.hpp" +#include "utilities/macros.hpp" + +template +static inline intptr_t** link_address(const frame& f) { + assert(FKind::is_instance(f), ""); + return FKind::interpreted + ? (intptr_t**)(f.fp() + frame::link_offset) + : (intptr_t**)(f.unextended_sp() + f.cb()->frame_size() - frame::sender_sp_offset); +} + +inline int ContinuationHelper::frame_align_words(int size) { +#ifdef _LP64 + return size & 1; +#else + return 0; +#endif +} + +inline intptr_t* ContinuationHelper::frame_align_pointer(intptr_t* sp) { +#ifdef _LP64 + sp = align_down(sp, frame::frame_alignment); +#endif + return sp; +} + +template +inline void ContinuationHelper::update_register_map(const frame& f, RegisterMap* map) { + frame::update_map_with_saved_link(map, link_address(f)); +} + +inline void ContinuationHelper::update_register_map_with_callee(const frame& f, RegisterMap* map) { + frame::update_map_with_saved_link(map, ContinuationHelper::Frame::callee_link_address(f)); +} + +inline void ContinuationHelper::push_pd(const frame& f) { + *(intptr_t**)(f.sp() - frame::sender_sp_offset) = f.fp(); +} + +inline void ContinuationHelper::set_anchor_to_entry_pd(JavaFrameAnchor* anchor, ContinuationEntry* entry) { + anchor->set_last_Java_fp(entry->entry_fp()); +} + +#ifdef ASSERT +inline void ContinuationHelper::set_anchor_pd(JavaFrameAnchor* anchor, intptr_t* sp) { + intptr_t* fp = *(intptr_t**)(sp - frame::sender_sp_offset); + anchor->set_last_Java_fp(fp); +} + +inline bool ContinuationHelper::Frame::assert_frame_laid_out(frame f) { + intptr_t* sp = f.sp(); + address pc = *(address*)(sp - frame::sender_sp_ret_address_offset()); + intptr_t* fp = *(intptr_t**)(sp - frame::sender_sp_offset); + assert(f.raw_pc() == pc, "f.ra_pc: " INTPTR_FORMAT " actual: " INTPTR_FORMAT, p2i(f.raw_pc()), p2i(pc)); + assert(f.fp() == fp, "f.fp: " INTPTR_FORMAT " actual: " INTPTR_FORMAT, p2i(f.fp()), p2i(fp)); + return f.raw_pc() == pc && f.fp() == fp; +} +#endif + +inline intptr_t** ContinuationHelper::Frame::callee_link_address(const frame& f) { + return (intptr_t**)(f.sp() - frame::sender_sp_offset); +} + +inline address* ContinuationHelper::Frame::return_pc_address(const frame& f) { + return (address*)(f.real_fp() - 1); +} + +inline address* ContinuationHelper::InterpretedFrame::return_pc_address(const frame& f) { + return (address*)(f.fp() + frame::return_addr_offset); +} + +inline void ContinuationHelper::InterpretedFrame::patch_sender_sp(frame& f, intptr_t* sp) { + assert(f.is_interpreted_frame(), ""); + intptr_t* la = f.addr_at(frame::interpreter_frame_sender_sp_offset); + *la = f.is_heap_frame() ? (intptr_t)(sp - f.fp()) : (intptr_t)sp; +} + +inline address ContinuationHelper::Frame::real_pc(const frame& f) { + address* pc_addr = &(((address*) f.sp())[-1]); + return *pc_addr; +} + +inline void ContinuationHelper::Frame::patch_pc(const frame& f, address pc) { + address* pc_addr = &(((address*) f.sp())[-1]); + *pc_addr = pc; +} + +inline intptr_t* ContinuationHelper::InterpretedFrame::frame_top(const frame& f, InterpreterOopMap* mask) { // inclusive; this will be copied with the frame + // interpreter_frame_last_sp_offset, points to unextended_sp includes arguments in the frame + // interpreter_frame_initial_sp_offset excludes expression stack slots + int expression_stack_sz = expression_stack_size(f, mask); + intptr_t* res = *(intptr_t**)f.addr_at(frame::interpreter_frame_initial_sp_offset) - expression_stack_sz; + assert(res == (intptr_t*)f.interpreter_frame_monitor_end() - expression_stack_sz, ""); + assert(res >= f.unextended_sp(), + "res: " INTPTR_FORMAT " initial_sp: " INTPTR_FORMAT " last_sp: " INTPTR_FORMAT " unextended_sp: " INTPTR_FORMAT " expression_stack_size: %d", + p2i(res), p2i(f.addr_at(frame::interpreter_frame_initial_sp_offset)), f.at(frame::interpreter_frame_last_sp_offset), p2i(f.unextended_sp()), expression_stack_sz); + return res; +} + +inline intptr_t* ContinuationHelper::InterpretedFrame::frame_bottom(const frame& f) { // exclusive; this will not be copied with the frame + return (intptr_t*)f.at(frame::interpreter_frame_locals_offset) + 1; // exclusive, so we add 1 word +} + +inline intptr_t* ContinuationHelper::InterpretedFrame::frame_top(const frame& f, int callee_argsize, bool callee_interpreted) { + return f.unextended_sp() + (callee_interpreted ? callee_argsize : 0); +} + +#endif // CPU_X86_CONTINUATIONHELPER_X86_INLINE_HPP diff --git a/src/hotspot/cpu/x86/frame_x86.cpp b/src/hotspot/cpu/x86/frame_x86.cpp index f1cf5ed0bca..66ab0bbd342 100644 --- a/src/hotspot/cpu/x86/frame_x86.cpp +++ b/src/hotspot/cpu/x86/frame_x86.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -31,6 +31,7 @@ #include "oops/method.hpp" #include "oops/oop.inline.hpp" #include "prims/methodHandles.hpp" +#include "runtime/continuation.hpp" #include "runtime/frame.inline.hpp" #include "runtime/handles.inline.hpp" #include "runtime/javaCalls.hpp" @@ -54,6 +55,9 @@ void RegisterMap::check_location_valid() { // Profiling/safepoint support bool frame::safe_for_sender(JavaThread *thread) { + if (is_heap_frame()) { + return true; + } address sp = (address)_sp; address fp = (address)_fp; address unextended_sp = (address)_unextended_sp; @@ -146,6 +150,12 @@ bool frame::safe_for_sender(JavaThread *thread) { saved_fp = (intptr_t*) *(sender_sp - frame::sender_sp_offset); } + if (Continuation::is_return_barrier_entry(sender_pc)) { + // If our sender_pc is the return barrier, then our "real" sender is the continuation entry + frame s = Continuation::continuation_bottom_sender(thread, *this, sender_sp); + sender_sp = s.sp(); + sender_pc = s.pc(); + } // If the potential sender is the interpreter then we can do some more checking if (Interpreter::contains(sender_pc)) { @@ -261,32 +271,39 @@ bool frame::safe_for_sender(JavaThread *thread) { void frame::patch_pc(Thread* thread, address pc) { assert(_cb == CodeCache::find_blob(pc), "unexpected pc"); address* pc_addr = &(((address*) sp())[-1]); + if (TracePcPatching) { tty->print_cr("patch_pc at address " INTPTR_FORMAT " [" INTPTR_FORMAT " -> " INTPTR_FORMAT "]", p2i(pc_addr), p2i(*pc_addr), p2i(pc)); } // Either the return address is the original one or we are going to // patch in the same address that's already there. - assert(_pc == *pc_addr || pc == *pc_addr, "must be"); + + assert(!Continuation::is_return_barrier_entry(*pc_addr), "return barrier"); + + assert(_pc == *pc_addr || pc == *pc_addr || *pc_addr == 0, ""); + DEBUG_ONLY(address old_pc = _pc;) *pc_addr = pc; + _pc = pc; // must be set before call to get_deopt_original_pc address original_pc = CompiledMethod::get_deopt_original_pc(this); if (original_pc != NULL) { - assert(original_pc == _pc, "expected original PC to be stored before patching"); + assert(original_pc == old_pc, "expected original PC to be stored before patching"); _deopt_state = is_deoptimized; - // leave _pc as is + _pc = original_pc; } else { _deopt_state = not_deoptimized; - _pc = pc; } -} + assert(!is_compiled_frame() || !_cb->as_compiled_method()->is_deopt_entry(_pc), "must be"); -bool frame::is_interpreted_frame() const { - return Interpreter::contains(pc()); -} - -int frame::frame_size(RegisterMap* map) const { - frame sender = this->sender(map); - return sender.sp() - sp(); +#ifdef ASSERT + { + frame f(this->sp(), this->unextended_sp(), this->fp(), pc); + assert(f.is_deoptimized_frame() == this->is_deoptimized_frame() && f.pc() == this->pc() && f.raw_pc() == this->raw_pc(), + "must be (f.is_deoptimized_frame(): %d this->is_deoptimized_frame(): %d " + "f.pc(): " INTPTR_FORMAT " this->pc(): " INTPTR_FORMAT " f.raw_pc(): " INTPTR_FORMAT " this->raw_pc(): " INTPTR_FORMAT ")", + f.is_deoptimized_frame(), this->is_deoptimized_frame(), p2i(f.pc()), p2i(this->pc()), p2i(f.raw_pc()), p2i(this->raw_pc())); + } +#endif } intptr_t* frame::entry_frame_argument_at(int offset) const { @@ -316,10 +333,10 @@ BasicObjectLock* frame::interpreter_frame_monitor_begin() const { } BasicObjectLock* frame::interpreter_frame_monitor_end() const { - BasicObjectLock* result = (BasicObjectLock*) *addr_at(interpreter_frame_monitor_block_top_offset); + BasicObjectLock* result = (BasicObjectLock*) at(interpreter_frame_monitor_block_top_offset); // make sure the pointer points inside the frame assert(sp() <= (intptr_t*) result, "monitor end should be above the stack pointer"); - assert((intptr_t*) result < fp(), "monitor end should be strictly below the frame pointer"); + assert((intptr_t*) result < fp(), "monitor end should be strictly below the frame pointer: result: " INTPTR_FORMAT " fp: " INTPTR_FORMAT, p2i(result), p2i(fp())); return result; } @@ -397,7 +414,7 @@ void frame::verify_deopt_original_pc(CompiledMethod* nm, intptr_t* unextended_sp address original_pc = nm->get_original_pc(&fr); assert(nm->insts_contains_inclusive(original_pc), - "original PC must be in the main code section of the the compiled method (or must be immediately following it)"); + "original PC must be in the main code section of the the compiled method (or must be immediately following it) original_pc: " INTPTR_FORMAT " unextended_sp: " INTPTR_FORMAT " name: %s", p2i(original_pc), p2i(unextended_sp), nm->name()); } #endif @@ -422,30 +439,6 @@ void frame::adjust_unextended_sp() { } #endif -//------------------------------------------------------------------------------ -// frame::update_map_with_saved_link -void frame::update_map_with_saved_link(RegisterMap* map, intptr_t** link_addr) { - // The interpreter and compiler(s) always save EBP/RBP in a known - // location on entry. We must record where that location is - // so this if EBP/RBP was live on callout from c2 we can find - // the saved copy no matter what it called. - - // Since the interpreter always saves EBP/RBP if we record where it is then - // we don't have to always save EBP/RBP on entry and exit to c2 compiled - // code, on entry will be enough. - map->set_location(rbp->as_VMReg(), (address) link_addr); -#ifdef AMD64 - // this is weird "H" ought to be at a higher address however the - // oopMaps seems to have the "H" regs at the same address and the - // vanilla register. - // XXXX make this go away - if (true) { - map->set_location(rbp->as_VMReg()->next(), (address) link_addr); - } -#endif // AMD64 -} - - //------------------------------------------------------------------------------ // frame::sender_for_interpreter_frame frame frame::sender_for_interpreter_frame(RegisterMap* map) const { @@ -455,6 +448,7 @@ frame frame::sender_for_interpreter_frame(RegisterMap* map) const { // This is the sp before any possible extension (adapter/locals). intptr_t* unextended_sp = interpreter_frame_sender_sp(); + intptr_t* sender_fp = link(); #if COMPILER2_OR_JVMCI if (map->update_map()) { @@ -462,75 +456,17 @@ frame frame::sender_for_interpreter_frame(RegisterMap* map) const { } #endif // COMPILER2_OR_JVMCI - return frame(sender_sp, unextended_sp, link(), sender_pc()); -} + address sender_pc = this->sender_pc(); - -//------------------------------------------------------------------------------ -// frame::sender_for_compiled_frame -frame frame::sender_for_compiled_frame(RegisterMap* map) const { - assert(map != NULL, "map must be set"); - - // frame owned by optimizing compiler - assert(_cb->frame_size() >= 0, "must have non-zero frame size"); - intptr_t* sender_sp = unextended_sp() + _cb->frame_size(); - intptr_t* unextended_sp = sender_sp; - - // On Intel the return_address is always the word on the stack - address sender_pc = (address) *(sender_sp-1); - - // This is the saved value of EBP which may or may not really be an FP. - // It is only an FP if the sender is an interpreter frame (or C1?). - intptr_t** saved_fp_addr = (intptr_t**) (sender_sp - frame::sender_sp_offset); - - if (map->update_map()) { - // Tell GC to use argument oopmaps for some runtime stubs that need it. - // For C1, the runtime stub might not have oop maps, so set this flag - // outside of update_register_map. - map->set_include_argument_oops(_cb->caller_must_gc_arguments(map->thread())); - if (_cb->oop_maps() != NULL) { - OopMapSet::update_register_map(this, map); + if (Continuation::is_return_barrier_entry(sender_pc)) { + if (map->walk_cont()) { // about to walk into an h-stack + return Continuation::top_frame(*this, map); + } else { + return Continuation::continuation_bottom_sender(map->thread(), *this, sender_sp); } - - // Since the prolog does the save and restore of EBP there is no oopmap - // for it so we must fill in its location as if there was an oopmap entry - // since if our caller was compiled code there could be live jvm state in it. - update_map_with_saved_link(map, saved_fp_addr); } - assert(sender_sp != sp(), "must have changed"); - return frame(sender_sp, unextended_sp, *saved_fp_addr, sender_pc); -} - - -//------------------------------------------------------------------------------ -// frame::sender_raw -frame frame::sender_raw(RegisterMap* map) const { - // Default is we done have to follow them. The sender_for_xxx will - // update it accordingly - map->set_include_argument_oops(false); - - if (is_entry_frame()) return sender_for_entry_frame(map); - if (is_optimized_entry_frame()) return sender_for_optimized_entry_frame(map); - if (is_interpreted_frame()) return sender_for_interpreter_frame(map); - assert(_cb == CodeCache::find_blob(pc()),"Must be the same"); - - if (_cb != NULL) { - return sender_for_compiled_frame(map); - } - // Must be native-compiled frame, i.e. the marshaling code for native - // methods that exists in the core system. - return frame(sender_sp(), link(), sender_pc()); -} - -frame frame::sender(RegisterMap* map) const { - frame result = sender_raw(map); - - if (map->process_frames()) { - StackWatermarkSet::on_iteration(map->thread(), result); - } - - return result; + return frame(sender_sp, unextended_sp, sender_fp, sender_pc); } bool frame::is_interpreted_frame_valid(JavaThread* thread) const { @@ -651,7 +587,6 @@ BasicType frame::interpreter_frame_result(oop* oop_result, jvalue* value_result) return type; } - intptr_t* frame::interpreter_frame_tos_at(jint offset) const { int index = (Interpreter::expr_offset_in_bytes(offset)/wordSize); return &interpreter_frame_tos_address()[index]; @@ -660,7 +595,7 @@ intptr_t* frame::interpreter_frame_tos_at(jint offset) const { #ifndef PRODUCT #define DESCRIBE_FP_OFFSET(name) \ - values.describe(frame_no, fp() + frame::name##_offset, #name) + values.describe(frame_no, fp() + frame::name##_offset, #name, 1) void frame::describe_pd(FrameValues& values, int frame_no) { if (is_interpreted_frame()) { @@ -683,7 +618,16 @@ void frame::describe_pd(FrameValues& values, int frame_no) { } #endif // AMD64 } + + intptr_t* ret_pc_loc = sp() - return_addr_offset; + address ret_pc = *(address*)ret_pc_loc; + if (Continuation::is_return_barrier_entry(ret_pc)) + values.describe(frame_no, ret_pc_loc, "return address (return barrier)"); + else + values.describe(frame_no, ret_pc_loc, err_msg("return address for #%d", frame_no)); + values.describe(frame_no, sp() - sender_sp_offset, err_msg("saved fp for #%d", frame_no), 0); } + #endif // !PRODUCT intptr_t *frame::initial_deoptimization_info() { @@ -691,19 +635,6 @@ intptr_t *frame::initial_deoptimization_info() { return fp(); } -intptr_t* frame::real_fp() const { - if (_cb != NULL) { - // use the frame size if valid - int size = _cb->frame_size(); - if (size > 0) { - return unextended_sp() + size; - } - } - // else rely on fp() - assert(! is_compiled_frame(), "unknown compiled frame size"); - return fp(); -} - #ifndef PRODUCT // This is a generic constructor which is only used by pns() in debug.cpp. frame::frame(void* sp, void* fp, void* pc) { diff --git a/src/hotspot/cpu/x86/frame_x86.hpp b/src/hotspot/cpu/x86/frame_x86.hpp index 26dbb2aa956..1193c6a5231 100644 --- a/src/hotspot/cpu/x86/frame_x86.hpp +++ b/src/hotspot/cpu/x86/frame_x86.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -86,16 +86,23 @@ entry_frame_after_call_words = 60, entry_frame_call_wrapper_offset = 2, - arg_reg_save_area_bytes = 32 // Register argument save area + arg_reg_save_area_bytes = 32, // Register argument save area #else entry_frame_after_call_words = 13, entry_frame_call_wrapper_offset = -6, - arg_reg_save_area_bytes = 0 + arg_reg_save_area_bytes = 0, #endif // _WIN64 #else - entry_frame_call_wrapper_offset = 2 + entry_frame_call_wrapper_offset = 2, #endif // AMD64 + + // size, in words, of frame metadata (e.g. pc and link) + metadata_words = sender_sp_offset, + // compiled frame alignment, in bytes + frame_alignment = 16, + // size, in words, of maximum shift in frame position due to alignment + align_wiggle = 1 }; intptr_t ptr_at(int offset) const { @@ -108,7 +115,10 @@ private: // an additional field beyond _sp and _pc: - intptr_t* _fp; // frame pointer + union { + intptr_t* _fp; // frame pointer + int _offset_fp; // relative frame pointer for use in stack-chunk frames + }; // The interpreter and adapters will extend the frame of the caller. // Since oopMaps are based on the sp of the caller before extension // we need to know that value. However in order to compute the address @@ -116,7 +126,11 @@ // use sp() to mean "raw" sp and unextended_sp() to mean the caller's // original sp. - intptr_t* _unextended_sp; + union { + intptr_t* _unextended_sp; + int _offset_unextended_sp; // for use in stack-chunk frames + }; + void adjust_unextended_sp() NOT_DEBUG_RETURN; intptr_t* ptr_at_addr(int offset) const { @@ -128,6 +142,8 @@ static void verify_deopt_original_pc(CompiledMethod* nm, intptr_t* unextended_sp); #endif + const ImmutableOopMap* get_oop_map() const; + public: // Constructors @@ -135,21 +151,29 @@ frame(intptr_t* sp, intptr_t* unextended_sp, intptr_t* fp, address pc); + frame(intptr_t* sp, intptr_t* unextended_sp, intptr_t* fp, address pc, CodeBlob* cb); + // used for heap frame construction by continuations + frame(intptr_t* sp, intptr_t* unextended_sp, intptr_t* fp, address pc, CodeBlob* cb, const ImmutableOopMap* oop_map, bool relative); + frame(intptr_t* sp, intptr_t* fp); void init(intptr_t* sp, intptr_t* fp, address pc); + void setup(address pc); // accessors for the instance variables // Note: not necessarily the real 'frame pointer' (see real_fp) - intptr_t* fp() const { return _fp; } + intptr_t* fp() const { assert_absolute(); return _fp; } + void set_fp(intptr_t* newfp) { _fp = newfp; } + int offset_fp() const { assert_offset(); return _offset_fp; } + void set_offset_fp(int value) { assert_on_heap(); _offset_fp = value; } inline address* sender_pc_addr() const; // expression stack tos if we are nested in a java call intptr_t* interpreter_frame_last_sp() const; - // helper to update a map with callee-saved RBP - static void update_map_with_saved_link(RegisterMap* map, intptr_t** link_addr); + template + static void update_map_with_saved_link(RegisterMapT* map, intptr_t** link_addr); // deoptimization support void interpreter_frame_set_last_sp(intptr_t* sp); @@ -157,6 +181,6 @@ static jint interpreter_frame_expression_stack_direction() { return -1; } // returns the sending frame, without applying any barriers - frame sender_raw(RegisterMap* map) const; + inline frame sender_raw(RegisterMap* map) const; #endif // CPU_X86_FRAME_X86_HPP diff --git a/src/hotspot/cpu/x86/frame_x86.inline.hpp b/src/hotspot/cpu/x86/frame_x86.inline.hpp index 23072238e16..cc36af29152 100644 --- a/src/hotspot/cpu/x86/frame_x86.inline.hpp +++ b/src/hotspot/cpu/x86/frame_x86.inline.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -25,8 +25,13 @@ #ifndef CPU_X86_FRAME_X86_INLINE_HPP #define CPU_X86_FRAME_X86_INLINE_HPP -#include "code/codeCache.hpp" +#include "code/codeBlob.inline.hpp" +#include "code/codeCache.inline.hpp" #include "code/vmreg.inline.hpp" +#include "compiler/oopMap.inline.hpp" +#include "interpreter/interpreter.hpp" +#include "interpreter/oopMapCache.hpp" +#include "runtime/sharedRuntime.hpp" #include "runtime/registerMap.hpp" // Inline functions for Intel frames: @@ -40,6 +45,9 @@ inline frame::frame() { _fp = NULL; _cb = NULL; _deopt_state = unknown; + _oop_map = NULL; + _on_heap = false; + DEBUG_ONLY(_frame_index = -1;) } inline void frame::init(intptr_t* sp, intptr_t* fp, address pc) { @@ -47,16 +55,30 @@ inline void frame::init(intptr_t* sp, intptr_t* fp, address pc) { _unextended_sp = sp; _fp = fp; _pc = pc; + _oop_map = NULL; + _on_heap = false; + DEBUG_ONLY(_frame_index = -1;) + assert(pc != NULL, "no pc?"); - _cb = CodeCache::find_blob(pc); + _cb = CodeCache::find_blob(pc); // not fast because this constructor can be used on native frames + setup(pc); +} + +inline void frame::setup(address pc) { adjust_unextended_sp(); address original_pc = CompiledMethod::get_deopt_original_pc(this); if (original_pc != NULL) { _pc = original_pc; _deopt_state = is_deoptimized; + assert(_cb == NULL || _cb->as_compiled_method()->insts_contains_inclusive(_pc), + "original PC must be in the main code section of the the compiled method (or must be immediately following it)"); } else { - _deopt_state = not_deoptimized; + if (_cb == SharedRuntime::deopt_blob()) { + _deopt_state = is_deoptimized; + } else { + _deopt_state = not_deoptimized; + } } } @@ -64,35 +86,71 @@ inline frame::frame(intptr_t* sp, intptr_t* fp, address pc) { init(sp, fp, pc); } +inline frame::frame(intptr_t* sp, intptr_t* unextended_sp, intptr_t* fp, address pc, CodeBlob* cb) { + _sp = sp; + _unextended_sp = unextended_sp; + _fp = fp; + _pc = pc; + assert(pc != NULL, "no pc?"); + _cb = cb; + _oop_map = NULL; + assert(_cb != NULL, "pc: " INTPTR_FORMAT, p2i(pc)); + _on_heap = false; + DEBUG_ONLY(_frame_index = -1;) + + setup(pc); +} + +inline frame::frame(intptr_t* sp, intptr_t* unextended_sp, intptr_t* fp, address pc, CodeBlob* cb, + const ImmutableOopMap* oop_map, bool on_heap) { + _sp = sp; + _unextended_sp = unextended_sp; + _fp = fp; + _pc = pc; + _cb = cb; + _oop_map = oop_map; + _deopt_state = not_deoptimized; + _on_heap = on_heap; + DEBUG_ONLY(_frame_index = -1;) + + // In thaw, non-heap frames use this constructor to pass oop_map. I don't know why. + assert(_on_heap || _cb != nullptr, "these frames are always heap frames"); + if (cb != NULL) { + setup(pc); + } +#ifdef ASSERT + // The following assertion has been disabled because it would sometime trap for Continuation.run, + // which is not *in* a continuation and therefore does not clear the _cont_fastpath flag, but this + // is benign even in fast mode (see Freeze::setup_jump) + // We might freeze deoptimized frame in slow mode + // assert(_pc == pc && _deopt_state == not_deoptimized, ""); +#endif +} + inline frame::frame(intptr_t* sp, intptr_t* unextended_sp, intptr_t* fp, address pc) { _sp = sp; _unextended_sp = unextended_sp; _fp = fp; _pc = pc; assert(pc != NULL, "no pc?"); - _cb = CodeCache::find_blob(pc); - adjust_unextended_sp(); + _cb = CodeCache::find_blob_fast(pc); + _oop_map = NULL; + assert(_cb != NULL, "pc: " INTPTR_FORMAT " sp: " INTPTR_FORMAT " unextended_sp: " INTPTR_FORMAT " fp: " INTPTR_FORMAT, p2i(pc), p2i(sp), p2i(unextended_sp), p2i(fp)); + _on_heap = false; + DEBUG_ONLY(_frame_index = -1;) - address original_pc = CompiledMethod::get_deopt_original_pc(this); - if (original_pc != NULL) { - _pc = original_pc; - assert(_cb->as_compiled_method()->insts_contains_inclusive(_pc), - "original PC must be in the main code section of the the compiled method (or must be immediately following it)"); - _deopt_state = is_deoptimized; - } else { - if (_cb->is_deoptimization_stub()) { - _deopt_state = is_deoptimized; - } else { - _deopt_state = not_deoptimized; - } - } + setup(pc); } +inline frame::frame(intptr_t* sp) : frame(sp, sp, *(intptr_t**)(sp - frame::sender_sp_offset), *(address*)(sp - 1)) {} + inline frame::frame(intptr_t* sp, intptr_t* fp) { _sp = sp; _unextended_sp = sp; _fp = fp; _pc = (address)(sp[-1]); + _on_heap = false; + DEBUG_ONLY(_frame_index = -1;) // Here's a sticky one. This constructor can be called via AsyncGetCallTrace // when last_Java_sp is non-null but the pc fetched is junk. If we are truly @@ -116,6 +174,7 @@ inline frame::frame(intptr_t* sp, intptr_t* fp) { } else { _deopt_state = not_deoptimized; } + _oop_map = NULL; } // Accessors @@ -138,35 +197,68 @@ inline intptr_t* frame::id(void) const { return unextended_sp(); } inline bool frame::is_older(intptr_t* id) const { assert(this->id() != NULL && id != NULL, "NULL frame id"); return this->id() > id ; } -inline intptr_t* frame::link() const { return (intptr_t*) *(intptr_t **)addr_at(link_offset); } +inline intptr_t* frame::link() const { return *(intptr_t **)addr_at(link_offset); } inline intptr_t* frame::link_or_null() const { intptr_t** ptr = (intptr_t **)addr_at(link_offset); return os::is_readable_pointer(ptr) ? *ptr : NULL; } -inline intptr_t* frame::unextended_sp() const { return _unextended_sp; } +inline intptr_t* frame::unextended_sp() const { assert_absolute(); return _unextended_sp; } +inline void frame::set_unextended_sp(intptr_t* value) { _unextended_sp = value; } +inline int frame::offset_unextended_sp() const { assert_offset(); return _offset_unextended_sp; } +inline void frame::set_offset_unextended_sp(int value) { assert_on_heap(); _offset_unextended_sp = value; } + +inline intptr_t* frame::real_fp() const { + if (_cb != NULL) { + // use the frame size if valid + int size = _cb->frame_size(); + if (size > 0) { + return unextended_sp() + size; + } + } + // else rely on fp() + assert(! is_compiled_frame(), "unknown compiled frame size"); + return fp(); +} + +inline int frame::frame_size() const { + return is_interpreted_frame() + ? sender_sp() - sp() + : cb()->frame_size(); +} + +inline int frame::compiled_frame_stack_argsize() const { + assert(cb()->is_compiled(), ""); + return (cb()->as_compiled_method()->method()->num_stack_arg_slots() * VMRegImpl::stack_slot_size) >> LogBytesPerWord; +} + +inline void frame::interpreted_frame_oop_map(InterpreterOopMap* mask) const { + assert(mask != NULL, ""); + Method* m = interpreter_frame_method(); + int bci = interpreter_frame_bci(); + m->mask_for(bci, mask); // OopMapCache::compute_one_oop_map(m, bci, mask); +} // Return address: -inline address* frame::sender_pc_addr() const { return (address*) addr_at( return_addr_offset); } +inline address* frame::sender_pc_addr() const { return (address*) addr_at(return_addr_offset); } inline address frame::sender_pc() const { return *sender_pc_addr(); } -inline intptr_t* frame::sender_sp() const { return addr_at( sender_sp_offset); } +inline intptr_t* frame::sender_sp() const { return addr_at(sender_sp_offset); } inline intptr_t** frame::interpreter_frame_locals_addr() const { return (intptr_t**)addr_at(interpreter_frame_locals_offset); } inline intptr_t* frame::interpreter_frame_last_sp() const { - return *(intptr_t**)addr_at(interpreter_frame_last_sp_offset); + return (intptr_t*)at(interpreter_frame_last_sp_offset); } inline intptr_t* frame::interpreter_frame_bcp_addr() const { return (intptr_t*)addr_at(interpreter_frame_bcp_offset); } - inline intptr_t* frame::interpreter_frame_mdp_addr() const { return (intptr_t*)addr_at(interpreter_frame_mdp_offset); } @@ -235,18 +327,145 @@ inline JavaCallWrapper** frame::entry_frame_call_wrapper_addr() const { PRAGMA_DIAG_PUSH PRAGMA_NONNULL_IGNORED inline oop frame::saved_oop_result(RegisterMap* map) const { - oop* result_adr = (oop *)map->location(rax->as_VMReg()); + oop* result_adr = (oop *)map->location(rax->as_VMReg(), sp()); guarantee(result_adr != NULL, "bad register save location"); - - return (*result_adr); + return *result_adr; } inline void frame::set_saved_oop_result(RegisterMap* map, oop obj) { - oop* result_adr = (oop *)map->location(rax->as_VMReg()); + oop* result_adr = (oop *)map->location(rax->as_VMReg(), sp()); guarantee(result_adr != NULL, "bad register save location"); *result_adr = obj; } PRAGMA_DIAG_POP +inline bool frame::is_interpreted_frame() const { + return Interpreter::contains(pc()); +} + +inline int frame::sender_sp_ret_address_offset() { + return frame::sender_sp_offset - frame::return_addr_offset; +} + +inline const ImmutableOopMap* frame::get_oop_map() const { + if (_cb == NULL) return NULL; + if (_cb->oop_maps() != NULL) { + NativePostCallNop* nop = nativePostCallNop_at(_pc); + if (nop != NULL && nop->displacement() != 0) { + int slot = ((nop->displacement() >> 24) & 0xff); + return _cb->oop_map_for_slot(slot, _pc); + } + const ImmutableOopMap* oop_map = OopMapSet::find_map(this); + return oop_map; + } + return NULL; +} + +//------------------------------------------------------------------------------ +// frame::sender + +inline frame frame::sender(RegisterMap* map) const { + frame result = sender_raw(map); + + if (map->process_frames() && !map->in_cont()) { + StackWatermarkSet::on_iteration(map->thread(), result); + } + + return result; +} + +inline frame frame::sender_raw(RegisterMap* map) const { + // Default is we done have to follow them. The sender_for_xxx will + // update it accordingly + map->set_include_argument_oops(false); + + if (map->in_cont()) { // already in an h-stack + return map->stack_chunk()->sender(*this, map); + } + + if (is_entry_frame()) return sender_for_entry_frame(map); + if (is_optimized_entry_frame()) return sender_for_optimized_entry_frame(map); + if (is_interpreted_frame()) return sender_for_interpreter_frame(map); + + assert(_cb == CodeCache::find_blob(pc()), "Must be the same"); + if (_cb != NULL) return sender_for_compiled_frame(map); + + // Must be native-compiled frame, i.e. the marshaling code for native + // methods that exists in the core system. + return frame(sender_sp(), link(), sender_pc()); +} + +inline frame frame::sender_for_compiled_frame(RegisterMap* map) const { + assert(map != NULL, "map must be set"); + + // frame owned by optimizing compiler + assert(_cb->frame_size() >= 0, "must have non-zero frame size"); + intptr_t* sender_sp = unextended_sp() + _cb->frame_size(); + assert(sender_sp == real_fp(), ""); + + // On Intel the return_address is always the word on the stack + address sender_pc = (address) *(sender_sp-1); + + // This is the saved value of EBP which may or may not really be an FP. + // It is only an FP if the sender is an interpreter frame (or C1?). + // saved_fp_addr should be correct even for a bottom thawed frame (with a return barrier) + intptr_t** saved_fp_addr = (intptr_t**) (sender_sp - frame::sender_sp_offset); + + if (map->update_map()) { + // Tell GC to use argument oopmaps for some runtime stubs that need it. + // For C1, the runtime stub might not have oop maps, so set this flag + // outside of update_register_map. + if (!_cb->is_compiled()) { // compiled frames do not use callee-saved registers + map->set_include_argument_oops(_cb->caller_must_gc_arguments(map->thread())); + if (oop_map() != NULL) { + _oop_map->update_register_map(this, map); + } + } else { + assert(!_cb->caller_must_gc_arguments(map->thread()), ""); + assert(!map->include_argument_oops(), ""); + assert(oop_map() == NULL || !oop_map()->has_any(OopMapValue::callee_saved_value), "callee-saved value in compiled frame"); + } + + // Since the prolog does the save and restore of EBP there is no oopmap + // for it so we must fill in its location as if there was an oopmap entry + // since if our caller was compiled code there could be live jvm state in it. + update_map_with_saved_link(map, saved_fp_addr); + } + + assert(sender_sp != sp(), "must have changed"); + + if (Continuation::is_return_barrier_entry(sender_pc)) { + if (map->walk_cont()) { // about to walk into an h-stack + return Continuation::top_frame(*this, map); + } else { + return Continuation::continuation_bottom_sender(map->thread(), *this, sender_sp); + } + } + + intptr_t* unextended_sp = sender_sp; + return frame(sender_sp, unextended_sp, *saved_fp_addr, sender_pc); +} + +template +void frame::update_map_with_saved_link(RegisterMapT* map, intptr_t** link_addr) { + // The interpreter and compiler(s) always save EBP/RBP in a known + // location on entry. We must record where that location is + // so this if EBP/RBP was live on callout from c2 we can find + // the saved copy no matter what it called. + + // Since the interpreter always saves EBP/RBP if we record where it is then + // we don't have to always save EBP/RBP on entry and exit to c2 compiled + // code, on entry will be enough. + map->set_location(rbp->as_VMReg(), (address) link_addr); +#ifdef AMD64 + // this is weird "H" ought to be at a higher address however the + // oopMaps seems to have the "H" regs at the same address and the + // vanilla register. + // XXXX make this go away + if (true) { + map->set_location(rbp->as_VMReg()->next(), (address) link_addr); + } +#endif // AMD64 +} #endif // CPU_X86_FRAME_X86_INLINE_HPP diff --git a/src/hotspot/cpu/x86/gc/g1/g1BarrierSetAssembler_x86.cpp b/src/hotspot/cpu/x86/gc/g1/g1BarrierSetAssembler_x86.cpp index 475a92d0f43..17597863d5b 100644 --- a/src/hotspot/cpu/x86/gc/g1/g1BarrierSetAssembler_x86.cpp +++ b/src/hotspot/cpu/x86/gc/g1/g1BarrierSetAssembler_x86.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -125,8 +125,24 @@ void G1BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorator bool on_reference = on_weak || on_phantom; ModRefBarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread); if (on_oop && on_reference) { - const Register thread = NOT_LP64(tmp_thread) LP64_ONLY(r15_thread); - NOT_LP64(__ get_thread(thread)); + Register thread = NOT_LP64(tmp_thread) LP64_ONLY(r15_thread); + +#ifndef _LP64 + // Work around the x86_32 bug that only manifests with Loom for some reason. + // MacroAssembler::resolve_weak_handle calls this barrier with tmp_thread == noreg. + if (thread == noreg) { + if (dst != rcx && tmp1 != rcx) { + thread = rcx; + } else if (dst != rdx && tmp1 != rdx) { + thread = rdx; + } else if (dst != rdi && tmp1 != rdi) { + thread = rdi; + } + } + assert_different_registers(dst, tmp1, thread); + __ push(thread); + __ get_thread(thread); +#endif // Generate the G1 pre-barrier code to log the value of // the referent field in an SATB buffer. @@ -137,6 +153,10 @@ void G1BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorator tmp1 /* tmp */, true /* tosca_live */, true /* expand_call */); + +#ifndef _LP64 + __ pop(thread); +#endif } } @@ -206,13 +226,7 @@ void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm, __ bind(runtime); // Determine and save the live input values - RegSet saved; - if (tosca_live) saved += RegSet::of(rax); - if (obj != noreg && obj != rax) saved += RegSet::of(obj); - if (pre_val != rax) saved += RegSet::of(pre_val); - NOT_LP64( saved += RegSet::of(thread); ) - - __ push_set(saved); + __ push_call_clobbered_registers(); // Calling the runtime using the regular call_VM_leaf mechanism generates // code (generated by InterpreterMacroAssember::call_VM_leaf_base) @@ -243,7 +257,8 @@ void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm, } else { __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), pre_val, thread); } - __ pop_set(saved); + + __ pop_call_clobbered_registers(); __ bind(done); } @@ -329,7 +344,6 @@ void G1BarrierSetAssembler::oop_store_at(MacroAssembler* masm, DecoratorSet deco Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) { bool in_heap = (decorators & IN_HEAP) != 0; bool as_normal = (decorators & AS_NORMAL) != 0; - assert((decorators & IS_DEST_UNINITIALIZED) == 0, "unsupported"); bool needs_pre_barrier = as_normal; bool needs_post_barrier = val != noreg && in_heap; diff --git a/src/hotspot/cpu/x86/gc/shared/barrierSetNMethod_x86.cpp b/src/hotspot/cpu/x86/gc/shared/barrierSetNMethod_x86.cpp index 9da848f65bc..b1b381afe5f 100644 --- a/src/hotspot/cpu/x86/gc/shared/barrierSetNMethod_x86.cpp +++ b/src/hotspot/cpu/x86/gc/shared/barrierSetNMethod_x86.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -174,6 +174,15 @@ void BarrierSetNMethod::disarm(nmethod* nm) { cmp->set_immediate(disarmed_value()); } +void BarrierSetNMethod::arm(nmethod* nm, int arm_value) { + if (!supports_entry_barrier(nm)) { + return; + } + + NativeNMethodCmpBarrier* cmp = native_nmethod_barrier(nm); + cmp->set_immediate(arm_value); +} + bool BarrierSetNMethod::is_armed(nmethod* nm) { if (!supports_entry_barrier(nm)) { return false; diff --git a/src/hotspot/cpu/x86/globals_x86.hpp b/src/hotspot/cpu/x86/globals_x86.hpp index f17f62a915d..d8543762a8e 100644 --- a/src/hotspot/cpu/x86/globals_x86.hpp +++ b/src/hotspot/cpu/x86/globals_x86.hpp @@ -63,10 +63,10 @@ define_pd_global(intx, InlineSmallCode, 1000); // Java_java_net_SocketOutputStream_socketWrite0() uses a 64k buffer on the // stack if compiled for unix and LP64. To pass stack overflow tests we need // 20 shadow pages. -#define DEFAULT_STACK_SHADOW_PAGES (NOT_WIN64(20) WIN64_ONLY(7) DEBUG_ONLY(+2)) +#define DEFAULT_STACK_SHADOW_PAGES (NOT_WIN64(20) WIN64_ONLY(7) DEBUG_ONLY(+4)) // For those clients that do not use write socket, we allow // the min range value to be below that of the default -#define MIN_STACK_SHADOW_PAGES (NOT_WIN64(10) WIN64_ONLY(7) DEBUG_ONLY(+2)) +#define MIN_STACK_SHADOW_PAGES (NOT_WIN64(10) WIN64_ONLY(7) DEBUG_ONLY(+4)) #else #define DEFAULT_STACK_SHADOW_PAGES (4 DEBUG_ONLY(+5)) #define MIN_STACK_SHADOW_PAGES DEFAULT_STACK_SHADOW_PAGES diff --git a/src/hotspot/cpu/x86/interp_masm_x86.cpp b/src/hotspot/cpu/x86/interp_masm_x86.cpp index 5f7947ba930..95e77042656 100644 --- a/src/hotspot/cpu/x86/interp_masm_x86.cpp +++ b/src/hotspot/cpu/x86/interp_masm_x86.cpp @@ -1064,6 +1064,9 @@ void InterpreterMacroAssembler::remove_activation( bind(unlock); unlock_object(robj); + NOT_LP64(get_thread(rthread);) + dec_held_monitor_count(rthread); + pop(state); // Check that for block-structured locking (i.e., that all locked @@ -1108,6 +1111,8 @@ void InterpreterMacroAssembler::remove_activation( push(state); mov(robj, rmon); // nop if robj and rmon are the same unlock_object(robj); + NOT_LP64(get_thread(rthread);) + dec_held_monitor_count(rthread); pop(state); if (install_monitor_exception) { @@ -1168,6 +1173,7 @@ void InterpreterMacroAssembler::remove_activation( leave(); // remove frame anchor pop(ret_addr); // get return address mov(rsp, rbx); // set sp to sender sp + pop_cont_fastpath(rthread); } void InterpreterMacroAssembler::get_method_counters(Register method, diff --git a/src/hotspot/cpu/x86/javaFrameAnchor_x86.hpp b/src/hotspot/cpu/x86/javaFrameAnchor_x86.hpp index 050e36e9dc4..0cea6568592 100644 --- a/src/hotspot/cpu/x86/javaFrameAnchor_x86.hpp +++ b/src/hotspot/cpu/x86/javaFrameAnchor_x86.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2002, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2002, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -77,4 +77,6 @@ public: intptr_t* last_Java_fp(void) { return _last_Java_fp; } + void set_last_Java_fp(intptr_t* fp) { _last_Java_fp = fp; } + #endif // CPU_X86_JAVAFRAMEANCHOR_X86_HPP diff --git a/src/hotspot/cpu/x86/macroAssembler_x86.cpp b/src/hotspot/cpu/x86/macroAssembler_x86.cpp index 17df08d6b11..7f74d809218 100644 --- a/src/hotspot/cpu/x86/macroAssembler_x86.cpp +++ b/src/hotspot/cpu/x86/macroAssembler_x86.cpp @@ -40,6 +40,7 @@ #include "oops/compressedOops.inline.hpp" #include "oops/klass.inline.hpp" #include "prims/methodHandles.hpp" +#include "runtime/continuation.hpp" #include "runtime/flags/flagSetting.hpp" #include "runtime/interfaceSupport.inline.hpp" #include "runtime/jniHandles.hpp" @@ -503,16 +504,12 @@ void MacroAssembler::call_VM_leaf_base(address entry_point, int num_args) { jcc(Assembler::zero, L); subq(rsp, 8); - { - call(RuntimeAddress(entry_point)); - } + call(RuntimeAddress(entry_point)); addq(rsp, 8); jmp(E); bind(L); - { - call(RuntimeAddress(entry_point)); - } + call(RuntimeAddress(entry_point)); bind(E); @@ -803,6 +800,15 @@ void MacroAssembler::warn(const char* msg) { pop(rbp); } +void MacroAssembler::_assert_asm(Assembler::Condition cc, const char* msg) { +#ifdef ASSERT + Label OK; + jcc(cc, OK); + stop(msg); + bind(OK); +#endif +} + void MacroAssembler::print_state() { address rip = pc(); pusha(); // get regs on stack @@ -1178,6 +1184,26 @@ void MacroAssembler::align(int modulus, int target) { } } +void MacroAssembler::push_f(XMMRegister r) { + subptr(rsp, wordSize); + movflt(Address(rsp, 0), r); +} + +void MacroAssembler::pop_f(XMMRegister r) { + movflt(r, Address(rsp, 0)); + addptr(rsp, wordSize); +} + +void MacroAssembler::push_d(XMMRegister r) { + subptr(rsp, 2 * wordSize); + movdbl(Address(rsp, 0), r); +} + +void MacroAssembler::pop_d(XMMRegister r) { + movdbl(r, Address(rsp, 0)); + addptr(rsp, 2 * Interpreter::stackElementSize); +} + void MacroAssembler::andpd(XMMRegister dst, AddressLiteral src, Register scratch_reg) { // Used in sign-masking with aligned address. assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes"); @@ -1635,6 +1661,20 @@ void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register call_VM_leaf(entry_point, 3); } +void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2, Register arg_3) { + LP64_ONLY(assert(arg_0 != c_rarg3, "smashed arg")); + LP64_ONLY(assert(arg_1 != c_rarg3, "smashed arg")); + LP64_ONLY(assert(arg_2 != c_rarg3, "smashed arg")); + pass_arg3(this, arg_3); + LP64_ONLY(assert(arg_0 != c_rarg2, "smashed arg")); + LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg")); + pass_arg2(this, arg_2); + LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg")); + pass_arg1(this, arg_1); + pass_arg0(this, arg_0); + call_VM_leaf(entry_point, 3); +} + void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0) { pass_arg0(this, arg_0); MacroAssembler::call_VM_leaf_base(entry_point, 1); @@ -1915,7 +1955,7 @@ void MacroAssembler::decrementl(Address dst, int value) { } void MacroAssembler::division_with_shift (Register reg, int shift_value) { - assert (shift_value > 0, "illegal shift value"); + assert(shift_value > 0, "illegal shift value"); Label _is_positive; testl (reg, reg); jcc (Assembler::positive, _is_positive); @@ -1954,16 +1994,28 @@ void MacroAssembler::enter() { mov(rbp, rsp); } +void MacroAssembler::post_call_nop() { + if (!Continuations::enabled()) { + return; + } + relocate(post_call_nop_Relocation::spec()); + emit_int8((int8_t)0x0f); + emit_int8((int8_t)0x1f); + emit_int8((int8_t)0x84); + emit_int8((int8_t)0x00); + emit_int32(0x00); +} + // A 5 byte nop that is safe for patching (see patch_verified_entry) void MacroAssembler::fat_nop() { if (UseAddressNop) { addr_nop_5(); } else { - emit_int8(0x26); // es: - emit_int8(0x2e); // cs: - emit_int8(0x64); // fs: - emit_int8(0x65); // gs: - emit_int8((unsigned char)0x90); + emit_int8((int8_t)0x26); // es: + emit_int8((int8_t)0x2e); // cs: + emit_int8((int8_t)0x64); // fs: + emit_int8((int8_t)0x65); // gs: + emit_int8((int8_t)0x90); } } @@ -2787,6 +2839,53 @@ void MacroAssembler::push_IU_state() { pusha(); } +void MacroAssembler::push_cont_fastpath(Register java_thread) { + if (!Continuations::enabled()) return; + Label done; + cmpptr(rsp, Address(java_thread, JavaThread::cont_fastpath_offset())); + jccb(Assembler::belowEqual, done); + movptr(Address(java_thread, JavaThread::cont_fastpath_offset()), rsp); + bind(done); +} + +void MacroAssembler::pop_cont_fastpath(Register java_thread) { + if (!Continuations::enabled()) return; + Label done; + cmpptr(rsp, Address(java_thread, JavaThread::cont_fastpath_offset())); + jccb(Assembler::below, done); + movptr(Address(java_thread, JavaThread::cont_fastpath_offset()), 0); + bind(done); +} + +void MacroAssembler::inc_held_monitor_count(Register java_thread) { + if (!Continuations::enabled()) return; + incrementl(Address(java_thread, JavaThread::held_monitor_count_offset())); +} + +void MacroAssembler::dec_held_monitor_count(Register java_thread) { + if (!Continuations::enabled()) return; + decrementl(Address(java_thread, JavaThread::held_monitor_count_offset())); +} + +void MacroAssembler::reset_held_monitor_count(Register java_thread) { + movl(Address(java_thread, JavaThread::held_monitor_count_offset()), (int32_t)0); +} + +#ifdef ASSERT +void MacroAssembler::stop_if_in_cont(Register cont, const char* name) { +#ifdef _LP64 + Label no_cont; + movptr(cont, Address(r15_thread, JavaThread::cont_entry_offset())); + testl(cont, cont); + jcc(Assembler::zero, no_cont); + stop(name); + bind(no_cont); +#else + Unimplemented(); +#endif +} +#endif + void MacroAssembler::reset_last_Java_frame(Register java_thread, bool clear_fp) { // determine java_thread register if (!java_thread->is_valid()) { java_thread = rdi; diff --git a/src/hotspot/cpu/x86/macroAssembler_x86.hpp b/src/hotspot/cpu/x86/macroAssembler_x86.hpp index ada7e26bc20..ad3d6186bb1 100644 --- a/src/hotspot/cpu/x86/macroAssembler_x86.hpp +++ b/src/hotspot/cpu/x86/macroAssembler_x86.hpp @@ -200,6 +200,7 @@ class MacroAssembler: public Assembler { void align(int modulus); void align(int modulus, int target); + void post_call_nop(); // A 5 byte nop that is safe for patching (see patch_verified_entry) void fat_nop(); @@ -294,6 +295,9 @@ class MacroAssembler: public Assembler { void call_VM_leaf(address entry_point, Register arg_1, Register arg_2, Register arg_3); + void call_VM_leaf(address entry_point, + Register arg_1, Register arg_2, Register arg_3, Register arg_4); + // These always tightly bind to MacroAssembler::call_VM_leaf_base // bypassing the virtual implementation void super_call_VM_leaf(address entry_point); @@ -519,6 +523,13 @@ class MacroAssembler: public Assembler { void push_CPU_state(); void pop_CPU_state(); + void push_cont_fastpath(Register java_thread); + void pop_cont_fastpath(Register java_thread); + void inc_held_monitor_count(Register java_thread); + void dec_held_monitor_count(Register java_thread); + void reset_held_monitor_count(Register java_thread); + DEBUG_ONLY(void stop_if_in_cont(Register cont_reg, const char* name);) + // Round up to a power of two void round_to(Register reg, int modulus); @@ -663,6 +674,10 @@ public: // prints msg and continues void warn(const char* msg); + void _assert_asm(Condition cc, const char* msg); +#define assert_asm0(cc, msg) _assert_asm(cc, FILE_AND_LINE ": " msg) +#define assert_asm(masm, command, cc, msg) DEBUG_ONLY((masm)->command; (masm)->_assert_asm(cc, FILE_AND_LINE ": " #command " " #cc ": " msg)) + // dumps registers and other state void print_state(); @@ -868,6 +883,11 @@ public: // Floating + void push_f(XMMRegister r); + void pop_f(XMMRegister r); + void push_d(XMMRegister r); + void pop_d(XMMRegister r); + void andpd(XMMRegister dst, Address src) { Assembler::andpd(dst, src); } void andpd(XMMRegister dst, AddressLiteral src, Register scratch_reg = rscratch1); void andpd(XMMRegister dst, XMMRegister src) { Assembler::andpd(dst, src); } diff --git a/src/hotspot/cpu/x86/nativeInst_x86.cpp b/src/hotspot/cpu/x86/nativeInst_x86.cpp index 5c29ebac315..b57c57bc803 100644 --- a/src/hotspot/cpu/x86/nativeInst_x86.cpp +++ b/src/hotspot/cpu/x86/nativeInst_x86.cpp @@ -633,7 +633,6 @@ void NativeGeneralJump::replace_mt_safe(address instr_addr, address code_buffer) assert(*((address)((intptr_t)instr_addr + i)) == a_byte, "mt safe patching failed"); } #endif - } @@ -649,3 +648,41 @@ address NativeGeneralJump::jump_destination() const { else return addr_at(0) + length + sbyte_at(offset); } + +void NativePostCallNop::make_deopt() { + /* makes the first 3 bytes into UD + * With the 8 bytes possibly (likely) split over cachelines the protocol on x86 looks like: + * + * Original state: NOP (4 bytes) offset (4 bytes) + * Writing the offset only touches the 4 last bytes (offset bytes) + * Making a deopt only touches the first 4 bytes and turns the NOP into a UD + * and to make disasembly look "reasonable" it turns the last byte into a + * TEST eax, offset so that the offset bytes of the NOP now becomes the imm32. + */ + + unsigned char patch[4]; + NativeDeoptInstruction::insert((address) patch, false); + patch[3] = 0xA9; // TEST eax, imm32 - this is just to keep disassembly looking correct and fills no real use. + address instr_addr = addr_at(0); + *(int32_t *)instr_addr = *(int32_t *)patch; + ICache::invalidate_range(instr_addr, instruction_size); +} + +void NativePostCallNop::patch(jint diff) { + assert(diff != 0, "must be"); + int32_t *code_pos = (int32_t *) addr_at(displacement_offset); + *((int32_t *)(code_pos)) = (int32_t) diff; +} + +void NativeDeoptInstruction::verify() { +} + +// Inserts an undefined instruction at a given pc +void NativeDeoptInstruction::insert(address code_pos, bool invalidate) { + *code_pos = instruction_prefix; + *(code_pos+1) = instruction_code; + *(code_pos+2) = 0x00; + if (invalidate) { + ICache::invalidate_range(code_pos, instruction_size); + } +} diff --git a/src/hotspot/cpu/x86/nativeInst_x86.hpp b/src/hotspot/cpu/x86/nativeInst_x86.hpp index a86128e7e4c..36178c72990 100644 --- a/src/hotspot/cpu/x86/nativeInst_x86.hpp +++ b/src/hotspot/cpu/x86/nativeInst_x86.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -82,7 +82,7 @@ class NativeInstruction { oop oop_at (int offset) const { return *(oop*) addr_at(offset); } - void set_char_at(int offset, char c) { *addr_at(offset) = (u_char)c; wrote(offset); } + void set_char_at(int offset, u_char c) { *addr_at(offset) = c; wrote(offset); } void set_int_at(int offset, jint i) { *(jint*)addr_at(offset) = i; wrote(offset); } void set_ptr_at (int offset, intptr_t ptr) { *(intptr_t*) addr_at(offset) = ptr; wrote(offset); } void set_oop_at (int offset, oop o) { *(oop*) addr_at(offset) = o; wrote(offset); } @@ -725,4 +725,56 @@ inline bool NativeInstruction::is_mov_literal64() { #endif // AMD64 } +class NativePostCallNop: public NativeInstruction { +public: + enum Intel_specific_constants { + instruction_code = 0x0f, + instruction_size = 8, + instruction_offset = 0, + displacement_offset = 4 + }; + + bool check() const { return int_at(0) == 0x841f0f; } + int displacement() const { return (jint) int_at(displacement_offset); } + void patch(jint diff); + void make_deopt(); +}; + +inline NativePostCallNop* nativePostCallNop_at(address address) { + NativePostCallNop* nop = (NativePostCallNop*) address; + if (nop->check()) { + return nop; + } + return NULL; +} + +inline NativePostCallNop* nativePostCallNop_unsafe_at(address address) { + NativePostCallNop* nop = (NativePostCallNop*) address; + assert(nop->check(), ""); + return nop; +} + +class NativeDeoptInstruction: public NativeInstruction { + public: + enum Intel_specific_constants { + instruction_prefix = 0x0F, + instruction_code = 0xFF, + instruction_size = 3, + instruction_offset = 0, + }; + + address instruction_address() const { return addr_at(instruction_offset); } + address next_instruction_address() const { return addr_at(instruction_size); } + + void verify(); + + static bool is_deopt_at(address instr) { + return ((*instr) & 0xFF) == NativeDeoptInstruction::instruction_prefix && + ((*(instr+1)) & 0xFF) == NativeDeoptInstruction::instruction_code; + } + + // MT-safe patching + static void insert(address code_pos, bool invalidate = true); +}; + #endif // CPU_X86_NATIVEINST_X86_HPP diff --git a/src/hotspot/cpu/x86/registerMap_x86.cpp b/src/hotspot/cpu/x86/registerMap_x86.cpp index 3acbb98bf76..f83c8cbaf06 100644 --- a/src/hotspot/cpu/x86/registerMap_x86.cpp +++ b/src/hotspot/cpu/x86/registerMap_x86.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2015, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -54,7 +54,7 @@ address RegisterMap::pd_location(VMReg reg) const { // XMM0-15 case (0 < offset_in_bytes < 16). No need to adjust base register (or offset). } } - address base_location = location(base_reg); + address base_location = location(base_reg, nullptr); if (base_location != NULL) { return base_location + offset_in_bytes; } @@ -63,5 +63,5 @@ address RegisterMap::pd_location(VMReg reg) const { } address RegisterMap::pd_location(VMReg base_reg, int slot_idx) const { - return location(base_reg->next(slot_idx)); + return location(base_reg->next(slot_idx), nullptr); } diff --git a/src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp b/src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp index 4d0eaa2784c..7c4a0e5cf42 100644 --- a/src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp +++ b/src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp @@ -28,6 +28,7 @@ #endif #include "asm/macroAssembler.hpp" #include "asm/macroAssembler.inline.hpp" +#include "code/compiledIC.hpp" #include "code/debugInfoRec.hpp" #include "code/icBuffer.hpp" #include "code/nativeInst.hpp" @@ -44,6 +45,8 @@ #include "oops/compiledICHolder.hpp" #include "oops/klass.inline.hpp" #include "prims/methodHandles.hpp" +#include "runtime/continuation.hpp" +#include "runtime/continuationEntry.inline.hpp" #include "runtime/jniHandles.hpp" #include "runtime/safepointMechanism.hpp" #include "runtime/sharedRuntime.hpp" @@ -936,6 +939,8 @@ void SharedRuntime::gen_i2c_adapter(MacroAssembler *masm, } } + __ push_cont_fastpath(r15_thread); // Set JavaThread::_cont_fastpath to the sp of the oldest interpreted frame we know about + // 6243940 We might end up in handle_wrong_method if // the callee is deoptimized as we race thru here. If that // happens we don't want to take a safepoint because the @@ -1424,6 +1429,103 @@ static void verify_oop_args(MacroAssembler* masm, } } +// defined in stubGenerator_x86_64.cpp +OopMap* continuation_enter_setup(MacroAssembler* masm, int& stack_slots); +void fill_continuation_entry(MacroAssembler* masm); +void continuation_enter_cleanup(MacroAssembler* masm); + +// enterSpecial(Continuation c, boolean isContinue, boolean isVirtualThread) +// On entry: c_rarg1 -- the continuation object +// c_rarg2 -- isContinue +// c_rarg3 -- isVirtualThread +static void gen_continuation_enter(MacroAssembler* masm, + const methodHandle& method, + const BasicType* sig_bt, + const VMRegPair* regs, + int& exception_offset, + OopMapSet*oop_maps, + int& frame_complete, + int& stack_slots) { + //verify_oop_args(masm, method, sig_bt, regs); + AddressLiteral resolve(SharedRuntime::get_resolve_static_call_stub(), + relocInfo::static_call_type); + + stack_slots = 2; // will be overwritten + address start = __ pc(); + + Label call_thaw, exit; + + __ enter(); + + //BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler(); + //bs->nmethod_entry_barrier(masm); + OopMap* map = continuation_enter_setup(masm, stack_slots); // kills rax + + // Frame is now completed as far as size and linkage. + frame_complete =__ pc() - start; + // if isContinue == 0 + // _enterSP = sp + // end + + fill_continuation_entry(masm); // kills rax + + __ cmpl(c_rarg2, 0); + __ jcc(Assembler::notEqual, call_thaw); + + int up = align_up((intptr_t) __ pc() + 1, 4) - (intptr_t) (__ pc() + 1); + if (up > 0) { + __ nop(up); + } + + address mark = __ pc(); + __ call(resolve); + oop_maps->add_gc_map(__ pc() - start, map); + __ post_call_nop(); + + __ jmp(exit); + + __ bind(call_thaw); + + __ movptr(rbx, (intptr_t) StubRoutines::cont_thaw()); + __ call(rbx); + oop_maps->add_gc_map(__ pc() - start, map->deep_copy()); + ContinuationEntry::return_pc_offset = __ pc() - start; + __ post_call_nop(); + + __ bind(exit); + continuation_enter_cleanup(masm); + __ pop(rbp); + __ ret(0); + + /// exception handling + + exception_offset = __ pc() - start; + + continuation_enter_cleanup(masm); + __ pop(rbp); + + __ movptr(rbx, rax); // save the exception + __ movptr(c_rarg0, Address(rsp, 0)); + + __ call_VM_leaf(CAST_FROM_FN_PTR(address, + SharedRuntime::exception_handler_for_return_address), + r15_thread, c_rarg0); + __ mov(rdi, rax); + __ movptr(rax, rbx); + __ mov(rbx, rdi); + __ pop(rdx); + + // continue at exception handler (return address removed) + // rax: exception + // rbx: exception handler + // rdx: throwing pc + __ verify_oop(rax); + __ jmp(rbx); + + CodeBuffer* cbuf = masm->code_section()->outer(); + address stub = CompiledStaticCall::emit_to_interp_stub(*cbuf, mark); +} + static void gen_special_dispatch(MacroAssembler* masm, const methodHandle& method, const BasicType* sig_bt, @@ -1505,6 +1607,37 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm, BasicType* in_sig_bt, VMRegPair* in_regs, BasicType ret_type) { + if (method->is_continuation_enter_intrinsic()) { + vmIntrinsics::ID iid = method->intrinsic_id(); + intptr_t start = (intptr_t)__ pc(); + int vep_offset = ((intptr_t)__ pc()) - start; + int exception_offset = 0; + int frame_complete = 0; + int stack_slots = 0; + OopMapSet* oop_maps = new OopMapSet(); + gen_continuation_enter(masm, + method, + in_sig_bt, + in_regs, + exception_offset, + oop_maps, + frame_complete, + stack_slots); + __ flush(); + nmethod* nm = nmethod::new_native_nmethod(method, + compile_id, + masm->code(), + vep_offset, + frame_complete, + stack_slots, + in_ByteSize(-1), + in_ByteSize(-1), + oop_maps, + exception_offset); + ContinuationEntry::set_enter_nmethod(nm); + return nm; + } + if (method->is_method_handle_intrinsic()) { vmIntrinsics::ID iid = method->intrinsic_id(); intptr_t start = (intptr_t)__ pc(); @@ -1949,7 +2082,6 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm, } // Slow path will re-enter here - __ bind(lock_done); } @@ -2087,7 +2219,6 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm, } __ bind(done); - } { SkipIfEqual skip(masm, &DTraceMethodProbes, false); @@ -2861,7 +2992,7 @@ SafepointBlob* SharedRuntime::generate_handler_blob(address call_ptr, int poll_t // address of the call in order to generate an oopmap. Hence, we do all the // work ourselves. - __ set_last_Java_frame(noreg, noreg, NULL); + __ set_last_Java_frame(noreg, noreg, NULL); // JavaFrameAnchor::capture_last_Java_pc() will get the pc from the return address, which we store next: // The return address must always be correct so that frame constructor never // sees an invalid pc. diff --git a/src/hotspot/cpu/x86/smallRegisterMap_x86.inline.hpp b/src/hotspot/cpu/x86/smallRegisterMap_x86.inline.hpp new file mode 100644 index 00000000000..5f21939a3aa --- /dev/null +++ b/src/hotspot/cpu/x86/smallRegisterMap_x86.inline.hpp @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_X86_SMALLREGISTERMAP_X86_INLINE_HPP +#define CPU_X86_SMALLREGISTERMAP_X86_INLINE_HPP + +#include "runtime/frame.inline.hpp" +#include "runtime/registerMap.hpp" + +// Java frames don't have callee saved registers (except for rbp), so we can use a smaller RegisterMap +class SmallRegisterMap { +public: + static constexpr SmallRegisterMap* instance = nullptr; +private: + static void assert_is_rbp(VMReg r) NOT_DEBUG_RETURN + DEBUG_ONLY({ assert(r == rbp->as_VMReg() || r == rbp->as_VMReg()->next(), "Reg: %s", r->name()); }) +public: + // as_RegisterMap is used when we didn't want to templatize and abstract over RegisterMap type to support SmallRegisterMap + // Consider enhancing SmallRegisterMap to support those cases + const RegisterMap* as_RegisterMap() const { return nullptr; } + RegisterMap* as_RegisterMap() { return nullptr; } + + RegisterMap* copy_to_RegisterMap(RegisterMap* map, intptr_t* sp) const { + map->clear(); + map->set_include_argument_oops(this->include_argument_oops()); + frame::update_map_with_saved_link(map, (intptr_t**)sp - frame::sender_sp_offset); + return map; + } + + SmallRegisterMap() {} + + SmallRegisterMap(const RegisterMap* map) { + #ifdef ASSERT + for(int i = 0; i < RegisterMap::reg_count; i++) { + VMReg r = VMRegImpl::as_VMReg(i); + if (map->location(r, (intptr_t*)nullptr) != nullptr) assert_is_rbp(r); + } + #endif + } + + inline address location(VMReg reg, intptr_t* sp) const { + assert_is_rbp(reg); + return (address)(sp - frame::sender_sp_offset); + } + + inline void set_location(VMReg reg, address loc) { assert_is_rbp(reg); } + + JavaThread* thread() const { + #ifndef ASSERT + guarantee (false, "unreachable"); + #endif + return nullptr; + } + + bool update_map() const { return false; } + bool walk_cont() const { return false; } + bool include_argument_oops() const { return false; } + void set_include_argument_oops(bool f) {} + bool in_cont() const { return false; } + stackChunkHandle stack_chunk() const { return stackChunkHandle(); } + +#ifdef ASSERT + bool should_skip_missing() const { return false; } + VMReg find_register_spilled_here(void* p, intptr_t* sp) { return rbp->as_VMReg(); } + void print() const { print_on(tty); } + void print_on(outputStream* st) const { st->print_cr("Small register map"); } +#endif +}; + +#endif // CPU_X86_SMALLREGISTERMAP_X86_INLINE_HPP diff --git a/src/hotspot/cpu/x86/stackChunkFrameStream_x86.inline.hpp b/src/hotspot/cpu/x86/stackChunkFrameStream_x86.inline.hpp new file mode 100644 index 00000000000..70f10dc646d --- /dev/null +++ b/src/hotspot/cpu/x86/stackChunkFrameStream_x86.inline.hpp @@ -0,0 +1,149 @@ +/* + * Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_X86_STACKCHUNKFRAMESTREAM_X86_INLINE_HPP +#define CPU_X86_STACKCHUNKFRAMESTREAM_X86_INLINE_HPP + +#include "interpreter/oopMapCache.hpp" +#include "runtime/frame.inline.hpp" +#include "runtime/registerMap.hpp" + +#ifdef ASSERT +template +inline bool StackChunkFrameStream::is_in_frame(void* p0) const { + assert(!is_done(), ""); + intptr_t* p = (intptr_t*)p0; + int argsize = is_compiled() ? (_cb->as_compiled_method()->method()->num_stack_arg_slots() * VMRegImpl::stack_slot_size) >> LogBytesPerWord : 0; + int frame_size = _cb->frame_size() + argsize; + return p == sp() - frame::sender_sp_offset || ((p - unextended_sp()) >= 0 && (p - unextended_sp()) < frame_size); +} +#endif + +template +inline frame StackChunkFrameStream::to_frame() const { + if (is_done()) { + return frame(_sp, _sp, nullptr, nullptr, nullptr, nullptr, true); + } else { + return frame(sp(), unextended_sp(), fp(), pc(), cb(), _oopmap, true); + } +} + +template +inline address StackChunkFrameStream::get_pc() const { + assert(!is_done(), ""); + return *(address*)(_sp - 1); +} + +template +inline intptr_t* StackChunkFrameStream::fp() const { + intptr_t* fp_addr = _sp - frame::sender_sp_offset; + return (frame_kind == ChunkFrames::Mixed && is_interpreted()) + ? fp_addr + *fp_addr // derelativize + : *(intptr_t**)fp_addr; +} + +template +inline intptr_t* StackChunkFrameStream::derelativize(int offset) const { + intptr_t* fp = this->fp(); + assert(fp != nullptr, ""); + return fp + fp[offset]; +} + +template +inline intptr_t* StackChunkFrameStream::unextended_sp_for_interpreter_frame() const { + assert_is_interpreted_and_frame_type_mixed(); + return derelativize(frame::interpreter_frame_last_sp_offset); +} + +template +intptr_t* StackChunkFrameStream::next_sp_for_interpreter_frame() const { + assert_is_interpreted_and_frame_type_mixed(); + return (derelativize(frame::interpreter_frame_locals_offset) + 1 >= _end) ? _end : fp() + frame::sender_sp_offset; +} + +template +inline void StackChunkFrameStream::next_for_interpreter_frame() { + assert_is_interpreted_and_frame_type_mixed(); + if (derelativize(frame::interpreter_frame_locals_offset) + 1 >= _end) { + _unextended_sp = _end; + _sp = _end; + } else { + intptr_t* fp = this->fp(); + _unextended_sp = fp + fp[frame::interpreter_frame_sender_sp_offset]; + _sp = fp + frame::sender_sp_offset; + } +} + +template +inline int StackChunkFrameStream::interpreter_frame_size() const { + assert_is_interpreted_and_frame_type_mixed(); + + intptr_t* top = unextended_sp(); // later subtract argsize if callee is interpreted + intptr_t* bottom = derelativize(frame::interpreter_frame_locals_offset) + 1; // the sender's unextended sp: derelativize(frame::interpreter_frame_sender_sp_offset); + return (int)(bottom - top); +} + +template +inline int StackChunkFrameStream::interpreter_frame_stack_argsize() const { + assert_is_interpreted_and_frame_type_mixed(); + int diff = (int)(derelativize(frame::interpreter_frame_locals_offset) - derelativize(frame::interpreter_frame_sender_sp_offset) + 1); + return diff; +} + +template +inline int StackChunkFrameStream::interpreter_frame_num_oops() const { + assert_is_interpreted_and_frame_type_mixed(); + ResourceMark rm; + InterpreterOopMap mask; + frame f = to_frame(); + f.interpreted_frame_oop_map(&mask); + return mask.num_oops() + + 1 // for the mirror oop + + ((intptr_t*)f.interpreter_frame_monitor_begin() + - (intptr_t*)f.interpreter_frame_monitor_end())/BasicObjectLock::size(); +} + +template<> +template<> +inline void StackChunkFrameStream::update_reg_map_pd(RegisterMap* map) { + if (map->update_map()) { + frame::update_map_with_saved_link(map, map->in_cont() ? (intptr_t**)(intptr_t)frame::sender_sp_offset + : (intptr_t**)(_sp - frame::sender_sp_offset)); + } +} + +template<> +template<> +inline void StackChunkFrameStream::update_reg_map_pd(RegisterMap* map) { + if (map->update_map()) { + frame::update_map_with_saved_link(map, map->in_cont() ? (intptr_t**)(intptr_t)frame::sender_sp_offset + : (intptr_t**)(_sp - frame::sender_sp_offset)); + } +} + +template +template +inline void StackChunkFrameStream::update_reg_map_pd(RegisterMapT* map) {} + +#endif // CPU_X86_STACKCHUNKFRAMESTREAM_X86_INLINE_HPP diff --git a/src/hotspot/cpu/x86/stackChunkOop_x86.inline.hpp b/src/hotspot/cpu/x86/stackChunkOop_x86.inline.hpp new file mode 100644 index 00000000000..4ff6dadabd7 --- /dev/null +++ b/src/hotspot/cpu/x86/stackChunkOop_x86.inline.hpp @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_X86_STACKCHUNKOOP_X86_INLINE_HPP +#define CPU_X86_STACKCHUNKOOP_X86_INLINE_HPP + +#include "runtime/frame.inline.hpp" + +inline void stackChunkOopDesc::relativize_frame_pd(frame& fr) const { + if (fr.is_interpreted_frame()) { + fr.set_offset_fp(relativize_address(fr.fp())); + } +} + +inline void stackChunkOopDesc::derelativize_frame_pd(frame& fr) const { + if (fr.is_interpreted_frame()) { + fr.set_fp(derelativize_address(fr.offset_fp())); + } +} + +#endif // CPU_X86_STACKCHUNKOOP_X86_INLINE_HPP diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_32.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_32.cpp index 16f30df3d61..f5f1e012418 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_32.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_32.cpp @@ -4053,10 +4053,10 @@ class StubGenerator: public StubCodeGenerator { } }; // end class declaration -#define UCM_TABLE_MAX_ENTRIES 8 -void StubGenerator_generate(CodeBuffer* code, bool all) { +#define UCM_TABLE_MAX_ENTRIES 16 +void StubGenerator_generate(CodeBuffer* code, int phase) { if (UnsafeCopyMemory::_table == NULL) { UnsafeCopyMemory::create_table(UCM_TABLE_MAX_ENTRIES); } - StubGenerator g(code, all); + StubGenerator g(code, phase); } diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp index 05e7b30d6d1..0c07c345ba2 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp @@ -38,8 +38,11 @@ #include "oops/method.hpp" #include "oops/objArrayKlass.hpp" #include "oops/oop.inline.hpp" +#include "prims/jvmtiExport.hpp" #include "prims/methodHandles.hpp" #include "runtime/arguments.hpp" +#include "runtime/continuation.hpp" +#include "runtime/continuationEntry.inline.hpp" #include "runtime/frame.inline.hpp" #include "runtime/handles.inline.hpp" #include "runtime/sharedRuntime.hpp" @@ -55,6 +58,9 @@ #if INCLUDE_ZGC #include "gc/z/zThreadLocalData.hpp" #endif +#if INCLUDE_JFR +#include "jfr/support/jfrIntrinsics.hpp" +#endif // Declaration and definition of StubGenerator (no .hpp file). // For a more detailed description of the stub routine structure @@ -73,6 +79,10 @@ #define BIND(label) bind(label); BLOCK_COMMENT(#label ":") const int MXCSR_MASK = 0xFFC0; // Mask out any pending exceptions +OopMap* continuation_enter_setup(MacroAssembler* masm, int& stack_slots); +void fill_continuation_entry(MacroAssembler* masm); +void continuation_enter_cleanup(MacroAssembler* masm); + // Stub Code definitions class StubGenerator: public StubCodeGenerator { @@ -383,6 +393,8 @@ class StubGenerator: public StubCodeGenerator { } #endif + __ pop_cont_fastpath(r15_thread); + // restore regs belonging to calling function #ifdef _WIN64 // emit the restores for xmm regs @@ -7414,6 +7426,249 @@ address generate_avx_ghash_processBlocks() { } + RuntimeStub* generate_cont_doYield() { + if (!Continuations::enabled()) return nullptr; + + const char *name = "cont_doYield"; + + enum layout { + rbp_off, + rbpH_off, + return_off, + return_off2, + framesize // inclusive of return address + }; + // assert(is_even(framesize/2), "sp not 16-byte aligned"); + + int insts_size = 512; + int locs_size = 64; + CodeBuffer code(name, insts_size, locs_size); + OopMapSet* oop_maps = new OopMapSet(); + MacroAssembler* masm = new MacroAssembler(&code); + MacroAssembler* _masm = masm; + + address start = __ pc(); + + __ enter(); + + __ movptr(c_rarg1, rsp); + + int frame_complete = __ pc() - start; + address the_pc = __ pc(); + + __ post_call_nop(); // this must be exactly after the pc value that is pushed into the frame info, we use this nop for fast CodeBlob lookup + + __ movptr(c_rarg0, r15_thread); + __ set_last_Java_frame(rsp, rbp, the_pc); + __ call_VM_leaf(Continuation::freeze_entry(), 2); + __ reset_last_Java_frame(true); + + Label pinned; + + __ testq(rax, rax); + __ jcc(Assembler::notZero, pinned); + + __ movptr(rsp, Address(r15_thread, JavaThread::cont_entry_offset())); + continuation_enter_cleanup(masm); + __ pop(rbp); + __ ret(0); + + __ bind(pinned); // pinned -- return to caller + + __ leave(); + __ ret(0); + + OopMap* map = new OopMap(framesize, 1); + oop_maps->add_gc_map(the_pc - start, map); + + RuntimeStub* stub = // codeBlob framesize is in words (not VMRegImpl::slot_size) + RuntimeStub::new_runtime_stub(name, + &code, + frame_complete, + (framesize >> (LogBytesPerWord - LogBytesPerInt)), + oop_maps, false); + return stub; + } + + address generate_cont_thaw(bool return_barrier, bool exception) { + assert(return_barrier || !exception, "must be"); + + address start = __ pc(); + + // TODO: Handle Valhalla return types. May require generating different return barriers. + + if (!return_barrier) { + __ pop(c_rarg3); // pop return address. if we don't do this, we get a drift, where the bottom-most frozen frame continuously grows + } else { + __ movptr(rsp, Address(r15_thread, JavaThread::cont_entry_offset())); + } + assert_asm(_masm, cmpptr(rsp, Address(r15_thread, JavaThread::cont_entry_offset())), Assembler::equal, "incorrect rsp"); + + if (return_barrier) { + __ push(rax); __ push_d(xmm0); // preserve possible return value from a method returning to the return barrier + } + + __ movl(c_rarg1, (return_barrier ? 1 : 0) + (exception ? 1 : 0)); + __ call_VM_leaf(CAST_FROM_FN_PTR(address, Continuation::prepare_thaw), r15_thread, c_rarg1); + __ movptr(rbx, rax); // rax contains the size of the frames to thaw, 0 if overflow or no more frames + + if (return_barrier) { + __ pop_d(xmm0); __ pop(rax); // restore return value (no safepoint in the call to thaw, so even an oop return value should be OK) + } + assert_asm(_masm, cmpptr(rsp, Address(r15_thread, JavaThread::cont_entry_offset())), Assembler::equal, "incorrect rsp"); + + Label thaw_success; + __ testq(rbx, rbx); // rbx contains the size of the frames to thaw, 0 if overflow or no more frames + __ jcc(Assembler::notZero, thaw_success); + __ jump(ExternalAddress(StubRoutines::throw_StackOverflowError_entry())); + __ bind(thaw_success); + + __ subq(rsp, rbx); // make room for the thawed frames + __ andptr(rsp, -16); // align + + if (return_barrier) { + __ push(rax); __ push_d(xmm0); // save original return value -- again + } + + // If we want, we can templatize thaw by kind, and have three different entries + if (exception) __ movl(c_rarg1, (int32_t)Continuation::thaw_exception); + else if (return_barrier) __ movl(c_rarg1, (int32_t)Continuation::thaw_return_barrier); + else __ movl(c_rarg1, (int32_t)Continuation::thaw_top); + + __ call_VM_leaf(Continuation::thaw_entry(), r15_thread, c_rarg1); + __ movptr(rbx, rax); // rax is the sp of the yielding frame + + if (return_barrier) { + __ pop_d(xmm0); __ pop(rax); // restore return value (no safepoint in the call to thaw, so even an oop return value should be OK) + } else { + __ movl(rax, 0); // return 0 (success) from doYield + } + + __ movptr(rsp, rbx); // we're now on the yield frame (which is in an address above us b/c rsp has been pushed down) + __ subptr(rsp, 2*wordSize); // now pointing to rbp spill + + if (exception) { + __ movptr(c_rarg1, Address(rsp, wordSize)); // return address + __ push(rax); // save return value contaning the exception oop + __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), r15_thread, c_rarg1); + __ movptr(rbx, rax); // the exception handler + __ pop(rax); // restore return value contaning the exception oop + __ pop(rbp); + __ pop(rdx); // rdx must contain the original pc in the case of exception; see OptoRuntime::generate_exception_blob + __ jmp(rbx); // the exception handler + } + + // We're "returning" into the topmost thawed frame; see Thaw::push_return_frame + __ pop(rbp); + __ ret(0); + + return start; + } + + address generate_cont_thaw() { + if (!Continuations::enabled()) return nullptr; + + StubCodeMark mark(this, "StubRoutines", "Cont thaw"); + address start = __ pc(); + generate_cont_thaw(false, false); + return start; + } + + address generate_cont_returnBarrier() { + if (!Continuations::enabled()) return nullptr; + + // TODO: will probably need multiple return barriers depending on return type + StubCodeMark mark(this, "StubRoutines", "cont return barrier"); + address start = __ pc(); + + generate_cont_thaw(true, false); + + return start; + } + + address generate_cont_returnBarrier_exception() { + if (!Continuations::enabled()) return nullptr; + + StubCodeMark mark(this, "StubRoutines", "cont return barrier exception handler"); + address start = __ pc(); + + generate_cont_thaw(true, true); + + return start; + } + +#if INCLUDE_JFR + + static void jfr_prologue(address the_pc, MacroAssembler* _masm) { + __ set_last_Java_frame(rsp, rbp, the_pc); + __ movptr(c_rarg0, r15_thread); + } + + // The handle is dereferenced through a load barrier. + static void jfr_epilogue(MacroAssembler* _masm) { + __ reset_last_Java_frame(true); + Label null_jobject; + __ testq(rax, rax); + __ jcc(Assembler::zero, null_jobject); + DecoratorSet decorators = ACCESS_READ | IN_NATIVE; + BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler(); + bs->load_at(_masm, decorators, T_OBJECT, rax, Address(rax, 0), c_rarg0, r15_thread); + __ bind(null_jobject); + } + + static RuntimeStub* generate_jfr_stub(const char* name, address entrypoint) { + enum layout { + rbp_off, + rbpH_off, + return_off, + return_off2, + framesize // inclusive of return address + }; + + int insts_size = 512; + int locs_size = 64; + CodeBuffer code(name, insts_size, locs_size); + OopMapSet* oop_maps = new OopMapSet(); + MacroAssembler* masm = new MacroAssembler(&code); + MacroAssembler* _masm = masm; + + address start = __ pc(); + __ enter(); + int frame_complete = __ pc() - start; + address the_pc = __ pc(); + jfr_prologue(the_pc, _masm); + __ call_VM_leaf(entrypoint, 1); + jfr_epilogue(_masm); + __ leave(); + __ ret(0); + + OopMap* map = new OopMap(framesize, 1); // rbp + oop_maps->add_gc_map(the_pc - start, map); + + RuntimeStub* stub = // codeBlob framesize is in words (not VMRegImpl::slot_size) + RuntimeStub::new_runtime_stub(name, &code, frame_complete, + (framesize >> (LogBytesPerWord - LogBytesPerInt)), + oop_maps, false); + return stub; + } + + // For c2: c_rarg0 is junk, call to runtime to write a checkpoint. + // It returns a jobject handle to the event writer. + // The handle is dereferenced and the return value is the event writer oop. + RuntimeStub* generate_jfr_write_checkpoint() { + return generate_jfr_stub("jfr_write_checkpoint", + CAST_FROM_FN_PTR(address, JfrIntrinsicSupport::write_checkpoint)); + } + + // For c1: call the corresponding runtime routine, it returns a jobject handle to the event writer. + // The handle is dereferenced and the return value is the event writer oop. + RuntimeStub* generate_jfr_get_event_writer() { + return generate_jfr_stub("jfr_get_event_writer", + CAST_FROM_FN_PTR(address, JfrIntrinsicSupport::event_writer)); + } + +#endif // INCLUDE_JFR + #undef __ #define __ masm-> @@ -7634,6 +7889,21 @@ address generate_avx_ghash_processBlocks() { } } + void generate_phase1() { + // Continuation stubs: + StubRoutines::_cont_thaw = generate_cont_thaw(); + StubRoutines::_cont_returnBarrier = generate_cont_returnBarrier(); + StubRoutines::_cont_returnBarrierExc = generate_cont_returnBarrier_exception(); + StubRoutines::_cont_doYield_stub = generate_cont_doYield(); + StubRoutines::_cont_doYield = StubRoutines::_cont_doYield_stub == nullptr ? nullptr + : StubRoutines::_cont_doYield_stub->entry_point(); + + JFR_ONLY(StubRoutines::_jfr_write_checkpoint_stub = generate_jfr_write_checkpoint();) + JFR_ONLY(StubRoutines::_jfr_write_checkpoint = StubRoutines::_jfr_write_checkpoint_stub->entry_point();) + JFR_ONLY(StubRoutines::_jfr_get_event_writer_stub = generate_jfr_get_event_writer();) + JFR_ONLY(StubRoutines::_jfr_get_event_writer = StubRoutines::_jfr_get_event_writer_stub->entry_point();) + } + void generate_all() { // Generates all stubs and initializes the entry points @@ -7897,19 +8167,90 @@ address generate_avx_ghash_processBlocks() { } public: - StubGenerator(CodeBuffer* code, bool all) : StubCodeGenerator(code) { - if (all) { - generate_all(); - } else { + StubGenerator(CodeBuffer* code, int phase) : StubCodeGenerator(code) { + if (phase == 0) { generate_initial(); + } else if (phase == 1) { + generate_phase1(); // stubs that must be available for the interpreter + } else { + generate_all(); } } }; // end class declaration #define UCM_TABLE_MAX_ENTRIES 16 -void StubGenerator_generate(CodeBuffer* code, bool all) { +void StubGenerator_generate(CodeBuffer* code, int phase) { if (UnsafeCopyMemory::_table == NULL) { UnsafeCopyMemory::create_table(UCM_TABLE_MAX_ENTRIES); } - StubGenerator g(code, all); + StubGenerator g(code, phase); } + +#undef __ +#define __ masm-> + +// on exit, rsp points to the ContinuationEntry +// kills rax +OopMap* continuation_enter_setup(MacroAssembler* masm, int& stack_slots) { + assert(ContinuationEntry::size() % VMRegImpl::stack_slot_size == 0, ""); + assert(in_bytes(ContinuationEntry::cont_offset()) % VMRegImpl::stack_slot_size == 0, ""); + assert(in_bytes(ContinuationEntry::chunk_offset()) % VMRegImpl::stack_slot_size == 0, ""); + + stack_slots += (int)ContinuationEntry::size()/wordSize; + __ subptr(rsp, (int32_t)ContinuationEntry::size()); // place Continuation metadata + + OopMap* map = new OopMap(((int)ContinuationEntry::size() + wordSize)/ VMRegImpl::stack_slot_size, 0 /* arg_slots*/); + ContinuationEntry::setup_oopmap(map); + + __ movptr(rax, Address(r15_thread, JavaThread::cont_entry_offset())); + __ movptr(Address(rsp, ContinuationEntry::parent_offset()), rax); + __ movptr(Address(r15_thread, JavaThread::cont_entry_offset()), rsp); + + return map; +} + +// on entry c_rarg1 points to the continuation +// rsp points to ContinuationEntry +// c_rarg3 -- isVirtualThread +// kills rax +void fill_continuation_entry(MacroAssembler* masm) { + DEBUG_ONLY(__ movl(Address(rsp, ContinuationEntry::cookie_offset()), 0x1234);) + + __ movptr(Address(rsp, ContinuationEntry::cont_offset()), c_rarg1); + __ movl (Address(rsp, ContinuationEntry::flags_offset()), c_rarg3); + __ movptr(Address(rsp, ContinuationEntry::chunk_offset()), (int32_t)0); + __ movl(Address(rsp, ContinuationEntry::argsize_offset()), (int32_t)0); + __ movl(Address(rsp, ContinuationEntry::pin_count_offset()), (int32_t)0); + + __ movptr(rax, Address(r15_thread, JavaThread::cont_fastpath_offset())); + __ movptr(Address(rsp, ContinuationEntry::parent_cont_fastpath_offset()), rax); + __ movl(rax, Address(r15_thread, JavaThread::held_monitor_count_offset())); + __ movl(Address(rsp, ContinuationEntry::parent_held_monitor_count_offset()), rax); + + __ movptr(Address(r15_thread, JavaThread::cont_fastpath_offset()), 0); + __ reset_held_monitor_count(r15_thread); +} + +// on entry, rsp points to the ContinuationEntry +// on exit, rsp points to the spilled rbp in the entry frame +// kills rbx, rcx +void continuation_enter_cleanup(MacroAssembler* masm) { +#ifndef PRODUCT + Label OK; + __ cmpptr(rsp, Address(r15_thread, JavaThread::cont_entry_offset())); + __ jcc(Assembler::equal, OK); + __ stop("incorrect rsp1"); + __ bind(OK); +#endif + + __ movptr(rbx, Address(rsp, ContinuationEntry::parent_cont_fastpath_offset())); + __ movptr(Address(r15_thread, JavaThread::cont_fastpath_offset()), rbx); + __ movl(rbx, Address(rsp, ContinuationEntry::parent_held_monitor_count_offset())); + __ movl(Address(r15_thread, JavaThread::held_monitor_count_offset()), rbx); + + __ movptr(rcx, Address(rsp, ContinuationEntry::parent_offset())); + __ movptr(Address(r15_thread, JavaThread::cont_entry_offset()), rcx); + __ addptr(rsp, (int32_t)ContinuationEntry::size()); +} + +#undef __ diff --git a/src/hotspot/cpu/x86/stubRoutines_x86.hpp b/src/hotspot/cpu/x86/stubRoutines_x86.hpp index 5119dde4fd5..d5b96089a60 100644 --- a/src/hotspot/cpu/x86/stubRoutines_x86.hpp +++ b/src/hotspot/cpu/x86/stubRoutines_x86.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2013, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -32,8 +32,8 @@ static bool returns_to_call_stub(address return_pc) { return return_pc == _call_stub_return_address; } enum platform_dependent_constants { - code_size1 = 20000 LP64_ONLY(+10000), // simply increase if too small (assembler will crash if too small) - code_size2 = 35300 LP64_ONLY(+35000) // simply increase if too small (assembler will crash if too small) + code_size1 = 20000 LP64_ONLY(+10000), // simply increase if too small (assembler will crash if too small) + code_size2 = 35300 LP64_ONLY(+35000) WINDOWS_ONLY(+2048) // simply increase if too small (assembler will crash if too small) }; class x86 { diff --git a/src/hotspot/cpu/x86/templateInterpreterGenerator_x86.cpp b/src/hotspot/cpu/x86/templateInterpreterGenerator_x86.cpp index a1ee9d67af8..748ac239029 100644 --- a/src/hotspot/cpu/x86/templateInterpreterGenerator_x86.cpp +++ b/src/hotspot/cpu/x86/templateInterpreterGenerator_x86.cpp @@ -40,6 +40,7 @@ #include "oops/oop.inline.hpp" #include "prims/jvmtiExport.hpp" #include "prims/jvmtiThreadState.hpp" +#include "runtime/continuation.hpp" #include "runtime/deoptimization.hpp" #include "runtime/frame.inline.hpp" #include "runtime/jniHandles.hpp" @@ -365,8 +366,16 @@ address TemplateInterpreterGenerator::generate_safept_entry_for( TosState state, address runtime_entry) { address entry = __ pc(); + + const Register rthread = NOT_LP64(rcx) LP64_ONLY(r15_thread); + __ push(state); + NOT_LP64(__ get_thread(rthread);) + __ push_cont_fastpath(rthread); __ call_VM(noreg, runtime_entry); + NOT_LP64(__ get_thread(rthread);) + __ pop_cont_fastpath(rthread); + __ dispatch_via(vtos, Interpreter::_normal_table.table_for(vtos)); return entry; } @@ -599,6 +608,10 @@ void TemplateInterpreterGenerator::lock_method() { const Register lockreg = NOT_LP64(rdx) LP64_ONLY(c_rarg1); __ movptr(lockreg, rsp); // object address __ lock_object(lockreg); + + Register rthread = NOT_LP64(rax) LP64_ONLY(r15_thread); + NOT_LP64(__ get_thread(rthread);) + __ inc_held_monitor_count(rthread); } // Generate a fixed interpreter frame. This is identical setup for @@ -650,6 +663,29 @@ void TemplateInterpreterGenerator::generate_fixed_frame(bool native_call) { // End of helpers +address TemplateInterpreterGenerator::generate_Continuation_doYield_entry(void) { + if (!Continuations::enabled()) return nullptr; + +#ifdef _LP64 + address entry = __ pc(); + assert(StubRoutines::cont_doYield() != NULL, "stub not yet generated"); + + // __ movl(c_rarg1, Address(rsp, wordSize)); // scopes + const Register thread1 = NOT_LP64(rdi) LP64_ONLY(r15_thread); + NOT_LP64(__ get_thread(thread1)); + __ push_cont_fastpath(thread1); + + __ jump(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::cont_doYield()))); + // return value is in rax + + return entry; +#else + // Not implemented. Allow startup of legacy Java code that does not touch + // Continuation.doYield yet. Throw AbstractMethodError on access. + return generate_abstract_entry(); +#endif +} + // Method entry for java.lang.ref.Reference.get. address TemplateInterpreterGenerator::generate_Reference_get_entry(void) { // Code: _aload_0, _getfield, _areturn @@ -1243,6 +1279,8 @@ address TemplateInterpreterGenerator::generate_native_entry(bool synchronized) { __ bind(unlock); __ unlock_object(regmon); + NOT_LP64(__ get_thread(thread);) + __ dec_held_monitor_count(thread); } __ bind(L); } @@ -1311,7 +1349,7 @@ address TemplateInterpreterGenerator::generate_normal_entry(bool synchronized) { bool inc_counter = UseCompiler || CountCompiledCalls || LogTouchedMethods; // ebx: Method* - // rbcp: sender sp + // rbcp: sender sp (set in InterpreterMacroAssembler::prepare_to_jump_from_interpreted / generate_call_stub) address entry_point = __ pc(); const Address constMethod(rbx, Method::const_offset()); diff --git a/src/hotspot/cpu/x86/templateInterpreterGenerator_x86_64.cpp b/src/hotspot/cpu/x86/templateInterpreterGenerator_x86_64.cpp index 1a1dad0cea3..e5355e19be8 100644 --- a/src/hotspot/cpu/x86/templateInterpreterGenerator_x86_64.cpp +++ b/src/hotspot/cpu/x86/templateInterpreterGenerator_x86_64.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2003, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2003, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -449,7 +449,7 @@ address TemplateInterpreterGenerator::generate_currentThread() { address entry_point = __ pc(); - __ movptr(rax, Address(r15_thread, JavaThread::threadObj_offset())); + __ movptr(rax, Address(r15_thread, JavaThread::vthread_offset())); __ resolve_oop_handle(rax, rscratch1); diff --git a/src/hotspot/cpu/x86/templateTable_x86.cpp b/src/hotspot/cpu/x86/templateTable_x86.cpp index 8b6c36365f7..99c184c3379 100644 --- a/src/hotspot/cpu/x86/templateTable_x86.cpp +++ b/src/hotspot/cpu/x86/templateTable_x86.cpp @@ -2592,8 +2592,11 @@ void TemplateTable::_return(TosState state) { #endif __ jcc(Assembler::zero, no_safepoint); __ push(state); + __ push_cont_fastpath(NOT_LP64(thread) LP64_ONLY(r15_thread)); __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::at_safepoint)); + NOT_LP64(__ get_thread(thread);) + __ pop_cont_fastpath(NOT_LP64(thread) LP64_ONLY(r15_thread)); __ pop(state); __ bind(no_safepoint); } @@ -4361,6 +4364,11 @@ void TemplateTable::monitorenter() { __ movptr(Address(rmon, BasicObjectLock::obj_offset_in_bytes()), rax); __ lock_object(rmon); + // The object is stored so counter should be increased even if stackoverflow is generated + Register rthread = LP64_ONLY(r15_thread) NOT_LP64(rbx); + NOT_LP64(__ get_thread(rthread);) + __ inc_held_monitor_count(rthread); + // check to make sure this monitor doesn't cause stack overflow after locking __ save_bcp(); // in case of exception __ generate_stack_overflow_check(0); @@ -4419,6 +4427,11 @@ void TemplateTable::monitorexit() { __ bind(found); __ push_ptr(rax); // make sure object is on stack (contract with oopMaps) __ unlock_object(rtop); + + Register rthread = LP64_ONLY(r15_thread) NOT_LP64(rax); + NOT_LP64(__ get_thread(rthread);) + __ dec_held_monitor_count(rthread); + __ pop_ptr(rax); // discard object } diff --git a/src/hotspot/cpu/x86/vm_version_x86.hpp b/src/hotspot/cpu/x86/vm_version_x86.hpp index c9676d3e0b5..ed1b8b43459 100644 --- a/src/hotspot/cpu/x86/vm_version_x86.hpp +++ b/src/hotspot/cpu/x86/vm_version_x86.hpp @@ -163,7 +163,10 @@ class VM_Version : public Abstract_VM_Version { mmx_amd : 1, mmx : 1, fxsr : 1, - : 4, + fxsr_opt : 1, + pdpe1gb : 1, + rdtscp : 1, + : 1, long_mode : 1, tdnow2 : 1, tdnow : 1; @@ -250,7 +253,11 @@ class VM_Version : public Abstract_VM_Version { avx512_bitalg : 1, : 1, avx512_vpopcntdq : 1, - : 17; + : 1, + : 1, + mawau : 5, + rdpid : 1, + : 9; } bits; }; @@ -260,7 +267,8 @@ class VM_Version : public Abstract_VM_Version { uint32_t : 2, avx512_4vnniw : 1, avx512_4fmaps : 1, - : 10, + fast_short_rep_mov : 1, + : 9, serialize : 1, : 17; } bits; @@ -361,7 +369,11 @@ protected: decl(AVX512_VBMI2, "avx512_vbmi2", 44) /* VBMI2 shift left double instructions */ \ decl(AVX512_VBMI, "avx512_vbmi", 45) /* Vector BMI instructions */ \ decl(HV, "hv", 46) /* Hypervisor instructions */ \ - decl(SERIALIZE, "serialize", 47) /* CPU SERIALIZE */ + decl(SERIALIZE, "serialize", 47) /* CPU SERIALIZE */ \ + \ + decl(RDTSCP, "rdtscp", 48) /* RDTSCP instruction */ \ + decl(RDPID, "rdpid", 49) /* RDPID instruction */ \ + decl(FSRM, "fsrm", 50) /* Fast Short REP MOV */ #define DECLARE_CPU_FEATURE_FLAG(id, name, bit) CPU_##id = (1ULL << bit), CPU_FEATURE_FLAGS(DECLARE_CPU_FEATURE_FLAG) @@ -611,6 +623,8 @@ protected: result |= CPU_AES; if (_cpuid_info.sef_cpuid7_ebx.bits.erms != 0) result |= CPU_ERMS; + if (_cpuid_info.sef_cpuid7_edx.bits.fast_short_rep_mov != 0) + result |= CPU_FSRM; if (_cpuid_info.std_cpuid1_ecx.bits.clmul != 0) result |= CPU_CLMUL; if (_cpuid_info.sef_cpuid7_ebx.bits.rtm != 0) @@ -625,6 +639,10 @@ protected: result |= CPU_FMA; if (_cpuid_info.sef_cpuid7_ebx.bits.clflushopt != 0) result |= CPU_FLUSHOPT; + if (_cpuid_info.ext_cpuid1_edx.bits.rdtscp != 0) + result |= CPU_RDTSCP; + if (_cpuid_info.sef_cpuid7_ecx.bits.rdpid != 0) + result |= CPU_RDPID; // AMD|Hygon features. if (is_amd_family()) { @@ -868,8 +886,11 @@ public: static bool supports_avx() { return (_features & CPU_AVX) != 0; } static bool supports_avx2() { return (_features & CPU_AVX2) != 0; } static bool supports_tsc() { return (_features & CPU_TSC) != 0; } + static bool supports_rdtscp() { return (_features & CPU_RDTSCP) != 0; } + static bool supports_rdpid() { return (_features & CPU_RDPID) != 0; } static bool supports_aes() { return (_features & CPU_AES) != 0; } static bool supports_erms() { return (_features & CPU_ERMS) != 0; } + static bool supports_fsrm() { return (_features & CPU_FSRM) != 0; } static bool supports_clmul() { return (_features & CPU_CLMUL) != 0; } static bool supports_rtm() { return (_features & CPU_RTM) != 0; } static bool supports_bmi1() { return (_features & CPU_BMI1) != 0; } diff --git a/src/hotspot/cpu/x86/x86.ad b/src/hotspot/cpu/x86/x86.ad index 72eabaec692..92b790eb768 100644 --- a/src/hotspot/cpu/x86/x86.ad +++ b/src/hotspot/cpu/x86/x86.ad @@ -2664,10 +2664,10 @@ static inline jlong high_bit_set(BasicType bt) { encode %{ enc_class call_epilog %{ + C2_MacroAssembler _masm(&cbuf); if (VerifyStackAtCalls) { // Check that stack depth is unchanged: find majik cookie on stack int framesize = ra_->reg2offset_unchecked(OptoReg::add(ra_->_matcher._old_SP, -3*VMRegImpl::slots_per_word)); - C2_MacroAssembler _masm(&cbuf); Label L; __ cmpptr(Address(rsp, framesize), (int32_t)0xbadb100d); __ jccb(Assembler::equal, L); diff --git a/src/hotspot/cpu/x86/x86_32.ad b/src/hotspot/cpu/x86/x86_32.ad index 00283e21743..d7409cddac1 100644 --- a/src/hotspot/cpu/x86/x86_32.ad +++ b/src/hotspot/cpu/x86/x86_32.ad @@ -1703,11 +1703,13 @@ encode %{ enc_class Java_To_Runtime (method meth) %{ // CALL Java_To_Runtime, Java_To_Runtime_Leaf // This is the instruction starting address for relocation info. + MacroAssembler _masm(&cbuf); cbuf.set_insts_mark(); $$$emit8$primary; // CALL directly to the runtime emit_d32_reloc(cbuf, ($meth$$method - (int)(cbuf.insts_end()) - 4), runtime_call_Relocation::spec(), RELOC_IMM32 ); + __ post_call_nop(); if (UseSSE >= 2) { MacroAssembler _masm(&cbuf); @@ -1759,6 +1761,7 @@ encode %{ enc_class Java_Static_Call (method meth) %{ // JAVA STATIC CALL // CALL to fixup routine. Fixup routine uses ScopeDesc info to determine // who we intended to call. + MacroAssembler _masm(&cbuf); cbuf.set_insts_mark(); $$$emit8$primary; @@ -1766,12 +1769,14 @@ encode %{ emit_d32_reloc(cbuf, ($meth$$method - (int)(cbuf.insts_end()) - 4), runtime_call_Relocation::spec(), RELOC_IMM32); + __ post_call_nop(); } else { int method_index = resolved_method_index(cbuf); RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index) : static_call_Relocation::spec(method_index); emit_d32_reloc(cbuf, ($meth$$method - (int)(cbuf.insts_end()) - 4), rspec, RELOC_DISP32); + __ post_call_nop(); // Emit stubs for static call. address stub = CompiledStaticCall::emit_to_interp_stub(cbuf); if (stub == NULL) { @@ -1784,6 +1789,7 @@ encode %{ enc_class Java_Dynamic_Call (method meth) %{ // JAVA DYNAMIC CALL MacroAssembler _masm(&cbuf); __ ic_call((address)$meth$$method, resolved_method_index(cbuf)); + __ post_call_nop(); %} enc_class Java_Compiled_Call (method meth) %{ // JAVA COMPILED CALL @@ -1791,11 +1797,12 @@ encode %{ assert( -128 <= disp && disp <= 127, "compiled_code_offset isn't small"); // CALL *[EAX+in_bytes(Method::from_compiled_code_entry_point_offset())] + MacroAssembler _masm(&cbuf); cbuf.set_insts_mark(); $$$emit8$primary; emit_rm(cbuf, 0x01, $secondary, EAX_enc ); // R/M byte emit_d8(cbuf, disp); // Displacement - + __ post_call_nop(); %} // Following encoding is no longer used, but may be restored if calling @@ -2737,9 +2744,11 @@ encode %{ // PUSH src2.lo emit_opcode(cbuf, 0x50+$src2$$reg ); // CALL directly to the runtime + MacroAssembler _masm(&cbuf); cbuf.set_insts_mark(); emit_opcode(cbuf,0xE8); // Call into runtime emit_d32_reloc(cbuf, (CAST_FROM_FN_PTR(address, SharedRuntime::ldiv) - cbuf.insts_end()) - 4, runtime_call_Relocation::spec(), RELOC_IMM32 ); + __ post_call_nop(); // Restore stack emit_opcode(cbuf, 0x83); // add SP, #framesize emit_rm(cbuf, 0x3, 0x00, ESP_enc); @@ -2756,9 +2765,11 @@ encode %{ // PUSH src2.lo emit_opcode(cbuf, 0x50+$src2$$reg ); // CALL directly to the runtime + MacroAssembler _masm(&cbuf); cbuf.set_insts_mark(); emit_opcode(cbuf,0xE8); // Call into runtime emit_d32_reloc(cbuf, (CAST_FROM_FN_PTR(address, SharedRuntime::lrem ) - cbuf.insts_end()) - 4, runtime_call_Relocation::spec(), RELOC_IMM32 ); + __ post_call_nop(); // Restore stack emit_opcode(cbuf, 0x83); // add SP, #framesize emit_rm(cbuf, 0x3, 0x00, ESP_enc); @@ -2826,10 +2837,12 @@ encode %{ %} enc_class enc_rethrow() %{ + MacroAssembler _masm(&cbuf); cbuf.set_insts_mark(); emit_opcode(cbuf, 0xE9); // jmp entry emit_d32_reloc(cbuf, (int)OptoRuntime::rethrow_stub() - ((int)cbuf.insts_end())-4, runtime_call_Relocation::spec(), RELOC_IMM32 ); + __ post_call_nop(); %} @@ -2875,9 +2888,11 @@ encode %{ emit_opcode(cbuf,0xD9 ); // FLD ST(i) emit_d8 (cbuf,0xC0-1+$src$$reg ); // CALL directly to the runtime + MacroAssembler _masm(&cbuf); cbuf.set_insts_mark(); emit_opcode(cbuf,0xE8); // Call into runtime emit_d32_reloc(cbuf, (StubRoutines::x86::d2i_wrapper() - cbuf.insts_end()) - 4, runtime_call_Relocation::spec(), RELOC_IMM32 ); + __ post_call_nop(); // Carry on here... %} @@ -2917,9 +2932,11 @@ encode %{ emit_opcode(cbuf,0xD9 ); // FLD ST(i) emit_d8 (cbuf,0xC0-1+$src$$reg ); // CALL directly to the runtime + MacroAssembler _masm(&cbuf); cbuf.set_insts_mark(); emit_opcode(cbuf,0xE8); // Call into runtime emit_d32_reloc(cbuf, (StubRoutines::x86::d2l_wrapper() - cbuf.insts_end()) - 4, runtime_call_Relocation::spec(), RELOC_IMM32 ); + __ post_call_nop(); // Carry on here... %} @@ -10770,6 +10787,7 @@ instruct convD2I_reg_reg( eAXRegI dst, eDXRegI tmp, regD src, eFlagsReg cr ) %{ __ fld_d(Address(rsp, 0)); __ addptr(rsp, 8); __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2i_wrapper()))); + __ post_call_nop(); __ bind(fast); %} ins_pipe( pipe_slow ); @@ -10845,6 +10863,7 @@ instruct convD2L_reg_reg( eADXRegL dst, regD src, eFlagsReg cr ) %{ __ fld_d(Address(rsp, 0)); __ addptr(rsp, 8); __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2l_wrapper()))); + __ post_call_nop(); __ bind(fast); %} ins_pipe( pipe_slow ); @@ -10900,6 +10919,7 @@ instruct convF2I_reg(eAXRegI dst, eDXRegI tmp, regF src, eFlagsReg cr ) %{ __ fld_s(Address(rsp, 0)); __ addptr(rsp, 4); __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2i_wrapper()))); + __ post_call_nop(); __ bind(fast); %} ins_pipe( pipe_slow ); @@ -10976,6 +10996,7 @@ instruct convF2L_reg_reg( eADXRegL dst, regF src, eFlagsReg cr ) %{ __ fld_s(Address(rsp, 0)); __ addptr(rsp, 4); __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2l_wrapper()))); + __ post_call_nop(); __ bind(fast); %} ins_pipe( pipe_slow ); diff --git a/src/hotspot/cpu/x86/x86_64.ad b/src/hotspot/cpu/x86/x86_64.ad index c81605a73d7..b9e67caa11f 100644 --- a/src/hotspot/cpu/x86/x86_64.ad +++ b/src/hotspot/cpu/x86/x86_64.ad @@ -2051,19 +2051,7 @@ encode %{ MacroAssembler _masm(&cbuf); __ mov64(r10, (int64_t) $meth$$method); __ call(r10); - %} - - enc_class Java_To_Interpreter(method meth) - %{ - // CALL Java_To_Interpreter - // This is the instruction starting address for relocation info. - cbuf.set_insts_mark(); - $$$emit8$primary; - // CALL directly to the runtime - emit_d32_reloc(cbuf, - (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4), - runtime_call_Relocation::spec(), - RELOC_DISP32); + __ post_call_nop(); %} enc_class Java_Static_Call(method meth) @@ -2071,6 +2059,7 @@ encode %{ // JAVA STATIC CALL // CALL to fixup routine. Fixup routine uses ScopeDesc info to // determine who we intended to call. + MacroAssembler _masm(&cbuf); cbuf.set_insts_mark(); $$$emit8$primary; @@ -2092,31 +2081,14 @@ encode %{ return; } } + _masm.clear_inst_mark(); + __ post_call_nop(); %} enc_class Java_Dynamic_Call(method meth) %{ MacroAssembler _masm(&cbuf); __ ic_call((address)$meth$$method, resolved_method_index(cbuf)); - %} - - enc_class Java_Compiled_Call(method meth) - %{ - // JAVA COMPILED CALL - int disp = in_bytes(Method:: from_compiled_offset()); - - // XXX XXX offset is 128 is 1.5 NON-PRODUCT !!! - // assert(-0x80 <= disp && disp < 0x80, "compiled_code_offset isn't small"); - - // callq *disp(%rax) - cbuf.set_insts_mark(); - $$$emit8$primary; - if (disp < 0x80) { - emit_rm(cbuf, 0x01, $secondary, RAX_enc); // R/M byte - emit_d8(cbuf, disp); // Displacement - } else { - emit_rm(cbuf, 0x02, $secondary, RAX_enc); // R/M byte - emit_d32(cbuf, disp); // Displacement - } + __ post_call_nop(); %} enc_class reg_opc_imm(rRegI dst, immI8 shift) diff --git a/src/hotspot/cpu/zero/continuationEntry_zero.inline.hpp b/src/hotspot/cpu/zero/continuationEntry_zero.inline.hpp new file mode 100644 index 00000000000..c7e6aabc4db --- /dev/null +++ b/src/hotspot/cpu/zero/continuationEntry_zero.inline.hpp @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_ZERO_CONTINUATIONENTRY_ZERO_INLINE_HPP +#define CPU_ZERO_CONTINUATIONENTRY_ZERO_INLINE_HPP + +#include "runtime/continuationEntry.hpp" + +// TODO: Implement + +inline frame ContinuationEntry::to_frame() const { + Unimplemented(); + return frame(); +} + +inline intptr_t* ContinuationEntry::entry_fp() const { + Unimplemented(); + return nullptr; +} + +inline void ContinuationEntry::update_register_map(RegisterMap* map) const { + Unimplemented(); +} + +#endif // CPU_ZERO_CONTINUATIONENTRY_ZERO_INLINE_HPP + diff --git a/src/hotspot/cpu/zero/continuationFreezeThaw_zero.inline.hpp b/src/hotspot/cpu/zero/continuationFreezeThaw_zero.inline.hpp new file mode 100644 index 00000000000..76a7d4454ba --- /dev/null +++ b/src/hotspot/cpu/zero/continuationFreezeThaw_zero.inline.hpp @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_ZERO_CONTINUATION_ZERO_INLINE_HPP +#define CPU_ZERO_CONTINUATION_ZERO_INLINE_HPP + +#include "oops/stackChunkOop.inline.hpp" +#include "runtime/frame.hpp" +#include "runtime/frame.inline.hpp" + +inline void FreezeBase::set_top_frame_metadata_pd(const frame& hf) { + Unimplemented(); +} + +template +inline frame FreezeBase::sender(const frame& f) { + Unimplemented(); + return frame(); +} + +template frame FreezeBase::new_heap_frame(frame& f, frame& caller) { + Unimplemented(); + return frame(); +} + +void FreezeBase::adjust_interpreted_frame_unextended_sp(frame& f) { + Unimplemented(); +} + +inline void FreezeBase::relativize_interpreted_frame_metadata(const frame& f, const frame& hf) { + Unimplemented(); +} + +inline void FreezeBase::patch_pd(frame& hf, const frame& caller) { + Unimplemented(); +} + +inline void FreezeBase::patch_stack_pd(intptr_t* frame_sp, intptr_t* heap_sp) { + Unimplemented(); +} + +inline frame ThawBase::new_entry_frame() { + Unimplemented(); + return frame(); +} + +template frame ThawBase::new_stack_frame(const frame& hf, frame& caller, bool bottom) { + Unimplemented(); + return frame(); +} + +inline void ThawBase::set_interpreter_frame_bottom(const frame& f, intptr_t* bottom) { + Unimplemented(); +} + +inline void ThawBase::derelativize_interpreted_frame_metadata(const frame& hf, const frame& f) { + Unimplemented(); +} + +inline intptr_t* ThawBase::align(const frame& hf, intptr_t* frame_sp, frame& caller, bool bottom) { + Unimplemented(); + return NULL; +} + +inline void ThawBase::patch_pd(frame& f, const frame& caller) { + Unimplemented(); +} + +void ThawBase::patch_chunk_pd(intptr_t* sp) { + Unimplemented(); +} + +inline void ThawBase::prefetch_chunk_pd(void* start, int size) { + Unimplemented(); +} + +#endif // CPU_ZERO_CONTINUATION_ZERO_INLINE_HPP diff --git a/src/hotspot/cpu/zero/continuationHelper_zero.inline.hpp b/src/hotspot/cpu/zero/continuationHelper_zero.inline.hpp new file mode 100644 index 00000000000..c04e128f04e --- /dev/null +++ b/src/hotspot/cpu/zero/continuationHelper_zero.inline.hpp @@ -0,0 +1,123 @@ +/* + * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_ZERO_CONTINUATIONHELPER_ZERO_INLINE_HPP +#define CPU_ZERO_CONTINUATIONHELPER_ZERO_INLINE_HPP + +#include "runtime/continuationHelper.hpp" + +template // TODO: maybe do the same CRTP trick with Interpreted and Compiled as with hframe +static inline intptr_t** link_address(const frame& f) { + Unimplemented(); + return NULL; +} + +inline int ContinuationHelper::frame_align_words(int size) { + Unimplemented(); + return 0; +} + +inline intptr_t* ContinuationHelper::frame_align_pointer(intptr_t* sp) { + Unimplemented(); + return NULL; +} + +template +inline void ContinuationHelper::update_register_map(const frame& f, RegisterMap* map) { + Unimplemented(); +} + +inline void ContinuationHelper::update_register_map_with_callee(const frame& f, RegisterMap* map) { + Unimplemented(); +} + +inline void ContinuationHelper::push_pd(const frame& f) { + Unimplemented(); +} + +inline void ContinuationHelper::set_anchor_to_entry_pd(JavaFrameAnchor* anchor, ContinuationEntry* cont) { + Unimplemented(); +} + +#ifdef ASSERT +inline void ContinuationHelper::set_anchor_pd(JavaFrameAnchor* anchor, intptr_t* sp) { + Unimplemented(); +} + +inline bool ContinuationHelper::Frame::assert_frame_laid_out(frame f) { + Unimplemented(); + return false; +} +#endif + +inline intptr_t** ContinuationHelper::Frame::callee_link_address(const frame& f) { + Unimplemented(); + return NULL; +} + +template +static inline intptr_t* real_fp(const frame& f) { + Unimplemented(); + return NULL; +} + +inline address* ContinuationHelper::InterpretedFrame::return_pc_address(const frame& f) { + Unimplemented(); + return NULL; +} + +inline void ContinuationHelper::InterpretedFrame::patch_sender_sp(frame& f, intptr_t* sp) { + Unimplemented(); +} + +inline address* ContinuationHelper::Frame::return_pc_address(const frame& f) { + Unimplemented(); + return NULL; +} + +inline address ContinuationHelper::Frame::real_pc(const frame& f) { + Unimplemented(); + return NULL; +} + +inline void ContinuationHelper::Frame::patch_pc(const frame& f, address pc) { + Unimplemented(); +} + +inline intptr_t* ContinuationHelper::InterpretedFrame::frame_top(const frame& f, InterpreterOopMap* mask) { // inclusive; this will be copied with the frame + Unimplemented(); + return NULL; +} + +inline intptr_t* ContinuationHelper::InterpretedFrame::frame_bottom(const frame& f) { // exclusive; this will not be copied with the frame + Unimplemented(); + return NULL; +} + +inline intptr_t* ContinuationHelper::InterpretedFrame::frame_top(const frame& f, int callee_argsize, bool callee_interpreted) { + Unimplemented(); + return NULL; +} + +#endif // CPU_ZERO_CONTINUATIONHELPER_ZERO_INLINE_HPP diff --git a/src/hotspot/cpu/zero/frame_zero.cpp b/src/hotspot/cpu/zero/frame_zero.cpp index 2e00d703377..6956c3e685e 100644 --- a/src/hotspot/cpu/zero/frame_zero.cpp +++ b/src/hotspot/cpu/zero/frame_zero.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2003, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2003, 2022, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2007, 2021, Red Hat, Inc. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * @@ -78,26 +78,11 @@ frame frame::sender_for_nonentry_frame(RegisterMap *map) const { return frame(zeroframe()->next(), sender_sp()); } -frame frame::sender(RegisterMap* map) const { - // Default is not to follow arguments; the various - // sender_for_xxx methods update this accordingly. - map->set_include_argument_oops(false); - - frame result = zeroframe()->is_entry_frame() ? - sender_for_entry_frame(map) : - sender_for_nonentry_frame(map); - - if (map->process_frames()) { - StackWatermarkSet::on_iteration(map->thread(), result); - } - - return result; -} - BasicObjectLock* frame::interpreter_frame_monitor_begin() const { return get_interpreterState()->monitor_base(); } +// Pointer beyond the "oldest/deepest" BasicObjectLock on stack. BasicObjectLock* frame::interpreter_frame_monitor_end() const { return (BasicObjectLock*) get_interpreterState()->stack_base(); } @@ -235,13 +220,6 @@ BasicType frame::interpreter_frame_result(oop* oop_result, return type; } -int frame::frame_size(RegisterMap* map) const { -#ifdef PRODUCT - ShouldNotCallThis(); -#endif // PRODUCT - return 0; // make javaVFrame::print_value work -} - intptr_t* frame::interpreter_frame_tos_at(jint offset) const { int index = (Interpreter::expr_offset_in_bytes(offset) / wordSize); return &interpreter_frame_tos_address()[index]; diff --git a/src/hotspot/cpu/zero/frame_zero.hpp b/src/hotspot/cpu/zero/frame_zero.hpp index 8c229143a15..bc8e45543a1 100644 --- a/src/hotspot/cpu/zero/frame_zero.hpp +++ b/src/hotspot/cpu/zero/frame_zero.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2003, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2003, 2022, Oracle and/or its affiliates. All rights reserved. * Copyright 2007, 2008, 2009, 2010 Red Hat, Inc. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * @@ -30,9 +30,15 @@ public: enum { - pc_return_offset = 0 + pc_return_offset = 0, + metadata_words = 0, + frame_alignment = 16, + // size, in words, of maximum shift in frame position due to alignment + align_wiggle = 1 }; + const ImmutableOopMap* get_oop_map() const; + // Constructor public: frame(ZeroFrame* zeroframe, intptr_t* sp); @@ -73,4 +79,11 @@ static jint interpreter_frame_expression_stack_direction() { return -1; } + inline address* sender_pc_addr() const; + + inline intptr_t* interpreter_frame_last_sp() const; + + template + static void update_map_with_saved_link(RegisterMapT* map, intptr_t** link_addr); + #endif // CPU_ZERO_FRAME_ZERO_HPP diff --git a/src/hotspot/cpu/zero/frame_zero.inline.hpp b/src/hotspot/cpu/zero/frame_zero.inline.hpp index dfca0e4bcb1..bec224337e8 100644 --- a/src/hotspot/cpu/zero/frame_zero.inline.hpp +++ b/src/hotspot/cpu/zero/frame_zero.inline.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2003, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2003, 2022, Oracle and/or its affiliates. All rights reserved. * Copyright 2007, 2008, 2009, 2010 Red Hat, Inc. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * @@ -36,13 +36,21 @@ inline frame::frame() { _pc = NULL; _cb = NULL; _deopt_state = unknown; + _on_heap = false; + DEBUG_ONLY(_frame_index = -1;) } inline address frame::sender_pc() const { ShouldNotCallThis(); return NULL; } +inline frame::frame(intptr_t* sp) { + Unimplemented(); +} + inline frame::frame(ZeroFrame* zf, intptr_t* sp) { _zeroframe = zf; _sp = sp; + _on_heap = false; + DEBUG_ONLY(_frame_index = -1;) switch (zeroframe()->type()) { case ZeroFrame::ENTRY_FRAME: _pc = StubRoutines::call_stub_return_pc(); @@ -168,4 +176,77 @@ inline intptr_t* frame::unextended_sp() const { return (intptr_t *) -1; } +inline const ImmutableOopMap* frame::get_oop_map() const { + Unimplemented(); + return NULL; +} + +inline int frame::compiled_frame_stack_argsize() const { + Unimplemented(); + return 0; +} + +inline void frame::interpreted_frame_oop_map(InterpreterOopMap* mask) const { + Unimplemented(); +} + +inline intptr_t* frame::interpreter_frame_last_sp() const { + Unimplemented(); + return NULL; +} + +inline int frame::sender_sp_ret_address_offset() { + Unimplemented(); + return 0; +} + +inline void frame::set_unextended_sp(intptr_t* value) { + Unimplemented(); +} + +inline int frame::offset_unextended_sp() const { + Unimplemented(); + return 0; +} + +inline void frame::set_offset_unextended_sp(int value) { + Unimplemented(); +} + +inline int frame::frame_size() const { +#ifdef PRODUCT + ShouldNotCallThis(); +#endif // PRODUCT + return 0; // make javaVFrame::print_value work +} + +inline address* frame::sender_pc_addr() const { + ShouldNotCallThis(); + return NULL; +} + +//------------------------------------------------------------------------------ +// frame::sender + +inline frame frame::sender(RegisterMap* map) const { + // Default is not to follow arguments; the various + // sender_for_xxx methods update this accordingly. + map->set_include_argument_oops(false); + + frame result = zeroframe()->is_entry_frame() ? + sender_for_entry_frame(map) : + sender_for_nonentry_frame(map); + + if (map->process_frames()) { + StackWatermarkSet::on_iteration(map->thread(), result); + } + + return result; +} + +template +void frame::update_map_with_saved_link(RegisterMapT* map, intptr_t** link_addr) { + Unimplemented(); +} + #endif // CPU_ZERO_FRAME_ZERO_INLINE_HPP diff --git a/src/hotspot/cpu/zero/nativeInst_zero.hpp b/src/hotspot/cpu/zero/nativeInst_zero.hpp index 9979ac07dea..9cafe5ca700 100644 --- a/src/hotspot/cpu/zero/nativeInst_zero.hpp +++ b/src/hotspot/cpu/zero/nativeInst_zero.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2003, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2003, 2022, Oracle and/or its affiliates. All rights reserved. * Copyright 2007 Red Hat, Inc. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * @@ -41,6 +41,7 @@ // - - NativeReturnX (return with argument) // - - NativePushConst // - - NativeTstRegMem +// - - NativeDeoptInstruction // The base class for different kinds of native instruction abstractions. // Provides the primitive operations to manipulate code relative to this. @@ -210,4 +211,35 @@ inline NativeGeneralJump* nativeGeneralJump_at(address address) { return NULL; } +class NativePostCallNop: public NativeInstruction { +public: + bool check() const { Unimplemented(); return false; } + int displacement() const { Unimplemented(); return 0; } + void patch(jint diff) { Unimplemented(); } + void make_deopt() { Unimplemented(); } +}; + +inline NativePostCallNop* nativePostCallNop_at(address address) { + Unimplemented(); + return NULL; +} + +class NativeDeoptInstruction: public NativeInstruction { +public: + address instruction_address() const { Unimplemented(); return NULL; } + address next_instruction_address() const { Unimplemented(); return NULL; } + + void verify() { Unimplemented(); } + + static bool is_deopt_at(address instr) { + Unimplemented(); + return false; + } + + // MT-safe patching + static void insert(address code_pos) { + Unimplemented(); + } +}; + #endif // CPU_ZERO_NATIVEINST_ZERO_HPP diff --git a/src/hotspot/cpu/zero/registerMap_zero.hpp b/src/hotspot/cpu/zero/registerMap_zero.hpp index d832d681b72..d8c5809543d 100644 --- a/src/hotspot/cpu/zero/registerMap_zero.hpp +++ b/src/hotspot/cpu/zero/registerMap_zero.hpp @@ -35,7 +35,7 @@ address pd_location(VMReg reg) const { return NULL; } address pd_location(VMReg base_reg, int slot_idx) const { - return location(base_reg->next(slot_idx)); + return location(base_reg->next(slot_idx), nullptr); } // no PD state to clear or copy: diff --git a/src/hotspot/cpu/zero/smallRegisterMap_zero.inline.hpp b/src/hotspot/cpu/zero/smallRegisterMap_zero.inline.hpp new file mode 100644 index 00000000000..43328fac76c --- /dev/null +++ b/src/hotspot/cpu/zero/smallRegisterMap_zero.inline.hpp @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_ZERO_SMALLREGISTERMAP_ZERO_INLINE_HPP +#define CPU_ZERO_SMALLREGISTERMAP_ZERO_INLINE_HPP + +#include "runtime/frame.inline.hpp" +#include "runtime/registerMap.hpp" + +// Java frames don't have callee saved registers (except for rfp), so we can use a smaller RegisterMap +class SmallRegisterMap { +public: + static constexpr SmallRegisterMap* instance = nullptr; +private: + static void assert_is_rfp(VMReg r) PRODUCT_RETURN + DEBUG_ONLY({ Unimplemented(); }) +public: + // as_RegisterMap is used when we didn't want to templatize and abstract over RegisterMap type to support SmallRegisterMap + // Consider enhancing SmallRegisterMap to support those cases + const RegisterMap* as_RegisterMap() const { return nullptr; } + RegisterMap* as_RegisterMap() { return nullptr; } + + RegisterMap* copy_to_RegisterMap(RegisterMap* map, intptr_t* sp) const { + Unimplemented(); + return map; + } + + SmallRegisterMap() {} + + SmallRegisterMap(const RegisterMap* map) { + Unimplemented(); + } + + inline address location(VMReg reg, intptr_t* sp) const { + Unimplemented(); + return NULL; + } + + inline void set_location(VMReg reg, address loc) { assert_is_rfp(reg); } + + JavaThread* thread() const { + #ifndef ASSERT + guarantee (false, ""); + #endif + return nullptr; + } + + bool update_map() const { return false; } + bool walk_cont() const { return false; } + bool include_argument_oops() const { return false; } + void set_include_argument_oops(bool f) {} + bool in_cont() const { return false; } + stackChunkHandle stack_chunk() const { return stackChunkHandle(); } + +#ifdef ASSERT + bool should_skip_missing() const { return false; } + VMReg find_register_spilled_here(void* p, intptr_t* sp) { + Unimplemented(); + return NULL; + } + void print() const { print_on(tty); } + void print_on(outputStream* st) const { st->print_cr("Small register map"); } +#endif +}; + +#endif // CPU_ZERO_SMALLREGISTERMAP_ZERO_INLINE_HPP diff --git a/src/hotspot/cpu/zero/stackChunkFrameStream_zero.inline.hpp b/src/hotspot/cpu/zero/stackChunkFrameStream_zero.inline.hpp new file mode 100644 index 00000000000..64eb4e00ced --- /dev/null +++ b/src/hotspot/cpu/zero/stackChunkFrameStream_zero.inline.hpp @@ -0,0 +1,115 @@ +/* + * Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_ZERO_STACKCHUNKFRAMESTREAM_ZERO_INLINE_HPP +#define CPU_ZERO_STACKCHUNKFRAMESTREAM_ZERO_INLINE_HPP + +#include "interpreter/oopMapCache.hpp" +#include "runtime/frame.inline.hpp" +#include "runtime/registerMap.hpp" + +#ifdef ASSERT +template +inline bool StackChunkFrameStream::is_in_frame(void* p0) const { + Unimplemented(); + return true; +} +#endif + +template +inline frame StackChunkFrameStream::to_frame() const { + Unimplemented(); + return frame(); +} + +template +inline address StackChunkFrameStream::get_pc() const { + Unimplemented(); + return NULL; +} + +template +inline intptr_t* StackChunkFrameStream::fp() const { + Unimplemented(); + return NULL; +} + +template +inline intptr_t* StackChunkFrameStream::derelativize(int offset) const { + Unimplemented(); + return NULL; +} + +template +inline intptr_t* StackChunkFrameStream::unextended_sp_for_interpreter_frame() const { + Unimplemented(); + return NULL; +} + +template +intptr_t* StackChunkFrameStream::next_sp_for_interpreter_frame() const { + Unimplemented(); + return NULL; +} + +template +inline void StackChunkFrameStream::next_for_interpreter_frame() { + Unimplemented(); +} + +template +inline int StackChunkFrameStream::interpreter_frame_size() const { + Unimplemented(); + return 0; +} + +template +inline int StackChunkFrameStream::interpreter_frame_stack_argsize() const { + Unimplemented(); + return 0; +} + +template +inline int StackChunkFrameStream::interpreter_frame_num_oops() const { + Unimplemented(); + return 0; +} + +template<> +template<> +inline void StackChunkFrameStream::update_reg_map_pd(RegisterMap* map) { + Unimplemented(); +} + +template<> +template<> +inline void StackChunkFrameStream::update_reg_map_pd(RegisterMap* map) { + Unimplemented(); +} + +template +template +inline void StackChunkFrameStream::update_reg_map_pd(RegisterMapT* map) {} + +#endif // CPU_ZERO_STACKCHUNKFRAMESTREAM_ZERO_INLINE_HPP diff --git a/src/hotspot/cpu/zero/stackChunkOop_zero.inline.hpp b/src/hotspot/cpu/zero/stackChunkOop_zero.inline.hpp new file mode 100644 index 00000000000..9b8c0aa6a9b --- /dev/null +++ b/src/hotspot/cpu/zero/stackChunkOop_zero.inline.hpp @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef CPU_ZERO_STACKCHUNKOOP_ZERO_INLINE_HPP +#define CPU_ZERO_STACKCHUNKOOP_ZERO_INLINE_HPP + +inline void stackChunkOopDesc::relativize_frame_pd(frame& fr) const { + Unimplemented(); +} + +inline void stackChunkOopDesc::derelativize_frame_pd(frame& fr) const { + Unimplemented(); +} + +#endif // CPU_ZERO_STACKCHUNKOOP_ZERO_INLINE_HPP diff --git a/src/hotspot/cpu/zero/stubGenerator_zero.cpp b/src/hotspot/cpu/zero/stubGenerator_zero.cpp index dbd31dbb570..4c7844deb87 100644 --- a/src/hotspot/cpu/zero/stubGenerator_zero.cpp +++ b/src/hotspot/cpu/zero/stubGenerator_zero.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2003, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2003, 2022, Oracle and/or its affiliates. All rights reserved. * Copyright 2007, 2008, 2010, 2015 Red Hat, Inc. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * @@ -234,8 +234,8 @@ class StubGenerator: public StubCodeGenerator { } }; -void StubGenerator_generate(CodeBuffer* code, bool all) { - StubGenerator g(code, all); +void StubGenerator_generate(CodeBuffer* code, int phase) { + StubGenerator g(code, phase); } EntryFrame *EntryFrame::build(const intptr_t* parameters, diff --git a/src/hotspot/os/posix/os_posix.cpp b/src/hotspot/os/posix/os_posix.cpp index 0ed4713a3db..9dbebcafc71 100644 --- a/src/hotspot/os/posix/os_posix.cpp +++ b/src/hotspot/os/posix/os_posix.cpp @@ -1029,7 +1029,8 @@ bool os::Posix::handle_stack_overflow(JavaThread* thread, address addr, address if (thread->thread_state() == _thread_in_Java) { #ifndef ARM // arm32 doesn't have this - if (overflow_state->in_stack_reserved_zone(addr)) { + // vthreads don't support this + if (!thread->is_vthread_mounted() && overflow_state->in_stack_reserved_zone(addr)) { frame fr; if (get_frame_at_stack_banging_point(thread, pc, ucVoid, &fr)) { assert(fr.is_java_frame(), "Must be a Java frame"); diff --git a/src/hotspot/os/posix/signals_posix.cpp b/src/hotspot/os/posix/signals_posix.cpp index 1a9d7da9158..adcc644df8d 100644 --- a/src/hotspot/os/posix/signals_posix.cpp +++ b/src/hotspot/os/posix/signals_posix.cpp @@ -25,6 +25,9 @@ #include "precompiled.hpp" #include "jvm.h" +#include "code/codeCache.hpp" +#include "code/compiledMethod.hpp" +#include "code/nativeInst.hpp" #include "logging/log.hpp" #include "runtime/atomic.hpp" #include "runtime/globals.hpp" @@ -622,6 +625,31 @@ int JVM_HANDLE_XXX_SIGNAL(int sig, siginfo_t* info, signal_was_handled = true; // unconditionally. } + // Check for UD trap caused by NOP patching. + // If it is, patch return address to be deopt handler. + if (!signal_was_handled) { + address pc = os::Posix::ucontext_get_pc(uc); + assert(pc != NULL, ""); + if (NativeDeoptInstruction::is_deopt_at(pc)) { + CodeBlob* cb = CodeCache::find_blob_unsafe(pc); + if (cb != NULL && cb->is_compiled()) { + MACOS_AARCH64_ONLY(ThreadWXEnable wx(WXWrite, t);) // can call PcDescCache::add_pc_desc + CompiledMethod* cm = cb->as_compiled_method(); + assert(cm->insts_contains_inclusive(pc), ""); + address deopt = cm->is_method_handle_return(pc) ? + cm->deopt_mh_handler_begin() : + cm->deopt_handler_begin(); + assert(deopt != NULL, ""); + + frame fr = os::fetch_frame_from_context(uc); + cm->set_original_pc(&fr, pc); + + os::Posix::ucontext_set_pc(uc, deopt); + signal_was_handled = true; + } + } + } + // Call platform dependent signal handler. if (!signal_was_handled) { JavaThread* const jt = (t != NULL && t->is_Java_thread()) ? (JavaThread*) t : NULL; diff --git a/src/hotspot/os/windows/os_windows.cpp b/src/hotspot/os/windows/os_windows.cpp index cf1ff6335b9..070831180ee 100644 --- a/src/hotspot/os/windows/os_windows.cpp +++ b/src/hotspot/os/windows/os_windows.cpp @@ -2719,6 +2719,26 @@ LONG WINAPI topLevelExceptionFilter(struct _EXCEPTION_POINTERS* exceptionInfo) { if (result==EXCEPTION_CONTINUE_EXECUTION) return result; } #endif + + if (in_java && (exception_code == EXCEPTION_ILLEGAL_INSTRUCTION || exception_code == EXCEPTION_ILLEGAL_INSTRUCTION_2)) { + // Check for UD trap caused by NOP patching. + // If it is, patch return address to be deopt handler. + if (NativeDeoptInstruction::is_deopt_at(pc)) { + CodeBlob* cb = CodeCache::find_blob_unsafe(pc); + if (cb != NULL && cb->is_compiled()) { + CompiledMethod* cm = cb->as_compiled_method(); + frame fr = os::fetch_frame_from_context((void*)exceptionInfo->ContextRecord); + address deopt = cm->is_method_handle_return(pc) ? + cm->deopt_mh_handler_begin() : + cm->deopt_handler_begin(); + assert(cm->insts_contains_inclusive(pc), ""); + cm->set_original_pc(&fr, pc); + // Set pc to handler + exceptionInfo->ContextRecord->PC_NAME = (DWORD64)deopt; + return EXCEPTION_CONTINUE_EXECUTION; + } + } + } } #if !defined(USE_VECTORED_EXCEPTION_HANDLING) diff --git a/src/hotspot/share/c1/c1_CodeStubs.hpp b/src/hotspot/share/c1/c1_CodeStubs.hpp index 97bb9d60377..20603fc5cc1 100644 --- a/src/hotspot/share/c1/c1_CodeStubs.hpp +++ b/src/hotspot/share/c1/c1_CodeStubs.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1999, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1999, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -57,6 +57,7 @@ class CodeStub: public CompilationResourceObj { virtual CodeEmitInfo* info() const { return NULL; } virtual bool is_exception_throw_stub() const { return false; } virtual bool is_simple_exception_stub() const { return false; } + virtual int nr_immediate_oops_patched() const { return 0; } #ifndef PRODUCT virtual void print_name(outputStream* out) const = 0; #endif @@ -412,6 +413,13 @@ class PatchingStub: public CodeStub { masm->bind(_patch_site_entry); } + virtual int nr_immediate_oops_patched() const { + if (_id == load_mirror_id || _id == load_appendix_id) { + return 1; + } + return 0; + } + void install(MacroAssembler* masm, LIR_PatchCode patch_code, Register obj, CodeEmitInfo* info) { _info = info; _obj = obj; diff --git a/src/hotspot/share/c1/c1_Compilation.cpp b/src/hotspot/share/c1/c1_Compilation.cpp index baabbbd147b..fc700261099 100644 --- a/src/hotspot/share/c1/c1_Compilation.cpp +++ b/src/hotspot/share/c1/c1_Compilation.cpp @@ -362,6 +362,7 @@ int Compilation::emit_code_body() { } #endif /* PRODUCT */ + _immediate_oops_patched = lir_asm.nr_immediate_oops_patched(); return frame_map()->framesize(); } @@ -381,6 +382,10 @@ int Compilation::compile_java_method() { BAILOUT_("mdo allocation failed", no_frame_size); } + if (method()->is_synchronized()) { + set_has_monitors(true); + } + { PhaseTraceTime timeit(_t_buildIR); build_hir(); @@ -425,7 +430,9 @@ void Compilation::install_code(int frame_size) { implicit_exception_table(), compiler(), has_unsafe_access(), - SharedRuntime::is_wide_vector(max_vector_size()) + SharedRuntime::is_wide_vector(max_vector_size()), + has_monitors(), + _immediate_oops_patched ); } @@ -563,6 +570,7 @@ Compilation::Compilation(AbstractCompiler* compiler, ciEnv* env, ciMethod* metho , _would_profile(false) , _has_method_handle_invokes(false) , _has_reserved_stack_access(method->has_reserved_stack_access()) +, _has_monitors(false) , _install_code(install_code) , _bailout_msg(NULL) , _exception_info_list(NULL) @@ -570,6 +578,7 @@ Compilation::Compilation(AbstractCompiler* compiler, ciEnv* env, ciMethod* metho , _code(buffer_blob) , _has_access_indexed(false) , _interpreter_frame_size(0) +, _immediate_oops_patched(0) , _current_instruction(NULL) #ifndef PRODUCT , _last_instruction_printed(NULL) diff --git a/src/hotspot/share/c1/c1_Compilation.hpp b/src/hotspot/share/c1/c1_Compilation.hpp index 02a2f367df3..5ffe9529efd 100644 --- a/src/hotspot/share/c1/c1_Compilation.hpp +++ b/src/hotspot/share/c1/c1_Compilation.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1999, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1999, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -81,6 +81,7 @@ class Compilation: public StackObj { bool _would_profile; bool _has_method_handle_invokes; // True if this method has MethodHandle invokes. bool _has_reserved_stack_access; + bool _has_monitors; // Fastpath monitors detection for Continuations bool _install_code; const char* _bailout_msg; ExceptionInfoList* _exception_info_list; @@ -91,6 +92,7 @@ class Compilation: public StackObj { CodeBuffer _code; bool _has_access_indexed; int _interpreter_frame_size; // Stack space needed in case of a deoptimization + int _immediate_oops_patched; // compilation helpers void initialize(); @@ -136,6 +138,7 @@ class Compilation: public StackObj { bool has_exception_handlers() const { return _has_exception_handlers; } bool has_fpu_code() const { return _has_fpu_code; } bool has_unsafe_access() const { return _has_unsafe_access; } + bool has_monitors() const { return _has_monitors; } bool has_irreducible_loops() const { return _has_irreducible_loops; } int max_vector_size() const { return 0; } ciMethod* method() const { return _method; } @@ -167,6 +170,7 @@ class Compilation: public StackObj { void set_has_irreducible_loops(bool f) { _has_irreducible_loops = f; } void set_would_profile(bool f) { _would_profile = f; } void set_has_access_indexed(bool f) { _has_access_indexed = f; } + void set_has_monitors(bool f) { _has_monitors = f; } // Add a set of exception handlers covering the given PC offset void add_exception_handlers_for_pco(int pco, XHandlers* exception_handlers); // Statistics gathering diff --git a/src/hotspot/share/c1/c1_Compiler.cpp b/src/hotspot/share/c1/c1_Compiler.cpp index 382f014aca7..24bedc1d6d3 100644 --- a/src/hotspot/share/c1/c1_Compiler.cpp +++ b/src/hotspot/share/c1/c1_Compiler.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1999, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1999, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -152,7 +152,9 @@ bool Compiler::is_intrinsic_supported(const methodHandle& method) { case vmIntrinsics::_isInstance: case vmIntrinsics::_isPrimitive: case vmIntrinsics::_getModifiers: + case vmIntrinsics::_currentCarrierThread: case vmIntrinsics::_currentThread: + case vmIntrinsics::_extentLocalCache: case vmIntrinsics::_dabs: case vmIntrinsics::_dsqrt: case vmIntrinsics::_dsqrt_strict: @@ -223,6 +225,7 @@ bool Compiler::is_intrinsic_supported(const methodHandle& method) { case vmIntrinsics::_compareAndSetReference: case vmIntrinsics::_getCharStringU: case vmIntrinsics::_putCharStringU: + case vmIntrinsics::_Continuation_doYield: #ifdef JFR_HAVE_INTRINSICS case vmIntrinsics::_counterTime: case vmIntrinsics::_getEventWriter: diff --git a/src/hotspot/share/c1/c1_GraphBuilder.cpp b/src/hotspot/share/c1/c1_GraphBuilder.cpp index 10974386fb4..2416c2e4131 100644 --- a/src/hotspot/share/c1/c1_GraphBuilder.cpp +++ b/src/hotspot/share/c1/c1_GraphBuilder.cpp @@ -2304,6 +2304,7 @@ void GraphBuilder::instance_of(int klass_index) { void GraphBuilder::monitorenter(Value x, int bci) { // save state before locking in case of deoptimization after a NullPointerException ValueStack* state_before = copy_state_for_exception_with_bci(bci); + compilation()->set_has_monitors(true); append_with_bci(new MonitorEnter(x, state()->lock(x), state_before), bci); kill_all(); } diff --git a/src/hotspot/share/c1/c1_LIRAssembler.cpp b/src/hotspot/share/c1/c1_LIRAssembler.cpp index 331db648756..0ce2485e632 100644 --- a/src/hotspot/share/c1/c1_LIRAssembler.cpp +++ b/src/hotspot/share/c1/c1_LIRAssembler.cpp @@ -108,6 +108,7 @@ LIR_Assembler::LIR_Assembler(Compilation* c): , _current_block(NULL) , _pending_non_safepoint(NULL) , _pending_non_safepoint_offset(0) + , _immediate_oops_patched(0) { _slow_case_stubs = new CodeStubList(); } @@ -129,6 +130,7 @@ void LIR_Assembler::check_codespace() { void LIR_Assembler::append_code_stub(CodeStub* stub) { + _immediate_oops_patched += stub->nr_immediate_oops_patched(); _slow_case_stubs->append(stub); } diff --git a/src/hotspot/share/c1/c1_LIRAssembler.hpp b/src/hotspot/share/c1/c1_LIRAssembler.hpp index 8a50667c88a..270a63d7cfc 100644 --- a/src/hotspot/share/c1/c1_LIRAssembler.hpp +++ b/src/hotspot/share/c1/c1_LIRAssembler.hpp @@ -44,6 +44,7 @@ class LIR_Assembler: public CompilationResourceObj { Instruction* _pending_non_safepoint; int _pending_non_safepoint_offset; + int _immediate_oops_patched; Label _unwind_handler_entry; @@ -257,6 +258,7 @@ class LIR_Assembler: public CompilationResourceObj { #include CPU_HEADER(c1_LIRAssembler) public: + int nr_immediate_oops_patched() const { return _immediate_oops_patched; } static int call_stub_size() { return _call_stub_size; diff --git a/src/hotspot/share/c1/c1_LIRGenerator.cpp b/src/hotspot/share/c1/c1_LIRGenerator.cpp index 8baf846438c..9103f93d60d 100644 --- a/src/hotspot/share/c1/c1_LIRGenerator.cpp +++ b/src/hotspot/share/c1/c1_LIRGenerator.cpp @@ -1314,17 +1314,6 @@ void LIRGenerator::do_getModifiers(Intrinsic* x) { __ branch_destination(L_done->label()); } -// Example: Thread.currentThread() -void LIRGenerator::do_currentThread(Intrinsic* x) { - assert(x->number_of_arguments() == 0, "wrong type"); - LIR_Opr temp = new_register(T_ADDRESS); - LIR_Opr reg = rlock_result(x); - __ move(new LIR_Address(getThreadPointer(), in_bytes(JavaThread::threadObj_offset()), T_ADDRESS), temp); - // threadObj = ((OopHandle)_threadObj)->resolve(); - access_load(IN_NATIVE, T_OBJECT, - LIR_OprFact::address(new LIR_Address(temp, T_OBJECT)), reg); -} - void LIRGenerator::do_getObjectSize(Intrinsic* x) { assert(x->number_of_arguments() == 3, "wrong type"); LIR_Opr result_reg = rlock_result(x); @@ -1431,6 +1420,28 @@ void LIRGenerator::do_getObjectSize(Intrinsic* x) { __ branch_destination(L_done->label()); } +void LIRGenerator::do_extentLocalCache(Intrinsic* x) { + do_JavaThreadField(x, JavaThread::extentLocalCache_offset()); +} + +// Example: Thread.currentCarrierThread() +void LIRGenerator::do_currentCarrierThread(Intrinsic* x) { + do_JavaThreadField(x, JavaThread::threadObj_offset()); +} + +void LIRGenerator::do_vthread(Intrinsic* x) { + do_JavaThreadField(x, JavaThread::vthread_offset()); +} + +void LIRGenerator::do_JavaThreadField(Intrinsic* x, ByteSize offset) { + assert(x->number_of_arguments() == 0, "wrong type"); + LIR_Opr temp = new_register(T_ADDRESS); + LIR_Opr reg = rlock_result(x); + __ move(new LIR_Address(getThreadPointer(), in_bytes(offset), T_ADDRESS), temp); + access_load(IN_NATIVE, T_OBJECT, + LIR_OprFact::address(new LIR_Address(temp, T_OBJECT)), reg); +} + void LIRGenerator::do_RegisterFinalizer(Intrinsic* x) { assert(x->number_of_arguments() == 1, "wrong type"); LIRItem receiver(x->argument_at(0), this); @@ -2890,23 +2901,14 @@ void LIRGenerator::do_IfOp(IfOp* x) { #ifdef JFR_HAVE_INTRINSICS void LIRGenerator::do_getEventWriter(Intrinsic* x) { - LabelObj* L_end = new LabelObj(); - - // FIXME T_ADDRESS should actually be T_METADATA but it can't because the - // meaning of these two is mixed up (see JDK-8026837). - LIR_Address* jobj_addr = new LIR_Address(getThreadPointer(), - in_bytes(THREAD_LOCAL_WRITER_OFFSET_JFR), - T_ADDRESS); + BasicTypeList signature(0); + CallingConvention* cc = frame_map()->c_calling_convention(&signature); + LIR_Opr reg = result_register_for(x->type()); + address entry = StubRoutines::jfr_get_event_writer(); + CodeEmitInfo* info = state_for(x, x->state()); + __ call_runtime(entry, getThreadTemp(), reg, cc->args(), info); LIR_Opr result = rlock_result(x); - __ move(LIR_OprFact::oopConst(NULL), result); - LIR_Opr jobj = new_register(T_METADATA); - __ move_wide(jobj_addr, jobj); - __ cmp(lir_cond_equal, jobj, LIR_OprFact::metadataConst(0)); - __ branch(lir_cond_equal, L_end->label()); - - access_load(IN_NATIVE, T_OBJECT, LIR_OprFact::address(new LIR_Address(jobj, T_OBJECT)), result); - - __ branch_destination(L_end->label()); + __ move(reg, result); } #endif @@ -2941,7 +2943,7 @@ void LIRGenerator::do_Intrinsic(Intrinsic* x) { do_getEventWriter(x); break; case vmIntrinsics::_counterTime: - do_RuntimeCall(CAST_FROM_FN_PTR(address, JFR_TIME_FUNCTION), x); + do_RuntimeCall(CAST_FROM_FN_PTR(address, JfrTime::time_function()), x); break; #endif @@ -2958,8 +2960,10 @@ void LIRGenerator::do_Intrinsic(Intrinsic* x) { case vmIntrinsics::_isPrimitive: do_isPrimitive(x); break; case vmIntrinsics::_getModifiers: do_getModifiers(x); break; case vmIntrinsics::_getClass: do_getClass(x); break; - case vmIntrinsics::_currentThread: do_currentThread(x); break; case vmIntrinsics::_getObjectSize: do_getObjectSize(x); break; + case vmIntrinsics::_currentCarrierThread: do_currentCarrierThread(x); break; + case vmIntrinsics::_currentThread: do_vthread(x); break; + case vmIntrinsics::_extentLocalCache: do_extentLocalCache(x); break; case vmIntrinsics::_dlog: // fall through case vmIntrinsics::_dlog10: // fall through @@ -3027,6 +3031,10 @@ void LIRGenerator::do_Intrinsic(Intrinsic* x) { do_vectorizedMismatch(x); break; + case vmIntrinsics::_Continuation_doYield: + do_continuation_doYield(x); + break; + case vmIntrinsics::_blackhole: do_blackhole(x); break; diff --git a/src/hotspot/share/c1/c1_LIRGenerator.hpp b/src/hotspot/share/c1/c1_LIRGenerator.hpp index f1044668cbf..9e0a31bac5f 100644 --- a/src/hotspot/share/c1/c1_LIRGenerator.hpp +++ b/src/hotspot/share/c1/c1_LIRGenerator.hpp @@ -255,8 +255,11 @@ class LIRGenerator: public InstructionVisitor, public BlockClosure { void do_isPrimitive(Intrinsic* x); void do_getModifiers(Intrinsic* x); void do_getClass(Intrinsic* x); - void do_currentThread(Intrinsic* x); void do_getObjectSize(Intrinsic* x); + void do_currentCarrierThread(Intrinsic* x); + void do_extentLocalCache(Intrinsic* x); + void do_vthread(Intrinsic* x); + void do_JavaThreadField(Intrinsic* x, ByteSize offset); void do_FmaIntrinsic(Intrinsic* x); void do_MathIntrinsic(Intrinsic* x); void do_LibmIntrinsic(Intrinsic* x); @@ -269,6 +272,7 @@ class LIRGenerator: public InstructionVisitor, public BlockClosure { void do_update_CRC32C(Intrinsic* x); void do_vectorizedMismatch(Intrinsic* x); void do_blackhole(Intrinsic* x); + void do_continuation_doYield(Intrinsic* x); public: LIR_Opr call_runtime(BasicTypeArray* signature, LIRItemList* args, address entry, ValueType* result_type, CodeEmitInfo* info); diff --git a/src/hotspot/share/c1/c1_Runtime1.cpp b/src/hotspot/share/c1/c1_Runtime1.cpp index 9ae102b7f8a..96b1b6e5276 100644 --- a/src/hotspot/share/c1/c1_Runtime1.cpp +++ b/src/hotspot/share/c1/c1_Runtime1.cpp @@ -323,7 +323,8 @@ const char* Runtime1::name_for_address(address entry) { FUNCTION_CASE(entry, is_instance_of); FUNCTION_CASE(entry, trace_block_entry); #ifdef JFR_HAVE_INTRINSICS - FUNCTION_CASE(entry, JFR_TIME_FUNCTION); + FUNCTION_CASE(entry, JfrTime::time_function()); + FUNCTION_CASE(entry, StubRoutines::jfr_get_event_writer()); #endif FUNCTION_CASE(entry, StubRoutines::updateBytesCRC32()); FUNCTION_CASE(entry, StubRoutines::updateBytesCRC32C()); @@ -335,6 +336,7 @@ const char* Runtime1::name_for_address(address entry) { FUNCTION_CASE(entry, StubRoutines::dsin()); FUNCTION_CASE(entry, StubRoutines::dcos()); FUNCTION_CASE(entry, StubRoutines::dtan()); + FUNCTION_CASE(entry, StubRoutines::cont_doYield()); #undef FUNCTION_CASE @@ -741,6 +743,7 @@ JRT_BLOCK_ENTRY(void, Runtime1::monitorenter(JavaThread* current, oopDesc* obj, } assert(obj == lock->obj(), "must match"); SharedRuntime::monitor_enter_helper(obj, lock->lock(), current); + current->inc_held_monitor_count(); JRT_END diff --git a/src/hotspot/share/cds/cppVtables.cpp b/src/hotspot/share/cds/cppVtables.cpp index 7f55defa32c..cf8c4d13049 100644 --- a/src/hotspot/share/cds/cppVtables.cpp +++ b/src/hotspot/share/cds/cppVtables.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -31,6 +31,7 @@ #include "oops/instanceClassLoaderKlass.hpp" #include "oops/instanceMirrorKlass.hpp" #include "oops/instanceRefKlass.hpp" +#include "oops/instanceStackChunkKlass.hpp" #include "oops/methodData.hpp" #include "oops/objArrayKlass.hpp" #include "oops/typeArrayKlass.hpp" @@ -57,6 +58,7 @@ f(InstanceClassLoaderKlass) \ f(InstanceMirrorKlass) \ f(InstanceRefKlass) \ + f(InstanceStackChunkKlass) \ f(Method) \ f(ObjArrayKlass) \ f(TypeArrayKlass) diff --git a/src/hotspot/share/cds/heapShared.cpp b/src/hotspot/share/cds/heapShared.cpp index bad73b6c71a..64e28c161fb 100644 --- a/src/hotspot/share/cds/heapShared.cpp +++ b/src/hotspot/share/cds/heapShared.cpp @@ -305,6 +305,8 @@ void HeapShared::clear_root(int index) { oop HeapShared::archive_object(oop obj) { assert(DumpSharedSpaces, "dump-time only"); + assert(!obj->is_stackChunk(), "do not archive stack chunks"); + oop ao = find_archived_heap_object(obj); if (ao != NULL) { // already archived diff --git a/src/hotspot/share/ci/ciEnv.cpp b/src/hotspot/share/ci/ciEnv.cpp index 9b0259e67b3..dfc2d178bef 100644 --- a/src/hotspot/share/ci/ciEnv.cpp +++ b/src/hotspot/share/ci/ciEnv.cpp @@ -1051,6 +1051,8 @@ void ciEnv::register_method(ciMethod* target, AbstractCompiler* compiler, bool has_unsafe_access, bool has_wide_vectors, + bool has_monitors, + int immediate_oops_patched, RTMState rtm_state, const GrowableArrayView& native_invokers) { VM_ENTRY_MARK; @@ -1150,6 +1152,8 @@ void ciEnv::register_method(ciMethod* target, if (nm != NULL) { nm->set_has_unsafe_access(has_unsafe_access); nm->set_has_wide_vectors(has_wide_vectors); + nm->set_has_monitors(has_monitors); + assert(!method->is_synchronized() || nm->has_monitors(), ""); #if INCLUDE_RTM_OPT nm->set_rtm_state(rtm_state); #endif @@ -1211,6 +1215,13 @@ void ciEnv::register_method(ciMethod* target, } } +// ------------------------------------------------------------------ +// ciEnv::find_system_klass +ciKlass* ciEnv::find_system_klass(ciSymbol* klass_name) { + VM_ENTRY_MARK; + return get_klass_by_name_impl(NULL, constantPoolHandle(), klass_name, false); +} + // ------------------------------------------------------------------ // ciEnv::comp_level int ciEnv::comp_level() { diff --git a/src/hotspot/share/ci/ciEnv.hpp b/src/hotspot/share/ci/ciEnv.hpp index 997b650f12a..a2f6b03e57a 100644 --- a/src/hotspot/share/ci/ciEnv.hpp +++ b/src/hotspot/share/ci/ciEnv.hpp @@ -392,6 +392,8 @@ public: AbstractCompiler* compiler, bool has_unsafe_access, bool has_wide_vectors, + bool has_monitors, + int immediate_oops_patched, RTMState rtm_state = NoRTM, const GrowableArrayView& native_invokers = GrowableArrayView::EMPTY); @@ -434,6 +436,8 @@ public: ciInstanceKlass* get_box_klass_for_primitive_type(BasicType type); + ciKlass* find_system_klass(ciSymbol* klass_name); + // Note: To find a class from its name string, use ciSymbol::make, // but consider adding to vmSymbols.hpp instead. diff --git a/src/hotspot/share/ci/ciMethod.hpp b/src/hotspot/share/ci/ciMethod.hpp index 5e0732dfeb7..01127696498 100644 --- a/src/hotspot/share/ci/ciMethod.hpp +++ b/src/hotspot/share/ci/ciMethod.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1999, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1999, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -199,11 +199,12 @@ class ciMethod : public ciMetadata { // Code size for inlining decisions. int code_size_for_inlining(); - bool caller_sensitive() const { return get_Method()->caller_sensitive(); } - bool force_inline() const { return get_Method()->force_inline(); } - bool dont_inline() const { return get_Method()->dont_inline(); } - bool intrinsic_candidate() const { return get_Method()->intrinsic_candidate(); } - bool is_static_initializer() const { return get_Method()->is_static_initializer(); } + bool caller_sensitive() const { return get_Method()->caller_sensitive(); } + bool force_inline() const { return get_Method()->force_inline(); } + bool dont_inline() const { return get_Method()->dont_inline(); } + bool intrinsic_candidate() const { return get_Method()->intrinsic_candidate(); } + bool is_static_initializer() const { return get_Method()->is_static_initializer(); } + bool changes_current_thread() const { return get_Method()->changes_current_thread(); } bool check_intrinsic_candidate() const { if (intrinsic_id() == vmIntrinsics::_blackhole) { diff --git a/src/hotspot/share/classfile/classFileParser.cpp b/src/hotspot/share/classfile/classFileParser.cpp index d8552e5d490..e787c553fe8 100644 --- a/src/hotspot/share/classfile/classFileParser.cpp +++ b/src/hotspot/share/classfile/classFileParser.cpp @@ -968,6 +968,8 @@ public: _method_CallerSensitive, _method_ForceInline, _method_DontInline, + _method_ChangesCurrentThread, + _method_JvmtiMountTransition, _method_InjectedProfile, _method_LambdaForm_Compiled, _method_Hidden, @@ -1987,6 +1989,16 @@ AnnotationCollector::annotation_index(const ClassLoaderData* loader_data, if (!privileged) break; // only allow in privileged code return _method_DontInline; } + case VM_SYMBOL_ENUM_NAME(jdk_internal_vm_annotation_ChangesCurrentThread_signature): { + if (_location != _in_method) break; // only allow for methods + if (!privileged) break; // only allow in privileged code + return _method_ChangesCurrentThread; + } + case VM_SYMBOL_ENUM_NAME(jdk_internal_vm_annotation_JvmtiMountTransition_signature): { + if (_location != _in_method) break; // only allow for methods + if (!privileged) break; // only allow in privileged code + return _method_JvmtiMountTransition; + } case VM_SYMBOL_ENUM_NAME(java_lang_invoke_InjectedProfile_signature): { if (_location != _in_method) break; // only allow for methods if (!privileged) break; // only allow in privileged code @@ -2063,6 +2075,10 @@ void MethodAnnotationCollector::apply_to(const methodHandle& m) { m->set_force_inline(true); if (has_annotation(_method_DontInline)) m->set_dont_inline(true); + if (has_annotation(_method_ChangesCurrentThread)) + m->set_changes_current_thread(true); + if (has_annotation(_method_JvmtiMountTransition)) + m->set_jvmti_mount_transition(true); if (has_annotation(_method_InjectedProfile)) m->set_has_injected_profile(true); if (has_annotation(_method_LambdaForm_Compiled) && m->intrinsic_id() == vmIntrinsics::_none) diff --git a/src/hotspot/share/classfile/javaClasses.cpp b/src/hotspot/share/classfile/javaClasses.cpp index c0594d1fd82..6c45f43bc25 100644 --- a/src/hotspot/share/classfile/javaClasses.cpp +++ b/src/hotspot/share/classfile/javaClasses.cpp @@ -62,9 +62,11 @@ #include "prims/jvmtiExport.hpp" #include "prims/methodHandles.hpp" #include "prims/resolvedMethodTable.hpp" +#include "runtime/continuationEntry.inline.hpp" #include "runtime/fieldDescriptor.inline.hpp" #include "runtime/frame.inline.hpp" #include "runtime/handles.inline.hpp" +#include "runtime/handshake.hpp" #include "runtime/init.hpp" #include "runtime/interfaceSupport.inline.hpp" #include "runtime/java.hpp" @@ -74,9 +76,11 @@ #include "runtime/safepoint.hpp" #include "runtime/safepointVerifiers.hpp" #include "runtime/thread.inline.hpp" +#include "runtime/threadSMR.hpp" #include "runtime/vframe.inline.hpp" #include "runtime/vm_version.hpp" #include "utilities/align.hpp" +#include "utilities/growableArray.hpp" #include "utilities/preserveException.hpp" #include "utilities/utf8.hpp" #if INCLUDE_JVMCI @@ -454,9 +458,9 @@ char* java_lang_String::as_platform_dependent_str(Handle java_string, TRAPS) { } char *native_platform_string; - JavaThread* thread = THREAD; - jstring js = (jstring) JNIHandles::make_local(thread, java_string()); - { + jstring js; + { JavaThread* thread = THREAD; + js = (jstring) JNIHandles::make_local(thread, java_string()); HandleMark hm(thread); ThreadToNativeFromVM ttn(thread); JNIEnv *env = thread->jni_environment(); @@ -464,6 +468,8 @@ char* java_lang_String::as_platform_dependent_str(Handle java_string, TRAPS) { native_platform_string = (_to_platform_string_fn)(env, js, &is_copy); assert(is_copy == JNI_TRUE, "is_copy value changed"); } + + // Uses a store barrier and therefore needs to be in vm state JNIHandles::destroy_local(js); return native_platform_string; @@ -1672,45 +1678,144 @@ void java_lang_Class::set_classRedefinedCount(oop the_class_mirror, int value) { // // Note: The stackSize field is only present starting in 1.4. +int java_lang_Thread_FieldHolder::_group_offset; +int java_lang_Thread_FieldHolder::_priority_offset; +int java_lang_Thread_FieldHolder::_stackSize_offset; +int java_lang_Thread_FieldHolder::_stillborn_offset; +int java_lang_Thread_FieldHolder::_daemon_offset; +int java_lang_Thread_FieldHolder::_thread_status_offset; + +#define THREAD_FIELD_HOLDER_FIELDS_DO(macro) \ + macro(_group_offset, k, vmSymbols::group_name(), threadgroup_signature, false); \ + macro(_priority_offset, k, vmSymbols::priority_name(), int_signature, false); \ + macro(_stackSize_offset, k, "stackSize", long_signature, false); \ + macro(_stillborn_offset, k, "stillborn", bool_signature, false); \ + macro(_daemon_offset, k, vmSymbols::daemon_name(), bool_signature, false); \ + macro(_thread_status_offset, k, "threadStatus", int_signature, false) + +void java_lang_Thread_FieldHolder::compute_offsets() { + assert(_group_offset == 0, "offsets should be initialized only once"); + + InstanceKlass* k = vmClasses::Thread_FieldHolder_klass(); + THREAD_FIELD_HOLDER_FIELDS_DO(FIELD_COMPUTE_OFFSET); +} + +#if INCLUDE_CDS +void java_lang_Thread_FieldHolder::serialize_offsets(SerializeClosure* f) { + THREAD_FIELD_HOLDER_FIELDS_DO(FIELD_SERIALIZE_OFFSET); +} +#endif + +oop java_lang_Thread_FieldHolder::threadGroup(oop holder) { + return holder->obj_field(_group_offset); +} + +ThreadPriority java_lang_Thread_FieldHolder::priority(oop holder) { + return (ThreadPriority)holder->int_field(_priority_offset); +} + +void java_lang_Thread_FieldHolder::set_priority(oop holder, ThreadPriority priority) { + holder->int_field_put(_priority_offset, priority); +} + +jlong java_lang_Thread_FieldHolder::stackSize(oop holder) { + return holder->long_field(_stackSize_offset); +} + +bool java_lang_Thread_FieldHolder::is_stillborn(oop holder) { + return holder->bool_field(_stillborn_offset) != 0; +} + +void java_lang_Thread_FieldHolder::set_stillborn(oop holder) { + holder->bool_field_put(_stillborn_offset, true); +} + +bool java_lang_Thread_FieldHolder::is_daemon(oop holder) { + return holder->bool_field(_daemon_offset) != 0; +} + +void java_lang_Thread_FieldHolder::set_daemon(oop holder) { + holder->bool_field_put(_daemon_offset, true); +} + +void java_lang_Thread_FieldHolder::set_thread_status(oop holder, JavaThreadStatus status) { + holder->int_field_put(_thread_status_offset, static_cast(status)); +} + +JavaThreadStatus java_lang_Thread_FieldHolder::get_thread_status(oop holder) { + return static_cast(holder->int_field(_thread_status_offset)); +} + + +int java_lang_Thread_Constants::_static_VTHREAD_GROUP_offset = 0; +int java_lang_Thread_Constants::_static_NOT_SUPPORTED_CLASSLOADER_offset = 0; + +#define THREAD_CONSTANTS_STATIC_FIELDS_DO(macro) \ + macro(_static_VTHREAD_GROUP_offset, k, "VTHREAD_GROUP", threadgroup_signature, true); \ + macro(_static_NOT_SUPPORTED_CLASSLOADER_offset, k, "NOT_SUPPORTED_CLASSLOADER", classloader_signature, true); + +void java_lang_Thread_Constants::compute_offsets() { + assert(_static_VTHREAD_GROUP_offset == 0, "offsets should be initialized only once"); + + InstanceKlass* k = vmClasses::Thread_Constants_klass(); + THREAD_CONSTANTS_STATIC_FIELDS_DO(FIELD_COMPUTE_OFFSET); +} + +#if INCLUDE_CDS +void java_lang_Thread_Constants::serialize_offsets(SerializeClosure* f) { + THREAD_CONSTANTS_STATIC_FIELDS_DO(FIELD_SERIALIZE_OFFSET); +} +#endif + +oop java_lang_Thread_Constants::get_VTHREAD_GROUP() { + InstanceKlass* k = vmClasses::Thread_Constants_klass(); + oop base = k->static_field_base_raw(); + return base->obj_field(_static_VTHREAD_GROUP_offset); +} + +oop java_lang_Thread_Constants::get_NOT_SUPPORTED_CLASSLOADER() { + InstanceKlass* k = vmClasses::Thread_Constants_klass(); + oop base = k->static_field_base_raw(); + return base->obj_field(_static_NOT_SUPPORTED_CLASSLOADER_offset); +} + +int java_lang_Thread::_holder_offset; int java_lang_Thread::_name_offset; -int java_lang_Thread::_group_offset; int java_lang_Thread::_contextClassLoader_offset; int java_lang_Thread::_inheritedAccessControlContext_offset; -int java_lang_Thread::_priority_offset; int java_lang_Thread::_eetop_offset; +int java_lang_Thread::_jvmti_thread_state_offset; int java_lang_Thread::_interrupted_offset; -int java_lang_Thread::_daemon_offset; -int java_lang_Thread::_stillborn_offset; -int java_lang_Thread::_stackSize_offset; int java_lang_Thread::_tid_offset; -int java_lang_Thread::_thread_status_offset; +int java_lang_Thread::_continuation_offset; int java_lang_Thread::_park_blocker_offset; +int java_lang_Thread::_extentLocalBindings_offset; +JFR_ONLY(int java_lang_Thread::_jfr_epoch_offset;) #define THREAD_FIELDS_DO(macro) \ + macro(_holder_offset, k, "holder", thread_fieldholder_signature, false); \ macro(_name_offset, k, vmSymbols::name_name(), string_signature, false); \ - macro(_group_offset, k, vmSymbols::group_name(), threadgroup_signature, false); \ macro(_contextClassLoader_offset, k, vmSymbols::contextClassLoader_name(), classloader_signature, false); \ macro(_inheritedAccessControlContext_offset, k, vmSymbols::inheritedAccessControlContext_name(), accesscontrolcontext_signature, false); \ - macro(_priority_offset, k, vmSymbols::priority_name(), int_signature, false); \ - macro(_daemon_offset, k, vmSymbols::daemon_name(), bool_signature, false); \ macro(_eetop_offset, k, "eetop", long_signature, false); \ macro(_interrupted_offset, k, "interrupted", bool_signature, false); \ - macro(_stillborn_offset, k, "stillborn", bool_signature, false); \ - macro(_stackSize_offset, k, "stackSize", long_signature, false); \ macro(_tid_offset, k, "tid", long_signature, false); \ - macro(_thread_status_offset, k, "threadStatus", int_signature, false); \ - macro(_park_blocker_offset, k, "parkBlocker", object_signature, false) + macro(_park_blocker_offset, k, "parkBlocker", object_signature, false); \ + macro(_continuation_offset, k, "cont", continuation_signature, false); \ + macro(_extentLocalBindings_offset, k, "extentLocalBindings", object_signature, false); void java_lang_Thread::compute_offsets() { - assert(_group_offset == 0, "offsets should be initialized only once"); + assert(_holder_offset == 0, "offsets should be initialized only once"); InstanceKlass* k = vmClasses::Thread_klass(); THREAD_FIELDS_DO(FIELD_COMPUTE_OFFSET); + THREAD_INJECTED_FIELDS(INJECTED_FIELD_COMPUTE_OFFSET); } #if INCLUDE_CDS void java_lang_Thread::serialize_offsets(SerializeClosure* f) { THREAD_FIELDS_DO(FIELD_SERIALIZE_OFFSET); + THREAD_INJECTED_FIELDS(INJECTED_FIELD_SERIALIZE_OFFSET); } #endif @@ -1718,11 +1823,26 @@ JavaThread* java_lang_Thread::thread(oop java_thread) { return (JavaThread*)java_thread->address_field(_eetop_offset); } - void java_lang_Thread::set_thread(oop java_thread, JavaThread* thread) { java_thread->address_field_put(_eetop_offset, (address)thread); } +JvmtiThreadState* java_lang_Thread::jvmti_thread_state(oop java_thread) { + return (JvmtiThreadState*)java_thread->address_field(_jvmti_thread_state_offset); +} + +void java_lang_Thread::set_jvmti_thread_state(oop java_thread, JvmtiThreadState* state) { + java_thread->address_field_put(_jvmti_thread_state_offset, (address)state); +} + +void java_lang_Thread::clear_extentLocalBindings(oop java_thread) { + java_thread->obj_field_put(_extentLocalBindings_offset, NULL); +} + +oop java_lang_Thread::holder(oop java_thread) { + return java_thread->obj_field(_holder_offset); +} + bool java_lang_Thread::interrupted(oop java_thread) { // Make sure the caller can safely access oops. assert(Thread::current()->is_VM_thread() || @@ -1753,28 +1873,38 @@ void java_lang_Thread::set_name(oop java_thread, oop name) { ThreadPriority java_lang_Thread::priority(oop java_thread) { - return (ThreadPriority)java_thread->int_field(_priority_offset); + oop holder = java_lang_Thread::holder(java_thread); + assert(holder != NULL, "Java Thread not initialized"); + return java_lang_Thread_FieldHolder::priority(holder); } void java_lang_Thread::set_priority(oop java_thread, ThreadPriority priority) { - java_thread->int_field_put(_priority_offset, priority); + oop holder = java_lang_Thread::holder(java_thread); + assert(holder != NULL, "Java Thread not initialized"); + java_lang_Thread_FieldHolder::set_priority(holder, priority); } oop java_lang_Thread::threadGroup(oop java_thread) { - return java_thread->obj_field(_group_offset); + oop holder = java_lang_Thread::holder(java_thread); + assert(holder != NULL, "Java Thread not initialized"); + return java_lang_Thread_FieldHolder::threadGroup(holder); } bool java_lang_Thread::is_stillborn(oop java_thread) { - return java_thread->bool_field(_stillborn_offset) != 0; + oop holder = java_lang_Thread::holder(java_thread); + assert(holder != NULL, "Java Thread not initialized"); + return java_lang_Thread_FieldHolder::is_stillborn(holder); } // We never have reason to turn the stillborn bit off void java_lang_Thread::set_stillborn(oop java_thread) { - java_thread->bool_field_put(_stillborn_offset, true); + oop holder = java_lang_Thread::holder(java_thread); + assert(holder != NULL, "Java Thread not initialized"); + java_lang_Thread_FieldHolder::set_stillborn(holder); } @@ -1785,12 +1915,16 @@ bool java_lang_Thread::is_alive(oop java_thread) { bool java_lang_Thread::is_daemon(oop java_thread) { - return java_thread->bool_field(_daemon_offset) != 0; + oop holder = java_lang_Thread::holder(java_thread); + assert(holder != NULL, "Java Thread not initialized"); + return java_lang_Thread_FieldHolder::is_daemon(holder); } void java_lang_Thread::set_daemon(oop java_thread) { - java_thread->bool_field_put(_daemon_offset, true); + oop holder = java_lang_Thread::holder(java_thread); + assert(holder != NULL, "Java Thread not initialized"); + java_lang_Thread_FieldHolder::set_daemon(holder); } oop java_lang_Thread::context_class_loader(oop java_thread) { @@ -1803,13 +1937,16 @@ oop java_lang_Thread::inherited_access_control_context(oop java_thread) { jlong java_lang_Thread::stackSize(oop java_thread) { - return java_thread->long_field(_stackSize_offset); + oop holder = java_lang_Thread::holder(java_thread); + assert(holder != NULL, "Java Thread not initialized"); + return java_lang_Thread_FieldHolder::stackSize(holder); } // Write the thread status value to threadStatus field in java.lang.Thread java class. -void java_lang_Thread::set_thread_status(oop java_thread, - JavaThreadStatus status) { - java_thread->int_field_put(_thread_status_offset, static_cast(status)); +void java_lang_Thread::set_thread_status(oop java_thread, JavaThreadStatus status) { + oop holder = java_lang_Thread::holder(java_thread); + assert(holder != NULL, "Java Thread not initialized"); + java_lang_Thread_FieldHolder::set_thread_status(holder, status); } // Read thread status value from threadStatus field in java.lang.Thread java class. @@ -1819,20 +1956,144 @@ JavaThreadStatus java_lang_Thread::get_thread_status(oop java_thread) { assert(Threads_lock->owned_by_self() || Thread::current()->is_VM_thread() || JavaThread::current()->thread_state() == _thread_in_vm, "Java Thread is not running in vm"); - return static_cast(java_thread->int_field(_thread_status_offset)); + oop holder = java_lang_Thread::holder(java_thread); + if (holder == NULL) { + return JavaThreadStatus::NEW; // Java Thread not initialized + } else { + return java_lang_Thread_FieldHolder::get_thread_status(holder); + } } - -jlong java_lang_Thread::thread_id(oop java_thread) { - return java_thread->long_field(_tid_offset); +ByteSize java_lang_Thread::thread_id_offset() { + return in_ByteSize(_tid_offset); } oop java_lang_Thread::park_blocker(oop java_thread) { return java_thread->obj_field(_park_blocker_offset); } +oop java_lang_Thread::async_get_stack_trace(oop java_thread, TRAPS) { + ThreadsListHandle tlh(JavaThread::current()); + JavaThread* thread; + bool is_virtual = java_lang_VirtualThread::is_instance(java_thread); + if (is_virtual) { + oop carrier_thread = java_lang_VirtualThread::carrier_thread(java_thread); + if (carrier_thread == NULL) { + return NULL; + } + thread = java_lang_Thread::thread(carrier_thread); + } else { + thread = java_lang_Thread::thread(java_thread); + } + if (thread == NULL) { + return NULL; + } + + class GetStackTraceClosure : public HandshakeClosure { + public: + const Handle _java_thread; + int _depth; + bool _retry_handshake; + GrowableArray* _methods; + GrowableArray* _bcis; + + GetStackTraceClosure(Handle java_thread) : + HandshakeClosure("GetStackTraceClosure"), _java_thread(java_thread), _depth(0), _retry_handshake(false) { + // Pick some initial length + int init_length = MaxJavaStackTraceDepth / 2; + _methods = new GrowableArray(init_length); + _bcis = new GrowableArray(init_length); + } + + bool read_reset_retry() { + bool ret = _retry_handshake; + // If we re-execute the handshake this method need to return false + // when the handshake cannot be performed. (E.g. thread terminating) + _retry_handshake = false; + return ret; + } + + void do_thread(Thread* th) { + if (!Thread::current()->is_Java_thread()) { + _retry_handshake = true; + return; + } + + JavaThread* thread = JavaThread::cast(th); + + if (!thread->has_last_Java_frame()) { + return; + } + + bool carrier = false; + if (java_lang_VirtualThread::is_instance(_java_thread())) { + // if (thread->vthread() != _java_thread()) // We might be inside a System.executeOnCarrierThread + const ContinuationEntry* ce = thread->vthread_continuation(); + if (ce == nullptr || ce->cont_oop() != java_lang_VirtualThread::continuation(_java_thread())) { + return; // not mounted + } + } else { + carrier = (thread->vthread_continuation() != NULL); + } + + const int max_depth = MaxJavaStackTraceDepth; + const bool skip_hidden = !ShowHiddenFrames; + + int total_count = 0; + for (vframeStream vfst(thread, false, false, carrier); // we don't process frames as we don't care about oops + !vfst.at_end() && (max_depth == 0 || max_depth != total_count); + vfst.next()) { + + if (skip_hidden && (vfst.method()->is_hidden() || + vfst.method()->is_continuation_enter_intrinsic())) { + continue; + } + + _methods->push(vfst.method()); + _bcis->push(vfst.bci()); + total_count++; + } + + _depth = total_count; + } + }; + + // Handshake with target + ResourceMark rm(THREAD); + HandleMark hm(THREAD); + GetStackTraceClosure gstc(Handle(THREAD, java_thread)); + do { + Handshake::execute(&gstc, &tlh, thread); + } while (gstc.read_reset_retry()); + + // Stop if no stack trace is found. + if (gstc._depth == 0) { + return NULL; + } + + // Convert to StackTraceElement array + InstanceKlass* k = vmClasses::StackTraceElement_klass(); + assert(k != NULL, "must be loaded in 1.4+"); + if (k->should_be_initialized()) { + k->initialize(CHECK_NULL); + } + objArrayHandle trace = oopFactory::new_objArray_handle(k, gstc._depth, CHECK_NULL); + + for (int i = 0; i < gstc._depth; i++) { + methodHandle method(THREAD, gstc._methods->at(i)); + oop element = java_lang_StackTraceElement::create(method, + gstc._bcis->at(i), + CHECK_NULL); + trace->obj_at_put(i, element); + } + + return trace(); +} + const char* java_lang_Thread::thread_status_name(oop java_thread) { - JavaThreadStatus status = static_cast(java_thread->int_field(_thread_status_offset)); + oop holder = java_lang_Thread::holder(java_thread); + assert(holder != NULL, "Java Thread not initialized"); + JavaThreadStatus status = java_lang_Thread_FieldHolder::get_thread_status(holder); switch (status) { case JavaThreadStatus::NEW : return "NEW"; case JavaThreadStatus::RUNNABLE : return "RUNNABLE"; @@ -1848,13 +2109,12 @@ const char* java_lang_Thread::thread_status_name(oop java_thread) { } int java_lang_ThreadGroup::_parent_offset; int java_lang_ThreadGroup::_name_offset; -int java_lang_ThreadGroup::_threads_offset; -int java_lang_ThreadGroup::_groups_offset; int java_lang_ThreadGroup::_maxPriority_offset; -int java_lang_ThreadGroup::_destroyed_offset; int java_lang_ThreadGroup::_daemon_offset; -int java_lang_ThreadGroup::_nthreads_offset; int java_lang_ThreadGroup::_ngroups_offset; +int java_lang_ThreadGroup::_groups_offset; +int java_lang_ThreadGroup::_nweaks_offset; +int java_lang_ThreadGroup::_weaks_offset; oop java_lang_ThreadGroup::parent(oop java_thread_group) { assert(oopDesc::is_oop(java_thread_group), "thread group must be oop"); @@ -1872,16 +2132,14 @@ const char* java_lang_ThreadGroup::name(oop java_thread_group) { return NULL; } -int java_lang_ThreadGroup::nthreads(oop java_thread_group) { +ThreadPriority java_lang_ThreadGroup::maxPriority(oop java_thread_group) { assert(oopDesc::is_oop(java_thread_group), "thread group must be oop"); - return java_thread_group->int_field(_nthreads_offset); + return (ThreadPriority) java_thread_group->int_field(_maxPriority_offset); } -objArrayOop java_lang_ThreadGroup::threads(oop java_thread_group) { - oop threads = java_thread_group->obj_field(_threads_offset); - assert(threads != NULL, "threadgroups should have threads"); - assert(threads->is_objArray(), "just checking"); // Todo: Add better type checking code - return objArrayOop(threads); +bool java_lang_ThreadGroup::is_daemon(oop java_thread_group) { + assert(oopDesc::is_oop(java_thread_group), "thread group must be oop"); + return java_thread_group->bool_field(_daemon_offset) != 0; } int java_lang_ThreadGroup::ngroups(oop java_thread_group) { @@ -1895,31 +2153,26 @@ objArrayOop java_lang_ThreadGroup::groups(oop java_thread_group) { return objArrayOop(groups); } -ThreadPriority java_lang_ThreadGroup::maxPriority(oop java_thread_group) { +int java_lang_ThreadGroup::nweaks(oop java_thread_group) { assert(oopDesc::is_oop(java_thread_group), "thread group must be oop"); - return (ThreadPriority) java_thread_group->int_field(_maxPriority_offset); + return java_thread_group->int_field(_nweaks_offset); } -bool java_lang_ThreadGroup::is_destroyed(oop java_thread_group) { - assert(oopDesc::is_oop(java_thread_group), "thread group must be oop"); - return java_thread_group->bool_field(_destroyed_offset) != 0; -} - -bool java_lang_ThreadGroup::is_daemon(oop java_thread_group) { - assert(oopDesc::is_oop(java_thread_group), "thread group must be oop"); - return java_thread_group->bool_field(_daemon_offset) != 0; +objArrayOop java_lang_ThreadGroup::weaks(oop java_thread_group) { + oop weaks = java_thread_group->obj_field(_weaks_offset); + assert(weaks == NULL || weaks->is_objArray(), "just checking"); + return objArrayOop(weaks); } #define THREADGROUP_FIELDS_DO(macro) \ - macro(_parent_offset, k, vmSymbols::parent_name(), threadgroup_signature, false); \ - macro(_name_offset, k, vmSymbols::name_name(), string_signature, false); \ - macro(_threads_offset, k, vmSymbols::threads_name(), thread_array_signature, false); \ - macro(_groups_offset, k, vmSymbols::groups_name(), threadgroup_array_signature, false); \ - macro(_maxPriority_offset, k, vmSymbols::maxPriority_name(), int_signature, false); \ - macro(_destroyed_offset, k, vmSymbols::destroyed_name(), bool_signature, false); \ - macro(_daemon_offset, k, vmSymbols::daemon_name(), bool_signature, false); \ - macro(_nthreads_offset, k, vmSymbols::nthreads_name(), int_signature, false); \ - macro(_ngroups_offset, k, vmSymbols::ngroups_name(), int_signature, false) + macro(_parent_offset, k, vmSymbols::parent_name(), threadgroup_signature, false); \ + macro(_name_offset, k, vmSymbols::name_name(), string_signature, false); \ + macro(_maxPriority_offset, k, vmSymbols::maxPriority_name(), int_signature, false); \ + macro(_daemon_offset, k, vmSymbols::daemon_name(), bool_signature, false); \ + macro(_ngroups_offset, k, vmSymbols::ngroups_name(), int_signature, false); \ + macro(_groups_offset, k, vmSymbols::groups_name(), threadgroup_array_signature, false); \ + macro(_nweaks_offset, k, vmSymbols::nweaks_name(), int_signature, false); \ + macro(_weaks_offset, k, vmSymbols::weaks_name(), weakreference_array_signature, false); void java_lang_ThreadGroup::compute_offsets() { assert(_parent_offset == 0, "offsets should be initialized only once"); @@ -1934,6 +2187,98 @@ void java_lang_ThreadGroup::serialize_offsets(SerializeClosure* f) { } #endif + +// java_lang_VirtualThread + +int java_lang_VirtualThread::static_notify_jvmti_events_offset; +int java_lang_VirtualThread::static_vthread_scope_offset; +int java_lang_VirtualThread::_carrierThread_offset; +int java_lang_VirtualThread::_continuation_offset; +int java_lang_VirtualThread::_state_offset; + +#define VTHREAD_FIELDS_DO(macro) \ + macro(static_notify_jvmti_events_offset, k, "notifyJvmtiEvents", bool_signature, true); \ + macro(static_vthread_scope_offset, k, "VTHREAD_SCOPE", continuationscope_signature, true); \ + macro(_carrierThread_offset, k, "carrierThread", thread_signature, false); \ + macro(_continuation_offset, k, "cont", continuation_signature, false); \ + macro(_state_offset, k, "state", int_signature, false) + +static bool vthread_notify_jvmti_events = JNI_FALSE; + +void java_lang_VirtualThread::compute_offsets() { + InstanceKlass* k = vmClasses::VirtualThread_klass(); + VTHREAD_FIELDS_DO(FIELD_COMPUTE_OFFSET); +} + +void java_lang_VirtualThread::init_static_notify_jvmti_events() { + if (vthread_notify_jvmti_events) { + InstanceKlass* ik = vmClasses::VirtualThread_klass(); + oop base = ik->static_field_base_raw(); + base->release_bool_field_put(static_notify_jvmti_events_offset, vthread_notify_jvmti_events); + } +} + +bool java_lang_VirtualThread::is_instance(oop obj) { + return obj != NULL && is_subclass(obj->klass()); +} + +oop java_lang_VirtualThread::carrier_thread(oop vthread) { + oop thread = vthread->obj_field(_carrierThread_offset); + return thread; +} + +oop java_lang_VirtualThread::continuation(oop vthread) { + oop cont = vthread->obj_field(_continuation_offset); + return cont; +} + +u2 java_lang_VirtualThread::state(oop vthread) { + return vthread->short_field_acquire(_state_offset); +} + +JavaThreadStatus java_lang_VirtualThread::map_state_to_thread_status(int state) { + JavaThreadStatus status = JavaThreadStatus::NEW; + switch (state) { + case NEW : + status = JavaThreadStatus::NEW; + break; + case STARTED : + case RUNNABLE : + case RUNNABLE_SUSPENDED : + case RUNNING : + case PARKING : + case YIELDING : + status = JavaThreadStatus::RUNNABLE; + break; + case PARKED : + case PARKED_SUSPENDED : + case PINNED : + status = JavaThreadStatus::PARKED; + break; + case TERMINATED : + status = JavaThreadStatus::TERMINATED; + break; + default: + ShouldNotReachHere(); + } + return status; +} + +#if INCLUDE_CDS +void java_lang_VirtualThread::serialize_offsets(SerializeClosure* f) { + VTHREAD_FIELDS_DO(FIELD_SERIALIZE_OFFSET); +} +#endif + +bool java_lang_VirtualThread::notify_jvmti_events() { + return vthread_notify_jvmti_events == JNI_TRUE; +} + +void java_lang_VirtualThread::set_notify_jvmti_events(bool enable) { + vthread_notify_jvmti_events = enable; +} + + // java_lang_Throwable int java_lang_Throwable::_backtrace_offset; @@ -2060,6 +2405,7 @@ class BacktraceBuilder: public StackObj { trace_bcis_offset = java_lang_Throwable::trace_bcis_offset, trace_mirrors_offset = java_lang_Throwable::trace_mirrors_offset, trace_names_offset = java_lang_Throwable::trace_names_offset, + trace_conts_offset = java_lang_Throwable::trace_conts_offset, trace_next_offset = java_lang_Throwable::trace_next_offset, trace_hidden_offset = java_lang_Throwable::trace_hidden_offset, trace_size = java_lang_Throwable::trace_size, @@ -2182,6 +2528,7 @@ class BacktraceBuilder: public StackObj { // from being unloaded while we still have this stack trace. assert(method->method_holder()->java_mirror() != NULL, "never push null for mirror"); _mirrors->obj_at_put(_index, method->method_holder()->java_mirror()); + _index++; } @@ -2440,13 +2787,14 @@ void java_lang_Throwable::fill_in_stack_trace(Handle throwable, const methodHand vframeStream st(thread, false /* stop_at_java_call_stub */, false /* process_frames */); #endif int total_count = 0; - RegisterMap map(thread, false /* update */, false /* process_frames */); + RegisterMap map(thread, false /* update */, false /* process_frames */, true /* walk_cont */); int decode_offset = 0; CompiledMethod* nm = NULL; bool skip_fillInStackTrace_check = false; bool skip_throwableInit_check = false; bool skip_hidden = !ShowHiddenFrames; - + bool show_carrier = ShowCarrierFrames; + ContinuationEntry* cont_entry = thread->last_continuation(); for (frame fr = thread->last_frame(); max_depth == 0 || max_depth != total_count;) { Method* method = NULL; int bci = 0; @@ -2459,10 +2807,25 @@ void java_lang_Throwable::fill_in_stack_trace(Handle throwable, const methodHand bci = stream.read_bci(); } else { if (fr.is_first_frame()) break; + + if (Continuation::is_continuation_enterSpecial(fr)) { + assert(cont_entry == Continuation::get_continuation_entry_for_entry_frame(thread, fr), ""); + if (!show_carrier && cont_entry->is_virtual_thread()) { + break; + } + cont_entry = cont_entry->parent(); + } + address pc = fr.pc(); if (fr.is_interpreted_frame()) { - address bcp = fr.interpreter_frame_bcp(); - method = fr.interpreter_frame_method(); + address bcp; + if (!map.in_cont()) { + bcp = fr.interpreter_frame_bcp(); + method = fr.interpreter_frame_method(); + } else { + bcp = map.stack_chunk()->interpreter_frame_bcp(fr); + method = map.stack_chunk()->interpreter_frame_method(fr); + } bci = method->bci_from(bcp); fr = fr.sender(&map); } else { @@ -2474,6 +2837,7 @@ void java_lang_Throwable::fill_in_stack_trace(Handle throwable, const methodHand continue; } nm = cb->as_compiled_method(); + assert(nm->method() != NULL, "must be"); if (nm->method()->is_native()) { method = nm->method(); bci = 0; @@ -2487,9 +2851,10 @@ void java_lang_Throwable::fill_in_stack_trace(Handle throwable, const methodHand } } #ifdef ASSERT - assert(st.method() == method && st.bci() == bci, - "Wrong stack trace"); - st.next(); + if (!st.at_end()) { // TODO LOOM remove once we show only vthread trace + assert(st.method() == method && st.bci() == bci, "Wrong stack trace"); + st.next(); + } #endif // the format of the stacktrace will be: @@ -2519,7 +2884,7 @@ void java_lang_Throwable::fill_in_stack_trace(Handle throwable, const methodHand skip_throwableInit_check = true; } } - if (method->is_hidden()) { + if (method->is_hidden() || method->is_continuation_enter_intrinsic()) { if (skip_hidden) { if (total_count == 0) { // The top frame will be hidden from the stack trace. @@ -2528,6 +2893,7 @@ void java_lang_Throwable::fill_in_stack_trace(Handle throwable, const methodHand continue; } } + bt.push(method, bci, CHECK); total_count++; } @@ -2607,20 +2973,20 @@ void java_lang_Throwable::fill_in_stack_trace_of_preallocated_backtrace(Handle t assert(java_lang_Throwable::unassigned_stacktrace() != NULL, "not initialized"); } -void java_lang_Throwable::get_stack_trace_elements(Handle throwable, +void java_lang_Throwable::get_stack_trace_elements(int depth, Handle backtrace, objArrayHandle stack_trace_array_h, TRAPS) { - if (throwable.is_null() || stack_trace_array_h.is_null()) { + if (backtrace.is_null() || stack_trace_array_h.is_null()) { THROW(vmSymbols::java_lang_NullPointerException()); } assert(stack_trace_array_h->is_objArray(), "Stack trace array should be an array of StackTraceElenent"); - if (stack_trace_array_h->length() != depth(throwable())) { + if (stack_trace_array_h->length() != depth) { THROW(vmSymbols::java_lang_IndexOutOfBoundsException()); } - objArrayHandle result(THREAD, objArrayOop(backtrace(throwable()))); + objArrayHandle result(THREAD, objArrayOop(backtrace())); BacktraceIterator iter(result, THREAD); int index = 0; @@ -2640,7 +3006,8 @@ void java_lang_Throwable::get_stack_trace_elements(Handle throwable, method, bte._version, bte._bci, - bte._name, CHECK); + bte._name, + CHECK); } } @@ -2839,10 +3206,12 @@ void java_lang_StackTraceElement::decode(const methodHandle& method, int bci, int java_lang_StackFrameInfo::_memberName_offset; int java_lang_StackFrameInfo::_bci_offset; int java_lang_StackFrameInfo::_version_offset; +int java_lang_StackFrameInfo::_contScope_offset; #define STACKFRAMEINFO_FIELDS_DO(macro) \ - macro(_memberName_offset, k, "memberName", object_signature, false); \ - macro(_bci_offset, k, "bci", int_signature, false) + macro(_memberName_offset, k, "memberName", object_signature, false); \ + macro(_bci_offset, k, "bci", int_signature, false); \ + macro(_contScope_offset, k, "contScope", continuationscope_signature, false) void java_lang_StackFrameInfo::compute_offsets() { InstanceKlass* k = vmClasses::StackFrameInfo_klass(); @@ -2866,10 +3235,11 @@ Method* java_lang_StackFrameInfo::get_method(Handle stackFrame, InstanceKlass* h return method; } -void java_lang_StackFrameInfo::set_method_and_bci(Handle stackFrame, const methodHandle& method, int bci, TRAPS) { +void java_lang_StackFrameInfo::set_method_and_bci(Handle stackFrame, const methodHandle& method, int bci, oop cont, TRAPS) { // set Method* or mid/cpref HandleMark hm(THREAD); Handle mname(THREAD, stackFrame->obj_field(_memberName_offset)); + Handle cont_h (THREAD, cont); InstanceKlass* ik = method->method_holder(); CallInfo info(method(), ik, CHECK); MethodHandles::init_method_MemberName(mname, info); @@ -2879,6 +3249,9 @@ void java_lang_StackFrameInfo::set_method_and_bci(Handle stackFrame, const metho int version = method->constants()->version(); assert((jushort)version == version, "version should be short"); java_lang_StackFrameInfo::set_version(stackFrame(), (short)version); + + oop contScope = cont_h() != NULL ? jdk_internal_vm_Continuation::scope(cont_h()) : (oop)NULL; + java_lang_StackFrameInfo::set_contScope(stackFrame(), contScope); } void java_lang_StackFrameInfo::to_stack_trace_element(Handle stackFrame, Handle stack_trace_element, TRAPS) { @@ -2892,8 +3265,7 @@ void java_lang_StackFrameInfo::to_stack_trace_element(Handle stackFrame, Handle short version = stackFrame->short_field(_version_offset); int bci = stackFrame->int_field(_bci_offset); Symbol* name = method->name(); - java_lang_StackTraceElement::fill_in(stack_trace_element, holder, methodHandle(THREAD, method), - version, bci, name, CHECK); + java_lang_StackTraceElement::fill_in(stack_trace_element, holder, methodHandle(THREAD, method), version, bci, name, CHECK); } void java_lang_StackFrameInfo::set_version(oop element, short value) { @@ -2905,6 +3277,10 @@ void java_lang_StackFrameInfo::set_bci(oop element, int value) { element->int_field_put(_bci_offset, value); } +void java_lang_StackFrameInfo::set_contScope(oop element, oop value) { + element->obj_field_put(_contScope_offset, value); +} + int java_lang_LiveStackFrameInfo::_monitors_offset; int java_lang_LiveStackFrameInfo::_locals_offset; int java_lang_LiveStackFrameInfo::_operands_offset; @@ -4627,6 +5003,86 @@ void java_lang_AssertionStatusDirectives::set_deflt(oop o, bool val) { o->bool_field_put(_deflt_offset, val); } +// Support for jdk.internal.vm.Continuation + +int jdk_internal_vm_ContinuationScope::_name_offset; +int jdk_internal_vm_Continuation::_scope_offset; +int jdk_internal_vm_Continuation::_target_offset; +int jdk_internal_vm_Continuation::_tail_offset; +int jdk_internal_vm_Continuation::_parent_offset; +int jdk_internal_vm_Continuation::_yieldInfo_offset; +int jdk_internal_vm_Continuation::_mounted_offset; +int jdk_internal_vm_Continuation::_done_offset; +int jdk_internal_vm_Continuation::_preempted_offset; + +#define CONTINUATIONSCOPE_FIELDS_DO(macro) \ + macro(_name_offset, k, vmSymbols::name_name(), string_signature, false); + +void jdk_internal_vm_ContinuationScope::compute_offsets() { + InstanceKlass* k = vmClasses::ContinuationScope_klass(); + CONTINUATIONSCOPE_FIELDS_DO(FIELD_COMPUTE_OFFSET); +} + +#if INCLUDE_CDS +void jdk_internal_vm_ContinuationScope::serialize_offsets(SerializeClosure* f) { + CONTINUATIONSCOPE_FIELDS_DO(FIELD_SERIALIZE_OFFSET); +} +#endif + +// Support for jdk.internal.vm.Continuation + +#define CONTINUATION_FIELDS_DO(macro) \ + macro(_scope_offset, k, vmSymbols::scope_name(), continuationscope_signature, false); \ + macro(_target_offset, k, vmSymbols::target_name(), runnable_signature, false); \ + macro(_parent_offset, k, vmSymbols::parent_name(), continuation_signature, false); \ + macro(_yieldInfo_offset, k, vmSymbols::yieldInfo_name(), object_signature, false); \ + macro(_tail_offset, k, vmSymbols::tail_name(), stackchunk_signature, false); \ + macro(_mounted_offset, k, vmSymbols::mounted_name(), bool_signature, false); \ + macro(_done_offset, k, vmSymbols::done_name(), bool_signature, false); \ + macro(_preempted_offset, k, "preempted", bool_signature, false); + +void jdk_internal_vm_Continuation::compute_offsets() { + InstanceKlass* k = vmClasses::Continuation_klass(); + CONTINUATION_FIELDS_DO(FIELD_COMPUTE_OFFSET); +} + +#if INCLUDE_CDS +void jdk_internal_vm_Continuation::serialize_offsets(SerializeClosure* f) { + CONTINUATION_FIELDS_DO(FIELD_SERIALIZE_OFFSET); +} +#endif + +// Support for jdk.internal.vm.StackChunk + +int jdk_internal_vm_StackChunk::_parent_offset; +int jdk_internal_vm_StackChunk::_size_offset; +int jdk_internal_vm_StackChunk::_sp_offset; +int jdk_internal_vm_StackChunk::_pc_offset; +int jdk_internal_vm_StackChunk::_argsize_offset; +int jdk_internal_vm_StackChunk::_flags_offset; +int jdk_internal_vm_StackChunk::_maxThawingSize_offset; +int jdk_internal_vm_StackChunk::_cont_offset; + +#define STACKCHUNK_FIELDS_DO(macro) \ + macro(_parent_offset, k, vmSymbols::parent_name(), stackchunk_signature, false); \ + macro(_size_offset, k, vmSymbols::size_name(), int_signature, false); \ + macro(_sp_offset, k, vmSymbols::sp_name(), int_signature, false); \ + macro(_argsize_offset, k, vmSymbols::argsize_name(), int_signature, false); + +void jdk_internal_vm_StackChunk::compute_offsets() { + InstanceKlass* k = vmClasses::StackChunk_klass(); + STACKCHUNK_FIELDS_DO(FIELD_COMPUTE_OFFSET); + STACKCHUNK_INJECTED_FIELDS(INJECTED_FIELD_COMPUTE_OFFSET); +} + +#if INCLUDE_CDS +void jdk_internal_vm_StackChunk::serialize_offsets(SerializeClosure* f) { + STACKCHUNK_FIELDS_DO(FIELD_SERIALIZE_OFFSET); + STACKCHUNK_INJECTED_FIELDS(INJECTED_FIELD_SERIALIZE_OFFSET); +} +#endif + + int java_util_concurrent_locks_AbstractOwnableSynchronizer::_owner_offset; #define AOS_FIELDS_DO(macro) \ @@ -5084,5 +5540,6 @@ int InjectedField::compute_offset() { void javaClasses_init() { JavaClasses::compute_offsets(); JavaClasses::check_offsets(); + java_lang_VirtualThread::init_static_notify_jvmti_events(); FilteredFieldsMap::initialize(); // must be done after computing offsets. } diff --git a/src/hotspot/share/classfile/javaClasses.hpp b/src/hotspot/share/classfile/javaClasses.hpp index 8a2d5fdf24c..5b315361c09 100644 --- a/src/hotspot/share/classfile/javaClasses.hpp +++ b/src/hotspot/share/classfile/javaClasses.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -28,10 +28,13 @@ #include "classfile/vmClasses.hpp" #include "oops/oop.hpp" #include "oops/instanceKlass.hpp" +#include "oops/stackChunkOop.hpp" #include "oops/symbol.hpp" #include "runtime/os.hpp" +#include "utilities/macros.hpp" #include "utilities/vmEnums.hpp" +class JvmtiThreadState; class RecordComponent; // Interface for manipulating the basic Java classes. @@ -47,7 +50,10 @@ class RecordComponent; f(java_lang_ClassLoader) \ f(java_lang_Throwable) \ f(java_lang_Thread) \ + f(java_lang_Thread_FieldHolder) \ + f(java_lang_Thread_Constants) \ f(java_lang_ThreadGroup) \ + f(java_lang_VirtualThread) \ f(java_lang_InternalError) \ f(java_lang_AssertionStatusDirectives) \ f(java_lang_ref_SoftReference) \ @@ -73,6 +79,9 @@ class RecordComponent; f(java_lang_StackTraceElement) \ f(java_lang_StackFrameInfo) \ f(java_lang_LiveStackFrameInfo) \ + f(jdk_internal_vm_ContinuationScope) \ + f(jdk_internal_vm_Continuation) \ + f(jdk_internal_vm_StackChunk) \ f(java_util_concurrent_locks_AbstractOwnableSynchronizer) \ f(jdk_internal_invoke_NativeEntryPoint) \ f(jdk_internal_misc_UnsafeConstants) \ @@ -390,23 +399,27 @@ class java_lang_Class : AllStatic { // Interface to java.lang.Thread objects +#define THREAD_INJECTED_FIELDS(macro) \ + macro(java_lang_Thread, jvmti_thread_state, intptr_signature, false) \ + JFR_ONLY(macro(java_lang_Thread, jfr_epoch, short_signature, false)) + class java_lang_Thread : AllStatic { + friend class java_lang_VirtualThread; private: // Note that for this class the layout changed between JDK1.2 and JDK1.3, // so we compute the offsets at startup rather than hard-wiring them. + static int _holder_offset; static int _name_offset; - static int _group_offset; static int _contextClassLoader_offset; static int _inheritedAccessControlContext_offset; - static int _priority_offset; static int _eetop_offset; + static int _jvmti_thread_state_offset; static int _interrupted_offset; - static int _daemon_offset; - static int _stillborn_offset; - static int _stackSize_offset; static int _tid_offset; - static int _thread_status_offset; + static int _continuation_offset; static int _park_blocker_offset; + static int _extentLocalBindings_offset; + JFR_ONLY(static int _jfr_epoch_offset;) static void compute_offsets(); @@ -417,6 +430,8 @@ class java_lang_Thread : AllStatic { static JavaThread* thread(oop java_thread); // Set JavaThread for instance static void set_thread(oop java_thread, JavaThread* thread); + // FieldHolder + static oop holder(oop java_thread); // Interrupted status static bool interrupted(oop java_thread); static void set_interrupted(oop java_thread, bool val); @@ -444,7 +459,16 @@ class java_lang_Thread : AllStatic { // Stack size hint static jlong stackSize(oop java_thread); // Thread ID - static jlong thread_id(oop java_thread); + static int64_t thread_id(oop java_thread); + static ByteSize thread_id_offset(); + // Continuation + static inline oop continuation(oop java_thread); + + static JvmtiThreadState* jvmti_thread_state(oop java_thread); + static void set_jvmti_thread_state(oop java_thread, JvmtiThreadState* state); + + // Clear all extent local bindings on error + static void clear_extentLocalBindings(oop java_thread); // Blocker object responsible for thread parking static oop park_blocker(oop java_thread); @@ -456,23 +480,82 @@ class java_lang_Thread : AllStatic { static const char* thread_status_name(oop java_thread_oop); + // Fill in current stack trace, can cause GC + static oop async_get_stack_trace(oop java_thread, TRAPS); + + JFR_ONLY(static u2 jfr_epoch(oop java_thread);) + JFR_ONLY(static void set_jfr_epoch(oop java_thread, u2 epoch);) + JFR_ONLY(static int jfr_epoch_offset() { CHECK_INIT(_jfr_epoch_offset); }) + // Debugging friend class JavaClasses; }; +// Interface to java.lang.Thread$FieldHolder objects + +class java_lang_Thread_FieldHolder : AllStatic { + private: + static int _group_offset; + static int _priority_offset; + static int _stackSize_offset; + static int _stillborn_offset; + static int _daemon_offset; + static int _thread_status_offset; + + static void compute_offsets(); + + public: + static void serialize_offsets(SerializeClosure* f) NOT_CDS_RETURN; + + static oop threadGroup(oop holder); + + static ThreadPriority priority(oop holder); + static void set_priority(oop holder, ThreadPriority priority); + + static jlong stackSize(oop holder); + + static bool is_stillborn(oop holder); + static void set_stillborn(oop holder); + + static bool is_daemon(oop holder); + static void set_daemon(oop holder); + + static void set_thread_status(oop holder, JavaThreadStatus); + static JavaThreadStatus get_thread_status(oop holder); + + friend class JavaClasses; +}; + +// Interface to java.lang.Thread$Constants objects + +class java_lang_Thread_Constants : AllStatic { + private: + static int _static_VTHREAD_GROUP_offset; + static int _static_NOT_SUPPORTED_CLASSLOADER_offset; + + static void compute_offsets(); + static void serialize_offsets(SerializeClosure* f) NOT_CDS_RETURN; + + public: + static oop get_VTHREAD_GROUP(); + static oop get_NOT_SUPPORTED_CLASSLOADER(); + + friend class JavaClasses; +}; + // Interface to java.lang.ThreadGroup objects class java_lang_ThreadGroup : AllStatic { private: static int _parent_offset; static int _name_offset; - static int _threads_offset; - static int _groups_offset; static int _maxPriority_offset; - static int _destroyed_offset; static int _daemon_offset; - static int _nthreads_offset; + static int _ngroups_offset; + static int _groups_offset; + static int _nweaks_offset; + static int _weaks_offset; static void compute_offsets(); @@ -480,29 +563,75 @@ class java_lang_ThreadGroup : AllStatic { static void serialize_offsets(SerializeClosure* f) NOT_CDS_RETURN; // parent ThreadGroup - static oop parent(oop java_thread_group); + static oop parent(oop java_thread_group); // name static const char* name(oop java_thread_group); - // ("name as oop" accessor is not necessary) - // Number of threads in group - static int nthreads(oop java_thread_group); - // threads - static objArrayOop threads(oop java_thread_group); - // Number of threads in group - static int ngroups(oop java_thread_group); - // groups - static objArrayOop groups(oop java_thread_group); // maxPriority in group static ThreadPriority maxPriority(oop java_thread_group); - // Destroyed - static bool is_destroyed(oop java_thread_group); // Daemon static bool is_daemon(oop java_thread_group); + + // Number of strongly reachable thread groups + static int ngroups(oop java_thread_group); + // Strongly reachable thread groups + static objArrayOop groups(oop java_thread_group); + // Number of weakly reachable thread groups + static int nweaks(oop java_thread_group); + // Weakly reachable thread groups + static objArrayOop weaks(oop java_thread_group); + // Debugging friend class JavaClasses; }; +// Interface to java.lang.VirtualThread objects + +class java_lang_VirtualThread : AllStatic { + private: + static int static_notify_jvmti_events_offset; + static int static_vthread_scope_offset; + static int _carrierThread_offset; + static int _continuation_offset; + static int _state_offset; + JFR_ONLY(static int _jfr_epoch_offset;) + public: + enum { + NEW = 0, + STARTED = 1, + RUNNABLE = 2, + RUNNING = 3, + PARKING = 4, + PARKED = 5, + PINNED = 6, + YIELDING = 7, + TERMINATED = 99, + + // can be suspended from scheduling when unmounted + SUSPENDED = 1 << 8, + RUNNABLE_SUSPENDED = (RUNNABLE | SUSPENDED), + PARKED_SUSPENDED = (PARKED | SUSPENDED) + }; + + static void compute_offsets(); + static void serialize_offsets(SerializeClosure* f) NOT_CDS_RETURN; + + // Testers + static bool is_subclass(Klass* klass) { + return klass->is_subclass_of(vmClasses::VirtualThread_klass()); + } + static bool is_instance(oop obj); + + static oop vthread_scope(); + static oop carrier_thread(oop vthread); + static oop continuation(oop vthread); + static u2 state(oop vthread); + static JavaThreadStatus map_state_to_thread_status(int state); + static bool notify_jvmti_events(); + static void set_notify_jvmti_events(bool enable); + static void init_static_notify_jvmti_events(); +}; + // Interface to java.lang.Throwable objects @@ -517,9 +646,10 @@ class java_lang_Throwable: AllStatic { trace_bcis_offset = 1, trace_mirrors_offset = 2, trace_names_offset = 3, - trace_next_offset = 4, - trace_hidden_offset = 5, - trace_size = 6, + trace_conts_offset = 4, + trace_next_offset = 5, + trace_hidden_offset = 6, + trace_size = 7, trace_chunk_size = 32 }; @@ -560,8 +690,9 @@ class java_lang_Throwable: AllStatic { // Fill in current stack trace, can cause GC static void fill_in_stack_trace(Handle throwable, const methodHandle& method, TRAPS); static void fill_in_stack_trace(Handle throwable, const methodHandle& method = methodHandle()); + // Programmatic access to stack trace - static void get_stack_trace_elements(Handle throwable, objArrayHandle stack_trace, TRAPS); + static void get_stack_trace_elements(int depth, Handle backtrace, objArrayHandle stack_trace, TRAPS); // For recreating class initialization error exceptions. static Handle get_cause_with_stack_trace(Handle throwable, TRAPS); @@ -905,6 +1036,7 @@ class java_lang_ref_Reference: AllStatic { public: // Accessors static inline oop weak_referent_no_keepalive(oop ref); + static inline oop weak_referent(oop ref); static inline oop phantom_referent_no_keepalive(oop ref); static inline oop unknown_referent_no_keepalive(oop ref); static inline void clear_referent(oop ref); @@ -949,6 +1081,112 @@ class java_lang_ref_SoftReference: public java_lang_ref_Reference { static void serialize_offsets(SerializeClosure* f) NOT_CDS_RETURN; }; +// Interface to jdk.internal.vm.ContinuationScope objects +class jdk_internal_vm_ContinuationScope: AllStatic { + friend class JavaClasses; + private: + static int _name_offset; + + static void compute_offsets(); + public: + static void serialize_offsets(SerializeClosure* f) NOT_CDS_RETURN; + + static inline oop name(oop ref); +}; + +// Interface to jdk.internal.vm.Continuation objects +class jdk_internal_vm_Continuation: AllStatic { + friend class JavaClasses; + private: + static int _scope_offset; + static int _target_offset; + static int _parent_offset; + static int _yieldInfo_offset; + static int _tail_offset; + static int _mounted_offset; + static int _done_offset; + static int _preempted_offset; + + static void compute_offsets(); + public: + static void serialize_offsets(SerializeClosure* f) NOT_CDS_RETURN; + // Accessors + static inline oop scope(oop continuation); + static inline oop target(oop continuation); + static inline oop parent(oop continuation); + static inline oop yieldInfo(oop continuation); + static inline void set_yieldInfo(oop continuation, oop value); + static inline stackChunkOop tail(oop continuation); + static inline void set_tail(oop continuation, stackChunkOop value); + static inline bool on_local_stack(oop continuation, address adr); + static inline bool done(oop continuation); + static inline bool is_preempted(oop continuation); + static inline void set_preempted(oop continuation, bool value); +}; + +// Interface to jdk.internal.vm.StackChunk objects +#define STACKCHUNK_INJECTED_FIELDS(macro) \ + macro(jdk_internal_vm_StackChunk, cont, continuation_signature, false) \ + macro(jdk_internal_vm_StackChunk, flags, byte_signature, false) \ + macro(jdk_internal_vm_StackChunk, pc, intptr_signature, false) \ + macro(jdk_internal_vm_StackChunk, maxThawingSize, int_signature, false) \ + +class jdk_internal_vm_StackChunk: AllStatic { + friend class JavaClasses; + private: + static int _parent_offset; + static int _size_offset; + static int _sp_offset; + static int _pc_offset; + static int _argsize_offset; + static int _flags_offset; + static int _maxThawingSize_offset; + static int _cont_offset; + + + static void compute_offsets(); + public: + static void serialize_offsets(SerializeClosure* f) NOT_CDS_RETURN; + + static inline int parent_offset() { return _parent_offset; } + static inline int cont_offset() { return _cont_offset; } + + // Accessors + static inline oop parent(oop chunk); + static inline void set_parent(oop chunk, oop value); + template + static inline bool is_parent_null(oop chunk); // bypasses barriers for a faster test + template + static inline void set_parent_raw(oop chunk, oop value); + + static inline int size(oop chunk); + static inline void set_size(HeapWord* chunk, int value); + + static inline int sp(oop chunk); + static inline void set_sp(oop chunk, int value); + static inline void set_sp(HeapWord* chunk, int value); // used while allocating + static inline intptr_t pc(oop chunk); + static inline void set_pc(oop chunk, intptr_t value); + static inline int argsize(oop chunk); + static inline void set_argsize(oop chunk, int value); + static inline uint8_t flags(oop chunk); + static inline void set_flags(oop chunk, uint8_t value); + static inline uint8_t flags_acquire(oop chunk); + static inline void release_set_flags(oop chunk, uint8_t value); + static inline bool try_set_flags(oop chunk, uint8_t expected_value, uint8_t new_value); + + static inline int maxThawingSize(oop chunk); + static inline void set_maxThawingSize(oop chunk, int value); + + // cont oop's processing is essential for the chunk's GC protocol + static inline oop cont(oop chunk); + static inline void set_cont(oop chunk, oop value); + template + static inline oop cont_raw(oop chunk); + template + static inline void set_cont_raw(oop chunk, oop value); +}; + // Interface to java.lang.invoke.MethodHandle objects class java_lang_invoke_MethodHandle: AllStatic { @@ -1485,15 +1723,17 @@ private: static int _memberName_offset; static int _bci_offset; static int _version_offset; + static int _contScope_offset; static Method* get_method(Handle stackFrame, InstanceKlass* holder, TRAPS); public: // Setters - static void set_method_and_bci(Handle stackFrame, const methodHandle& method, int bci, TRAPS); + static void set_method_and_bci(Handle stackFrame, const methodHandle& method, int bci, oop cont, TRAPS); static void set_bci(oop info, int value); static void set_version(oop info, short value); + static void set_contScope(oop info, oop value); static void compute_offsets(); static void serialize_offsets(SerializeClosure* f) NOT_CDS_RETURN; @@ -1764,7 +2004,9 @@ class InjectedField { CALLSITECONTEXT_INJECTED_FIELDS(macro) \ STACKFRAMEINFO_INJECTED_FIELDS(macro) \ MODULE_INJECTED_FIELDS(macro) \ - INTERNALERROR_INJECTED_FIELDS(macro) + THREAD_INJECTED_FIELDS(macro) \ + INTERNALERROR_INJECTED_FIELDS(macro) \ + STACKCHUNK_INJECTED_FIELDS(macro) // Interface to hard-coded offset checking diff --git a/src/hotspot/share/classfile/javaClasses.inline.hpp b/src/hotspot/share/classfile/javaClasses.inline.hpp index f3f2b2a7e27..3508d8a6841 100644 --- a/src/hotspot/share/classfile/javaClasses.inline.hpp +++ b/src/hotspot/share/classfile/javaClasses.inline.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2015, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -28,9 +28,13 @@ #include "classfile/javaClasses.hpp" #include "oops/access.inline.hpp" +#include "oops/instanceKlass.inline.hpp" +#include "logging/log.hpp" #include "oops/method.hpp" #include "oops/oop.inline.hpp" #include "oops/oopsHierarchy.hpp" +#include "oops/stackChunkOop.inline.hpp" +#include "oops/typeArrayOop.inline.hpp" void java_lang_String::set_coder(oop string, jbyte coder) { string->byte_field_put(_coder_offset, coder); @@ -133,6 +137,10 @@ oop java_lang_ref_Reference::weak_referent_no_keepalive(oop ref) { return ref->obj_field_access(_referent_offset); } +oop java_lang_ref_Reference::weak_referent(oop ref) { + return ref->obj_field_access(_referent_offset); +} + oop java_lang_ref_Reference::phantom_referent_no_keepalive(oop ref) { return ref->obj_field_access(_referent_offset); } @@ -189,6 +197,183 @@ bool java_lang_ref_Reference::is_phantom(oop ref) { return InstanceKlass::cast(ref->klass())->reference_type() == REF_PHANTOM; } +inline oop java_lang_Thread::continuation(oop java_thread) { + return java_thread->obj_field(_continuation_offset); +} + +inline int64_t java_lang_Thread::thread_id(oop java_thread) { + return java_thread->long_field(_tid_offset); +} + +inline oop java_lang_VirtualThread::vthread_scope() { + oop base = vmClasses::VirtualThread_klass()->static_field_base_raw(); + return base->obj_field(static_vthread_scope_offset); +} + +#if INCLUDE_JFR +inline u2 java_lang_Thread::jfr_epoch(oop ref) { + return ref->short_field(_jfr_epoch_offset); +} + +inline void java_lang_Thread::set_jfr_epoch(oop ref, u2 epoch) { + ref->short_field_put(_jfr_epoch_offset, epoch); +} +#endif // INCLUDE_JFR + +inline oop jdk_internal_vm_ContinuationScope::name(oop ref) { + return ref->obj_field(_name_offset); +} + +inline oop jdk_internal_vm_Continuation::scope(oop continuation) { + return continuation->obj_field(_scope_offset); +} + +inline oop jdk_internal_vm_Continuation::target(oop continuation) { + return continuation->obj_field(_target_offset); +} + +inline oop jdk_internal_vm_Continuation::parent(oop continuation) { + return continuation->obj_field(_parent_offset); +} + +inline oop jdk_internal_vm_Continuation::yieldInfo(oop continuation) { + return continuation->obj_field(_yieldInfo_offset); +} + +inline void jdk_internal_vm_Continuation::set_yieldInfo(oop continuation, oop value) { + continuation->obj_field_put(_yieldInfo_offset, value); +} + +inline stackChunkOop jdk_internal_vm_Continuation::tail(oop continuation) { + return stackChunkOopDesc::cast(continuation->obj_field(_tail_offset)); +} + +inline void jdk_internal_vm_Continuation::set_tail(oop continuation, stackChunkOop value) { + continuation->obj_field_put(_tail_offset, value); +} + +inline bool jdk_internal_vm_Continuation::done(oop continuation) { + return continuation->bool_field(_done_offset); +} + +inline bool jdk_internal_vm_Continuation::is_preempted(oop continuation) { + return continuation->bool_field(_preempted_offset); +} + +inline void jdk_internal_vm_Continuation::set_preempted(oop continuation, bool value) { + continuation->bool_field_put(_preempted_offset, (jboolean)value); +} + +inline oop jdk_internal_vm_StackChunk::parent(oop chunk) { + return chunk->obj_field(_parent_offset); +} + +inline void jdk_internal_vm_StackChunk::set_parent(oop chunk, oop value) { + chunk->obj_field_put(_parent_offset, value); +} + +template +inline bool jdk_internal_vm_StackChunk::is_parent_null(oop chunk) { + return (oop)RawAccess<>::oop_load(chunk->field_addr

(_parent_offset)) == NULL; +} + +template +inline void jdk_internal_vm_StackChunk::set_parent_raw(oop chunk, oop value) { + RawAccess<>::oop_store(chunk->field_addr

(_parent_offset), value); +} + +inline oop jdk_internal_vm_StackChunk::cont(oop chunk) { + return chunk->obj_field(_cont_offset); +} + +inline void jdk_internal_vm_StackChunk::set_cont(oop chunk, oop value) { + chunk->obj_field_put(_cont_offset, value); +} + +template +inline oop jdk_internal_vm_StackChunk::cont_raw(oop chunk) { + return (oop)RawAccess<>::oop_load(chunk->field_addr

(_cont_offset)); +} + +template +inline void jdk_internal_vm_StackChunk::set_cont_raw(oop chunk, oop value) { + RawAccess<>::oop_store(chunk->field_addr

(_cont_offset), value); +} + +inline int jdk_internal_vm_StackChunk::size(oop chunk) { + return chunk->int_field(_size_offset); +} + +inline void jdk_internal_vm_StackChunk::set_size(HeapWord* chunk, int value) { + // Used by StackChunkAllocator before the Object has been finished, + // so don't cast too oop and use int_field_put in this function. + assert(_size_offset != 0, "must be set"); + *(int*)(((char*)chunk) + _size_offset) = (int)value; +} + +inline int jdk_internal_vm_StackChunk::sp(oop chunk) { + return chunk->int_field(_sp_offset); +} + +inline void jdk_internal_vm_StackChunk::set_sp(oop chunk, int value) { + chunk->int_field_put(_sp_offset, value); +} + +inline void jdk_internal_vm_StackChunk::set_sp(HeapWord* chunk, int value) { + // Used by StackChunkAllocator before the Object has been finished, + // so don't cast too oop and use int_field_put in this function. + assert(_sp_offset != 0, "must be set"); + *(int*)(((char*)chunk) + _sp_offset) = (int)value; +} + +inline intptr_t jdk_internal_vm_StackChunk::pc(oop chunk) { + return chunk->long_field(_pc_offset); +} + +inline void jdk_internal_vm_StackChunk::set_pc(oop chunk, intptr_t value) { + chunk->long_field_put(_pc_offset, value); +} + +inline int jdk_internal_vm_StackChunk::argsize(oop chunk) { + return chunk->int_field(_argsize_offset); +} + +inline void jdk_internal_vm_StackChunk::set_argsize(oop chunk, int value) { + chunk->int_field_put(_argsize_offset, value); +} + +inline uint8_t jdk_internal_vm_StackChunk::flags(oop chunk) { + return Atomic::load(chunk->field_addr(_flags_offset)); +} + +inline void jdk_internal_vm_StackChunk::set_flags(oop chunk, uint8_t value) { + Atomic::store(chunk->field_addr(_flags_offset), value); +} + +inline uint8_t jdk_internal_vm_StackChunk::flags_acquire(oop chunk) { + return Atomic::load_acquire(chunk->field_addr(_flags_offset)); +} + +inline void jdk_internal_vm_StackChunk::release_set_flags(oop chunk, uint8_t value) { + Atomic::release_store(chunk->field_addr(_flags_offset), value); +} + +inline bool jdk_internal_vm_StackChunk::try_set_flags(oop chunk, uint8_t expected_value, uint8_t new_value) { + return Atomic::cmpxchg(chunk->field_addr(_flags_offset), expected_value, new_value) == expected_value; +} + +inline int jdk_internal_vm_StackChunk::maxThawingSize(oop chunk) { + return chunk->int_field(_maxThawingSize_offset); +} + +inline void jdk_internal_vm_StackChunk::set_maxThawingSize(oop chunk, int value) { +#ifdef ASSERT + jint old = maxThawingSize(chunk); + log_develop_trace(continuations)("%s max_size: %d -> %d", value >= old ? "add" : "sub", old, value); +#endif + chunk->int_field_put(_maxThawingSize_offset, value); +} + inline void java_lang_invoke_CallSite::set_target_volatile(oop site, oop target) { site->obj_field_put_volatile(_target_offset, target); } diff --git a/src/hotspot/share/classfile/vmClassMacros.hpp b/src/hotspot/share/classfile/vmClassMacros.hpp index 2b65130f1da..207207011b9 100644 --- a/src/hotspot/share/classfile/vmClassMacros.hpp +++ b/src/hotspot/share/classfile/vmClassMacros.hpp @@ -87,7 +87,10 @@ do_klass(Finalizer_klass, java_lang_ref_Finalizer ) \ \ do_klass(Thread_klass, java_lang_Thread ) \ + do_klass(Thread_FieldHolder_klass, java_lang_Thread_FieldHolder ) \ + do_klass(Thread_Constants_klass, java_lang_Thread_Constants ) \ do_klass(ThreadGroup_klass, java_lang_ThreadGroup ) \ + do_klass(VirtualThread_klass, java_lang_VirtualThread ) \ do_klass(Properties_klass, java_util_Properties ) \ do_klass(Module_klass, java_lang_Module ) \ do_klass(reflect_AccessibleObject_klass, java_lang_reflect_AccessibleObject ) \ @@ -96,6 +99,11 @@ do_klass(reflect_Method_klass, java_lang_reflect_Method ) \ do_klass(reflect_Constructor_klass, java_lang_reflect_Constructor ) \ \ + do_klass(Runnable_klass, java_lang_Runnable ) \ + do_klass(ContinuationScope_klass, jdk_internal_vm_ContinuationScope ) \ + do_klass(Continuation_klass, jdk_internal_vm_Continuation ) \ + do_klass(StackChunk_klass, jdk_internal_vm_StackChunk ) \ + \ /* NOTE: needed too early in bootstrapping process to have checks based on JDK version */ \ /* It's okay if this turns out to be NULL in non-1.4 JDKs. */ \ do_klass(reflect_MagicAccessorImpl_klass, reflect_MagicAccessorImpl ) \ diff --git a/src/hotspot/share/classfile/vmClasses.cpp b/src/hotspot/share/classfile/vmClasses.cpp index 66e802c74df..161b0b7bed3 100644 --- a/src/hotspot/share/classfile/vmClasses.cpp +++ b/src/hotspot/share/classfile/vmClasses.cpp @@ -35,6 +35,7 @@ #include "memory/universe.hpp" #include "oops/instanceKlass.hpp" #include "oops/instanceRefKlass.hpp" +#include "oops/instanceStackChunkKlass.hpp" #include "prims/jvmtiExport.hpp" #include "runtime/globals.hpp" @@ -215,6 +216,8 @@ void vmClasses::resolve_all(TRAPS) { } } #endif + + InstanceStackChunkKlass::init_offset_of_stack(); } #if INCLUDE_CDS diff --git a/src/hotspot/share/classfile/vmIntrinsics.cpp b/src/hotspot/share/classfile/vmIntrinsics.cpp index a329669bed3..deb1982b16e 100644 --- a/src/hotspot/share/classfile/vmIntrinsics.cpp +++ b/src/hotspot/share/classfile/vmIntrinsics.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -75,7 +75,9 @@ bool vmIntrinsics::preserves_state(vmIntrinsics::ID id) { case vmIntrinsics::_longBitsToDouble: case vmIntrinsics::_getClass: case vmIntrinsics::_isInstance: + case vmIntrinsics::_currentCarrierThread: case vmIntrinsics::_currentThread: + case vmIntrinsics::_extentLocalCache: case vmIntrinsics::_dabs: case vmIntrinsics::_fabs: case vmIntrinsics::_iabs: @@ -92,6 +94,7 @@ bool vmIntrinsics::preserves_state(vmIntrinsics::ID id) { case vmIntrinsics::_Preconditions_checkIndex: case vmIntrinsics::_Preconditions_checkLongIndex: case vmIntrinsics::_Reference_get: + case vmIntrinsics::_Continuation_doYield: case vmIntrinsics::_updateCRC32: case vmIntrinsics::_updateBytesCRC32: case vmIntrinsics::_updateByteBufferCRC32: @@ -114,7 +117,6 @@ bool vmIntrinsics::can_trap(vmIntrinsics::ID id) { switch(id) { #ifdef JFR_HAVE_INTRINSICS case vmIntrinsics::_counterTime: - case vmIntrinsics::_getClassId: #endif case vmIntrinsics::_currentTimeMillis: case vmIntrinsics::_nanoTime: @@ -122,7 +124,11 @@ bool vmIntrinsics::can_trap(vmIntrinsics::ID id) { case vmIntrinsics::_intBitsToFloat: case vmIntrinsics::_doubleToRawLongBits: case vmIntrinsics::_longBitsToDouble: + case vmIntrinsics::_currentCarrierThread: case vmIntrinsics::_currentThread: + case vmIntrinsics::_setCurrentThread: + case vmIntrinsics::_extentLocalCache: + case vmIntrinsics::_setExtentLocalCache: case vmIntrinsics::_dabs: case vmIntrinsics::_fabs: case vmIntrinsics::_iabs: @@ -231,6 +237,7 @@ bool vmIntrinsics::disabled_by_jvm_flags(vmIntrinsics::ID id) { case vmIntrinsics::_fullFence: case vmIntrinsics::_countPositives: case vmIntrinsics::_Reference_get: + case vmIntrinsics::_Continuation_doYield: break; default: return true; @@ -252,9 +259,13 @@ bool vmIntrinsics::disabled_by_jvm_flags(vmIntrinsics::ID id) { case vmIntrinsics::_getClass: if (!InlineClassNatives) return true; break; + case vmIntrinsics::_currentCarrierThread: case vmIntrinsics::_currentThread: if (!InlineThreadNatives) return true; break; + case vmIntrinsics::_setCurrentThread: + case vmIntrinsics::_extentLocalCache: + case vmIntrinsics::_setExtentLocalCache: case vmIntrinsics::_floatToRawIntBits: case vmIntrinsics::_intBitsToFloat: case vmIntrinsics::_doubleToRawLongBits: diff --git a/src/hotspot/share/classfile/vmIntrinsics.hpp b/src/hotspot/share/classfile/vmIntrinsics.hpp index dd60d2dd0af..67ddd1fbf0f 100644 --- a/src/hotspot/share/classfile/vmIntrinsics.hpp +++ b/src/hotspot/share/classfile/vmIntrinsics.hpp @@ -253,9 +253,20 @@ class methodHandle; do_intrinsic(_arraycopy, java_lang_System, arraycopy_name, arraycopy_signature, F_SN) \ do_name( arraycopy_name, "arraycopy") \ do_signature(arraycopy_signature, "(Ljava/lang/Object;ILjava/lang/Object;II)V") \ + \ + do_intrinsic(_currentCarrierThread, java_lang_Thread, currentCarrierThread_name, currentThread_signature, F_SN) \ + do_name( currentCarrierThread_name, "currentCarrierThread") \ do_intrinsic(_currentThread, java_lang_Thread, currentThread_name, currentThread_signature, F_SN) \ do_name( currentThread_name, "currentThread") \ do_signature(currentThread_signature, "()Ljava/lang/Thread;") \ + do_intrinsic(_extentLocalCache, java_lang_Thread, extentLocalCache_name, extentLocalCache_signature, F_SN) \ + do_name( extentLocalCache_name, "extentLocalCache") \ + do_signature(extentLocalCache_signature, "()[Ljava/lang/Object;") \ + do_intrinsic(_setExtentLocalCache, java_lang_Thread, setExtentLocalCache_name, setExtentLocalCache_signature, F_SN) \ + do_name( setExtentLocalCache_name, "setExtentLocalCache") \ + do_signature(setExtentLocalCache_signature, "([Ljava/lang/Object;)V") \ + do_intrinsic(_setCurrentThread, java_lang_Thread, setCurrentThread_name, thread_void_signature, F_RN) \ + do_name( setCurrentThread_name, "setCurrentThread") \ \ /* reflective intrinsics, for java/lang/Class, etc. */ \ do_intrinsic(_isAssignableFrom, java_lang_Class, isAssignableFrom_name, class_boolean_signature, F_RN) \ @@ -513,6 +524,17 @@ class methodHandle; do_intrinsic(_updateByteBufferAdler32, java_util_zip_Adler32, updateByteBuffer_A_name, updateByteBuffer_signature, F_SN) \ do_name( updateByteBuffer_A_name, "updateByteBuffer") \ \ + /* jdk/internal/vm/Continuation */ \ + do_class(jdk_internal_vm_Continuation, "jdk/internal/vm/Continuation") \ + do_intrinsic(_Continuation_enter, jdk_internal_vm_Continuation, enter_name, continuationEnter_signature, F_S) \ + do_signature(continuationEnter_signature, "(Ljdk/internal/vm/Continuation;Z)V") \ + do_intrinsic(_Continuation_enterSpecial, jdk_internal_vm_Continuation, enterSpecial_name, continuationEnterSpecial_signature, F_SN) \ + do_signature(continuationEnterSpecial_signature, "(Ljdk/internal/vm/Continuation;ZZ)V") \ + do_signature(continuationGetStacks_signature, "(III)V") \ + do_alias(continuationOnPinned_signature, int_void_signature) \ + do_intrinsic(_Continuation_doYield, jdk_internal_vm_Continuation, doYield_name, continuationDoYield_signature, F_S) \ + do_alias( continuationDoYield_signature, void_int_signature) \ + \ /* support for UnsafeConstants */ \ do_class(jdk_internal_misc_UnsafeConstants, "jdk/internal/misc/UnsafeConstants") \ \ diff --git a/src/hotspot/share/classfile/vmSymbols.hpp b/src/hotspot/share/classfile/vmSymbols.hpp index 0381c67deed..83f033abb79 100644 --- a/src/hotspot/share/classfile/vmSymbols.hpp +++ b/src/hotspot/share/classfile/vmSymbols.hpp @@ -62,11 +62,17 @@ template(java_lang_StringLatin1, "java/lang/StringLatin1") \ template(java_lang_StringUTF16, "java/lang/StringUTF16") \ template(java_lang_Thread, "java/lang/Thread") \ + template(java_lang_Thread_FieldHolder, "java/lang/Thread$FieldHolder") \ + template(java_lang_Thread_Constants, "java/lang/Thread$Constants") \ template(java_lang_ThreadGroup, "java/lang/ThreadGroup") \ + template(java_lang_VirtualThread, "java/lang/VirtualThread") \ template(java_lang_Cloneable, "java/lang/Cloneable") \ template(java_lang_Throwable, "java/lang/Throwable") \ template(java_lang_ClassLoader, "java/lang/ClassLoader") \ template(java_lang_ThreadDeath, "java/lang/ThreadDeath") \ + template(java_lang_Runnable, "java/lang/Runnable") \ + template(jdk_internal_vm_ContinuationScope, "jdk/internal/vm/ContinuationScope") \ + template(jdk_internal_vm_StackChunk, "jdk/internal/vm/StackChunk") \ template(java_lang_Boolean, "java/lang/Boolean") \ template(java_lang_Character, "java/lang/Character") \ template(java_lang_Character_CharacterCache, "java/lang/Character$CharacterCache") \ @@ -310,6 +316,10 @@ template(jdk_internal_misc_Scoped_signature, "Ljdk/internal/misc/ScopedMemoryAccess$Scoped;") \ template(jdk_internal_vm_annotation_IntrinsicCandidate_signature, "Ljdk/internal/vm/annotation/IntrinsicCandidate;") \ template(jdk_internal_vm_annotation_Stable_signature, "Ljdk/internal/vm/annotation/Stable;") \ + \ + template(jdk_internal_vm_annotation_ChangesCurrentThread_signature, "Ljdk/internal/vm/annotation/ChangesCurrentThread;") \ + template(jdk_internal_vm_annotation_JvmtiMountTransition_signature, "Ljdk/internal/vm/annotation/JvmtiMountTransition;") \ + \ /* Support for JSR 292 & invokedynamic (JDK 1.7 and above) */ \ template(java_lang_invoke_CallSite, "java/lang/invoke/CallSite") \ template(java_lang_invoke_ConstantCallSite, "java/lang/invoke/ConstantCallSite") \ @@ -382,16 +392,15 @@ template(group_name, "group") \ template(daemon_name, "daemon") \ template(run_method_name, "run") \ + template(interrupt_method_name, "interrupt") \ template(exit_method_name, "exit") \ - template(add_method_name, "add") \ template(remove_method_name, "remove") \ template(parent_name, "parent") \ - template(threads_name, "threads") \ - template(groups_name, "groups") \ - template(maxPriority_name, "maxPriority") \ - template(destroyed_name, "destroyed") \ - template(nthreads_name, "nthreads") \ template(ngroups_name, "ngroups") \ + template(groups_name, "groups") \ + template(nweaks_name, "nweaks") \ + template(weaks_name, "weaks") \ + template(maxPriority_name, "maxPriority") \ template(shutdown_name, "shutdown") \ template(finalize_method_name, "finalize") \ template(reference_lock_name, "lock") \ @@ -399,6 +408,33 @@ template(run_finalization_name, "runFinalization") \ template(dispatchUncaughtException_name, "dispatchUncaughtException") \ template(loadClass_name, "loadClass") \ + template(doYield_name, "doYield") \ + template(enter_name, "enter") \ + template(enterSpecial_name, "enterSpecial") \ + template(onContinue_name, "onContinue0") \ + template(getStacks_name, "getStacks") \ + template(onPinned_name, "onPinned0") \ + template(scope_name, "scope") \ + template(yieldInfo_name, "yieldInfo") \ + template(tail_name, "tail") \ + template(size_name, "size") \ + template(argsize_name, "argsize") \ + template(mode_name, "mode") \ + template(numFrames_name, "numFrames") \ + template(numOops_name, "numOops") \ + template(stack_name, "stack") \ + template(maxSize_name, "maxSize") \ + template(reset_name, "reset") \ + template(done_name, "done") \ + template(mounted_name, "mounted") \ + template(numInterpretedFrames_name, "numInterpretedFrames") \ + template(jfrTraceId_name, "jfrTraceId") \ + template(fp_name, "fp") \ + template(sp_name, "sp") \ + template(pc_name, "pc") \ + template(cs_name, "cs") \ + template(refStack_name, "refStack") \ + template(refSP_name, "refSP") \ template(get_name, "get") \ template(refersTo0_name, "refersTo0") \ template(put_name, "put") \ @@ -418,7 +454,7 @@ template(contextClassLoader_name, "contextClassLoader") \ template(inheritedAccessControlContext_name, "inheritedAccessControlContext") \ template(getClassContext_name, "getClassContext") \ - template(wait_name, "wait") \ + template(wait_name, "wait0") \ template(checkPackageAccess_name, "checkPackageAccess") \ template(newInstance0_name, "newInstance0") \ template(forName_name, "forName") \ @@ -460,6 +496,7 @@ template(signers_name, "signers_name") \ template(source_file_name, "source_file") \ template(loader_data_name, "loader_data") \ + template(cont_name, "cont") \ template(vmdependencies_name, "vmdependencies") \ template(last_cleanup_name, "last_cleanup") \ template(loader_name, "loader") \ @@ -479,6 +516,7 @@ template(java_lang_Byte_array_signature, "[Ljava/lang/Byte;") \ template(java_lang_Boolean_signature, "Ljava/lang/Boolean;") \ template(url_code_signer_array_void_signature, "(Ljava/net/URL;[Ljava/security/CodeSigner;)V") \ + template(jvmti_thread_state_name, "jvmti_thread_state") \ template(module_entry_name, "module_entry") \ template(resolved_references_name, "") \ template(init_lock_name, "") \ @@ -489,6 +527,8 @@ template(data_cache_line_flush_size_name, "DATA_CACHE_LINE_FLUSH_SIZE") \ template(during_unsafe_access_name, "during_unsafe_access") \ template(checkIndex_name, "checkIndex") \ + template(jfr_epoch_name, "jfr_epoch") \ + template(maxThawingSize_name, "maxThawingSize") \ \ /* name symbols needed by intrinsics */ \ VM_INTRINSICS_DO(VM_INTRINSIC_IGNORE, VM_SYMBOL_IGNORE, template, VM_SYMBOL_IGNORE, VM_ALIAS_IGNORE) \ @@ -532,6 +572,11 @@ template(char_array_signature, "[C") \ template(int_array_signature, "[I") \ template(long_array_signature, "[J") \ + template(runnable_signature, "Ljava/lang/Runnable;") \ + template(continuation_signature, "Ljdk/internal/vm/Continuation;") \ + template(continuationscope_signature, "Ljdk/internal/vm/ContinuationScope;") \ + template(stackchunk_signature, "Ljdk/internal/vm/StackChunk;") \ + template(vthread_signature, "Ljava/lang/VirtualThread;") \ template(object_void_signature, "(Ljava/lang/Object;)V") \ template(object_int_signature, "(Ljava/lang/Object;)I") \ template(long_object_long_signature, "(JLjava/lang/Object;)J") \ @@ -539,6 +584,8 @@ template(object_object_signature, "(Ljava/lang/Object;)Ljava/lang/Object;") \ template(string_void_signature, "(Ljava/lang/String;)V") \ template(string_int_signature, "(Ljava/lang/String;)I") \ + template(string_byte_array_signature, "(Ljava/lang/String;)[B") \ + template(string_bool_byte_array_signature, "(Ljava/lang/String;Z)[B") \ template(throwable_signature, "Ljava/lang/Throwable;") \ template(throwable_void_signature, "(Ljava/lang/Throwable;)V") \ template(void_throwable_signature, "()Ljava/lang/Throwable;") \ @@ -564,7 +611,7 @@ template(char_array_void_signature, "([C)V") \ template(int_int_void_signature, "(II)V") \ template(long_long_void_signature, "(JJ)V") \ - template(void_byte_array_signature, "()[B") \ + template(void_byte_array_signature, "()[B") \ template(void_classloader_signature, "()Ljava/lang/ClassLoader;") \ template(void_BuiltinClassLoader_signature, "()Ljdk/internal/loader/BuiltinClassLoader;") \ template(void_object_signature, "()Ljava/lang/Object;") \ @@ -579,6 +626,7 @@ template(accesscontrolcontext_signature, "Ljava/security/AccessControlContext;") \ template(class_protectiondomain_signature, "(Ljava/lang/Class;Ljava/security/ProtectionDomain;)V") \ template(thread_signature, "Ljava/lang/Thread;") \ + template(thread_fieldholder_signature, "Ljava/lang/Thread$FieldHolder;") \ template(thread_array_signature, "[Ljava/lang/Thread;") \ template(threadgroup_signature, "Ljava/lang/ThreadGroup;") \ template(threadgroup_array_signature, "[Ljava/lang/ThreadGroup;") \ @@ -591,6 +639,7 @@ template(string_array_signature, "[Ljava/lang/String;") \ template(reference_signature, "Ljava/lang/ref/Reference;") \ template(referencequeue_signature, "Ljava/lang/ref/ReferenceQueue;") \ + template(weakreference_array_signature, "[Ljava/lang/ref/WeakReference;") \ template(executable_signature, "Ljava/lang/reflect/Executable;") \ template(module_signature, "Ljava/lang/Module;") \ template(concurrenthashmap_signature, "Ljava/util/concurrent/ConcurrentHashMap;") \ @@ -718,6 +767,11 @@ template(url_void_signature, "(Ljava/net/URL;)V") \ template(url_array_classloader_void_signature, "([Ljava/net/URL;Ljava/lang/ClassLoader;)V") \ \ + /* Thread.dump_to_file jcmd */ \ + template(jdk_internal_vm_ThreadDumper, "jdk/internal/vm/ThreadDumper") \ + template(dumpThreads_name, "dumpThreads") \ + template(dumpThreadsToJson_name, "dumpThreadsToJson") \ + /*end*/ // enum for figuring positions and size of Symbol::_vm_symbols[] diff --git a/src/hotspot/share/code/codeBlob.cpp b/src/hotspot/share/code/codeBlob.cpp index a1fcea77e48..0b45161958c 100644 --- a/src/hotspot/share/code/codeBlob.cpp +++ b/src/hotspot/share/code/codeBlob.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1998, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1998, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -78,7 +78,7 @@ unsigned int CodeBlob::allocation_size(CodeBuffer* cb, int header_size) { return size; } -CodeBlob::CodeBlob(const char* name, CompilerType type, const CodeBlobLayout& layout, int frame_complete_offset, int frame_size, ImmutableOopMapSet* oop_maps, bool caller_must_gc_arguments) : +CodeBlob::CodeBlob(const char* name, CompilerType type, const CodeBlobLayout& layout, int frame_complete_offset, int frame_size, ImmutableOopMapSet* oop_maps, bool caller_must_gc_arguments, bool compiled) : _type(type), _size(layout.size()), _header_size(layout.header_size()), @@ -93,6 +93,7 @@ CodeBlob::CodeBlob(const char* name, CompilerType type, const CodeBlobLayout& la _relocation_end(layout.relocation_end()), _oop_maps(oop_maps), _caller_must_gc_arguments(caller_must_gc_arguments), + _is_compiled(compiled), _name(name) { assert(is_aligned(layout.size(), oopSize), "unaligned size"); @@ -106,7 +107,7 @@ CodeBlob::CodeBlob(const char* name, CompilerType type, const CodeBlobLayout& la S390_ONLY(_ctable_offset = 0;) // avoid uninitialized fields } -CodeBlob::CodeBlob(const char* name, CompilerType type, const CodeBlobLayout& layout, CodeBuffer* cb /*UNUSED*/, int frame_complete_offset, int frame_size, OopMapSet* oop_maps, bool caller_must_gc_arguments) : +CodeBlob::CodeBlob(const char* name, CompilerType type, const CodeBlobLayout& layout, CodeBuffer* cb /*UNUSED*/, int frame_complete_offset, int frame_size, OopMapSet* oop_maps, bool caller_must_gc_arguments, bool compiled) : _type(type), _size(layout.size()), _header_size(layout.header_size()), @@ -120,6 +121,7 @@ CodeBlob::CodeBlob(const char* name, CompilerType type, const CodeBlobLayout& la _relocation_begin(layout.relocation_begin()), _relocation_end(layout.relocation_end()), _caller_must_gc_arguments(caller_must_gc_arguments), + _is_compiled(compiled), _name(name) { assert(is_aligned(_size, oopSize), "unaligned size"); @@ -211,7 +213,7 @@ void RuntimeBlob::trace_new_stub(RuntimeBlob* stub, const char* name1, const cha MemoryService::track_code_cache_memory_usage(); } -const ImmutableOopMap* CodeBlob::oop_map_for_return_address(address return_address) { +const ImmutableOopMap* CodeBlob::oop_map_for_return_address(address return_address) const { assert(_oop_maps != NULL, "nope"); return _oop_maps->find_map_at_offset((intptr_t) return_address - (intptr_t) code_begin()); } diff --git a/src/hotspot/share/code/codeBlob.hpp b/src/hotspot/share/code/codeBlob.hpp index 3994fbfd42c..3f744fae633 100644 --- a/src/hotspot/share/code/codeBlob.hpp +++ b/src/hotspot/share/code/codeBlob.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1998, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1998, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -27,6 +27,7 @@ #include "asm/codeBuffer.hpp" #include "compiler/compilerDefinitions.hpp" +#include "compiler/oopMap.hpp" #include "runtime/javaFrameAnchor.hpp" #include "runtime/frame.hpp" #include "runtime/handles.hpp" @@ -109,6 +110,8 @@ protected: ImmutableOopMapSet* _oop_maps; // OopMap for this CodeBlob bool _caller_must_gc_arguments; + bool _is_compiled; + const char* _name; S390_ONLY(int _ctable_offset;) @@ -122,9 +125,12 @@ protected: } #endif // not PRODUCT - CodeBlob(const char* name, CompilerType type, const CodeBlobLayout& layout, int frame_complete_offset, int frame_size, ImmutableOopMapSet* oop_maps, bool caller_must_gc_arguments); - CodeBlob(const char* name, CompilerType type, const CodeBlobLayout& layout, CodeBuffer* cb, int frame_complete_offset, int frame_size, OopMapSet* oop_maps, bool caller_must_gc_arguments); - + CodeBlob(const char* name, CompilerType type, const CodeBlobLayout& layout, int frame_complete_offset, + int frame_size, ImmutableOopMapSet* oop_maps, + bool caller_must_gc_arguments, bool compiled = false); + CodeBlob(const char* name, CompilerType type, const CodeBlobLayout& layout, CodeBuffer* cb, int frame_complete_offset, + int frame_size, OopMapSet* oop_maps, + bool caller_must_gc_arguments, bool compiled = false); public: // Only used by unit test. CodeBlob() : _type(compiler_none) {} @@ -147,8 +153,9 @@ public: virtual bool is_adapter_blob() const { return false; } virtual bool is_vtable_blob() const { return false; } virtual bool is_method_handles_adapter_blob() const { return false; } - virtual bool is_compiled() const { return false; } - virtual bool is_optimized_entry_blob() const { return false; } + virtual bool is_optimized_entry_blob() const { return false; } + bool is_compiled() const { return _is_compiled; } + const bool* is_compiled_addr() const { return &_is_compiled; } inline bool is_compiled_by_c1() const { return _type == compiler_c1; }; inline bool is_compiled_by_c2() const { return _type == compiler_c2; }; @@ -217,7 +224,9 @@ public: // OopMap for frame ImmutableOopMapSet* oop_maps() const { return _oop_maps; } void set_oop_maps(OopMapSet* p); - const ImmutableOopMap* oop_map_for_return_address(address return_address); + + const ImmutableOopMap* oop_map_for_slot(int slot, address return_address) const; + const ImmutableOopMap* oop_map_for_return_address(address return_address) const; virtual void preserve_callee_argument_oops(frame fr, const RegisterMap* reg_map, OopClosure* f) = 0; // Frame support. Sizes are in word units. diff --git a/src/hotspot/share/code/codeBlob.inline.hpp b/src/hotspot/share/code/codeBlob.inline.hpp new file mode 100644 index 00000000000..9fce879a67e --- /dev/null +++ b/src/hotspot/share/code/codeBlob.inline.hpp @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_CODE_CODEBLOB_INLINE_HPP +#define SHARE_CODE_CODEBLOB_INLINE_HPP + +#include "code/codeBlob.hpp" + +#include "compiler/oopMap.inline.hpp" +#include "utilities/debug.hpp" +#include "utilities/globalDefinitions.hpp" + +inline const ImmutableOopMap* CodeBlob::oop_map_for_slot(int slot, address return_address) const { + assert(_oop_maps != NULL, "nope"); + return _oop_maps->find_map_at_slot(slot, (intptr_t) return_address - (intptr_t) code_begin()); +} + +#endif // SHARE_CODE_CODEBLOB_INLINE_HPP diff --git a/src/hotspot/share/code/codeCache.cpp b/src/hotspot/share/code/codeCache.cpp index 3f367afa900..2affd893fc2 100644 --- a/src/hotspot/share/code/codeCache.cpp +++ b/src/hotspot/share/code/codeCache.cpp @@ -36,6 +36,7 @@ #include "compiler/compilationPolicy.hpp" #include "compiler/compileBroker.hpp" #include "compiler/oopMap.hpp" +#include "gc/shared/barrierSetNMethod.hpp" #include "gc/shared/collectedHeap.hpp" #include "jfr/jfrEvents.hpp" #include "logging/log.hpp" @@ -167,6 +168,9 @@ address CodeCache::_high_bound = 0; int CodeCache::_number_of_nmethods_with_dependencies = 0; ExceptionCache* volatile CodeCache::_exception_cache_purge_list = NULL; +int CodeCache::Sweep::_compiled_method_iterators = 0; +bool CodeCache::Sweep::_pending_sweep = false; + // Initialize arrays of CodeHeap subsets GrowableArray* CodeCache::_heaps = new(ResourceObj::C_HEAP, mtCode) GrowableArray (CodeBlobType::All, mtCode); GrowableArray* CodeCache::_compiled_heaps = new(ResourceObj::C_HEAP, mtCode) GrowableArray (CodeBlobType::All, mtCode); @@ -474,6 +478,40 @@ CodeHeap* CodeCache::get_code_heap(int code_blob_type) { return NULL; } +void CodeCache::Sweep::begin_compiled_method_iteration() { + MutexLocker ml(CodeCache_lock, Mutex::_no_safepoint_check_flag); + // Reach a state without concurrent sweeping + while (_compiled_method_iterators < 0) { + CodeCache_lock->wait_without_safepoint_check(); + } + _compiled_method_iterators++; +} + +void CodeCache::Sweep::end_compiled_method_iteration() { + MutexLocker ml(CodeCache_lock, Mutex::_no_safepoint_check_flag); + // Let the sweeper run again, if we stalled it + _compiled_method_iterators--; + if (_pending_sweep) { + CodeCache_lock->notify_all(); + } +} + +void CodeCache::Sweep::begin() { + assert_locked_or_safepoint(CodeCache_lock); + _pending_sweep = true; + while (_compiled_method_iterators > 0) { + CodeCache_lock->wait_without_safepoint_check(); + } + _pending_sweep = false; + _compiled_method_iterators = -1; +} + +void CodeCache::Sweep::end() { + assert_locked_or_safepoint(CodeCache_lock); + _compiled_method_iterators = 0; + CodeCache_lock->notify_all(); +} + CodeBlob* CodeCache::first_blob(CodeHeap* heap) { assert_locked_or_safepoint(CodeCache_lock); assert(heap != NULL, "heap is null"); @@ -1139,7 +1177,9 @@ void CodeCache::mark_all_nmethods_for_evol_deoptimization() { while(iter.next()) { CompiledMethod* nm = iter.method(); if (!nm->method()->is_method_handle_intrinsic()) { - nm->mark_for_deoptimization(); + if (nm->can_be_deoptimized()) { + nm->mark_for_deoptimization(); + } if (nm->has_evol_metadata()) { add_to_old_table(nm); } @@ -1191,17 +1231,23 @@ int CodeCache::mark_for_deoptimization(Method* dependee) { return number_of_marked_CodeBlobs; } -void CodeCache::make_marked_nmethods_not_entrant() { - assert_locked_or_safepoint(CodeCache_lock); - CompiledMethodIterator iter(CompiledMethodIterator::only_alive_and_not_unloading); +void CodeCache::make_marked_nmethods_deoptimized() { + SweeperBlockingCompiledMethodIterator iter(SweeperBlockingCompiledMethodIterator::only_alive_and_not_unloading); while(iter.next()) { CompiledMethod* nm = iter.method(); - if (nm->is_marked_for_deoptimization()) { + if (nm->is_marked_for_deoptimization() && !nm->has_been_deoptimized() && nm->can_be_deoptimized()) { nm->make_not_entrant(); + make_nmethod_deoptimized(nm); } } } +void CodeCache::make_nmethod_deoptimized(CompiledMethod* nm) { + if (nm->is_marked_for_deoptimization() && nm->can_be_deoptimized()) { + nm->make_deoptimized(); + } +} + // Flushes compiled methods dependent on dependee. void CodeCache::flush_dependents_on(InstanceKlass* dependee) { assert_lock_strong(Compile_lock); diff --git a/src/hotspot/share/code/codeCache.hpp b/src/hotspot/share/code/codeCache.hpp index b6a6cae685a..0e2414ffa3a 100644 --- a/src/hotspot/share/code/codeCache.hpp +++ b/src/hotspot/share/code/codeCache.hpp @@ -76,11 +76,12 @@ class ExceptionCache; class KlassDepChange; class OopClosure; class ShenandoahParallelCodeHeapIterator; +class NativePostCallNop; class CodeCache : AllStatic { friend class VMStructs; friend class JVMCIVMStructs; - template friend class CodeBlobIterator; + template friend class CodeBlobIterator; friend class WhiteBox; friend class CodeCacheLoader; friend class ShenandoahParallelCodeHeapIterator; @@ -115,7 +116,23 @@ class CodeCache : AllStatic { static CodeBlob* first_blob(CodeHeap* heap); // Returns the first CodeBlob on the given CodeHeap static CodeBlob* first_blob(int code_blob_type); // Returns the first CodeBlob of the given type static CodeBlob* next_blob(CodeHeap* heap, CodeBlob* cb); // Returns the next CodeBlob on the given CodeHeap + public: + class Sweep { + friend class CodeCache; + template friend class CodeBlobIterator; + private: + static int _compiled_method_iterators; + static bool _pending_sweep; + public: + static void begin(); + static void end(); + private: + static void begin_compiled_method_iteration(); + static void end_compiled_method_iteration(); + }; + + private: static size_t bytes_allocated_in_freelists(); static int allocated_segments(); static size_t freelists_length(); @@ -152,6 +169,9 @@ class CodeCache : AllStatic { // Lookup static CodeBlob* find_blob(void* start); // Returns the CodeBlob containing the given address static CodeBlob* find_blob_unsafe(void* start); // Same as find_blob but does not fail if looking up a zombie method + static CodeBlob* find_blob_fast(void* start); // Returns the CodeBlob containing the given address + static CodeBlob* find_blob_and_oopmap(void* start, int& slot); // Returns the CodeBlob containing the given address + static int find_oopmap_slot_fast(void* start); // Returns a fast oopmap slot if there is any; -1 otherwise static nmethod* find_nmethod(void* start); // Returns the nmethod containing the given address static CompiledMethod* find_compiled(void* start); @@ -179,7 +199,9 @@ class CodeCache : AllStatic { static void do_unloading(BoolObjectClosure* is_alive, bool unloading_occurred); static uint8_t unloading_cycle() { return _unloading_cycle; } + static void increment_unloading_cycle(); + static void release_exception_cache(ExceptionCache* entry); static void purge_exception_caches(); @@ -267,7 +289,8 @@ class CodeCache : AllStatic { public: static void mark_all_nmethods_for_deoptimization(); static int mark_for_deoptimization(Method* dependee); - static void make_marked_nmethods_not_entrant(); + static void make_marked_nmethods_deoptimized(); + static void make_nmethod_deoptimized(CompiledMethod* nm); // Flushing and deoptimization static void flush_dependents_on(InstanceKlass* dependee); @@ -304,8 +327,8 @@ class CodeCache : AllStatic { }; -// Iterator to iterate over nmethods in the CodeCache. -template class CodeBlobIterator : public StackObj { +// Iterator to iterate over code blobs in the CodeCache. +template class CodeBlobIterator : public StackObj { public: enum LivenessFilter { all_blobs, only_alive, only_alive_and_not_unloading }; @@ -316,11 +339,7 @@ template class CodeBlobIterator : public StackObj { bool _only_alive; bool _only_not_unloading; - public: - CodeBlobIterator(LivenessFilter filter, T* nm = NULL) - : _only_alive(filter == only_alive || filter == only_alive_and_not_unloading), - _only_not_unloading(filter == only_alive_and_not_unloading) - { + void initialize_iteration(T* nm) { if (Filter::heaps() == NULL) { return; } @@ -336,10 +355,7 @@ template class CodeBlobIterator : public StackObj { } } - // Advance iterator to next blob - bool next() { - assert_locked_or_safepoint(CodeCache_lock); - + bool next_impl() { for (;;) { // Walk through heaps as required if (!next_blob()) { @@ -367,6 +383,36 @@ template class CodeBlobIterator : public StackObj { } } + public: + CodeBlobIterator(LivenessFilter filter, T* nm = NULL) + : _only_alive(filter == only_alive || filter == only_alive_and_not_unloading), + _only_not_unloading(filter == only_alive_and_not_unloading) + { + if (is_compiled_method) { + CodeCache::Sweep::begin_compiled_method_iteration(); + initialize_iteration(nm); + } else { + initialize_iteration(nm); + } + } + + ~CodeBlobIterator() { + if (is_compiled_method) { + CodeCache::Sweep::end_compiled_method_iteration(); + } + } + + // Advance iterator to next blob + bool next() { + if (is_compiled_method) { + MutexLocker ml(CodeCache_lock, Mutex::_no_safepoint_check_flag); + return next_impl(); + } else { + assert_locked_or_safepoint(CodeCache_lock); + return next_impl(); + } + } + bool end() const { return _code_blob == NULL; } T* method() const { return (T*)_code_blob; } @@ -396,7 +442,6 @@ private: } }; - struct CompiledMethodFilter { static bool apply(CodeBlob* cb) { return cb->is_compiled(); } static const GrowableArray* heaps() { return CodeCache::compiled_heaps(); } @@ -413,8 +458,10 @@ struct AllCodeBlobsFilter { static const GrowableArray* heaps() { return CodeCache::heaps(); } }; -typedef CodeBlobIterator CompiledMethodIterator; -typedef CodeBlobIterator NMethodIterator; -typedef CodeBlobIterator AllCodeBlobsIterator; +typedef CodeBlobIterator CompiledMethodIterator; +typedef CodeBlobIterator NMethodIterator; +typedef CodeBlobIterator AllCodeBlobsIterator; + +typedef CodeBlobIterator SweeperBlockingCompiledMethodIterator; #endif // SHARE_CODE_CODECACHE_HPP diff --git a/src/hotspot/share/code/codeCache.inline.hpp b/src/hotspot/share/code/codeCache.inline.hpp new file mode 100644 index 00000000000..04e5d034c9f --- /dev/null +++ b/src/hotspot/share/code/codeCache.inline.hpp @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2018, 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_COMPILER_CODECACHE_INLINE_HPP +#define SHARE_VM_COMPILER_CODECACHE_INLINE_HPP + +#include "code/codeCache.hpp" + +#include "code/nativeInst.hpp" + +inline CodeBlob* CodeCache::find_blob_fast(void* pc) { + int slot = 0; + return find_blob_and_oopmap(pc, slot); +} + +inline CodeBlob* CodeCache::find_blob_and_oopmap(void* pc, int& slot) { + NativePostCallNop* nop = nativePostCallNop_at((address) pc); + CodeBlob* cb; + if (nop != NULL && nop->displacement() != 0) { + int offset = (nop->displacement() & 0xffffff); + cb = (CodeBlob*) ((address) pc - offset); + slot = ((nop->displacement() >> 24) & 0xff); + } else { + cb = CodeCache::find_blob(pc); + slot = -1; + } + assert(cb != NULL, "must be"); + return cb; +} + +inline int CodeCache::find_oopmap_slot_fast(void* pc) { + NativePostCallNop* nop = nativePostCallNop_at((address) pc); + return (nop != NULL && nop->displacement() != 0) + ? ((nop->displacement() >> 24) & 0xff) + : -1; +} + +#endif // SHARE_VM_COMPILER_CODECACHE_INLINE_HPP diff --git a/src/hotspot/share/code/compiledIC.hpp b/src/hotspot/share/code/compiledIC.hpp index 38ec5c5aa63..6cdaf20c22d 100644 --- a/src/hotspot/share/code/compiledIC.hpp +++ b/src/hotspot/share/code/compiledIC.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -245,7 +245,7 @@ class CompiledIC: public ResourceObj { bool is_icholder_call() const; - address end_of_call() { return _call->return_address(); } + address end_of_call() const { return _call->return_address(); } // MT-safe patching of inline caches. Note: Only safe to call is_xxx when holding the CompiledIC_ock // so you are guaranteed that no patching takes place. The same goes for verify. @@ -364,6 +364,7 @@ public: virtual bool is_call_to_interpreted() const = 0; virtual address instruction_address() const = 0; + virtual address end_of_call() const = 0; protected: virtual address resolve_call_stub() const = 0; virtual void set_destination_mt_safe(address dest) = 0; @@ -409,6 +410,7 @@ private: // Delegation address destination() const { return _call->destination(); } + address end_of_call() const { return _call->return_address(); } // State virtual bool is_call_to_interpreted() const; diff --git a/src/hotspot/share/code/compiledMethod.cpp b/src/hotspot/share/code/compiledMethod.cpp index 695906d1cba..83c33408ea3 100644 --- a/src/hotspot/share/code/compiledMethod.cpp +++ b/src/hotspot/share/code/compiledMethod.cpp @@ -40,17 +40,20 @@ #include "oops/klass.inline.hpp" #include "oops/methodData.hpp" #include "oops/method.inline.hpp" +#include "oops/weakHandle.inline.hpp" #include "prims/methodHandles.hpp" #include "runtime/atomic.hpp" #include "runtime/deoptimization.hpp" +#include "runtime/frame.inline.hpp" +#include "runtime/jniHandles.inline.hpp" #include "runtime/handles.inline.hpp" #include "runtime/mutexLocker.hpp" #include "runtime/sharedRuntime.hpp" CompiledMethod::CompiledMethod(Method* method, const char* name, CompilerType type, const CodeBlobLayout& layout, int frame_complete_offset, int frame_size, ImmutableOopMapSet* oop_maps, - bool caller_must_gc_arguments) - : CodeBlob(name, type, layout, frame_complete_offset, frame_size, oop_maps, caller_must_gc_arguments), + bool caller_must_gc_arguments, bool compiled) + : CodeBlob(name, type, layout, frame_complete_offset, frame_size, oop_maps, caller_must_gc_arguments, compiled), _mark_for_deoptimization_status(not_marked), _method(method), _gc_data(NULL) @@ -60,9 +63,9 @@ CompiledMethod::CompiledMethod(Method* method, const char* name, CompilerType ty CompiledMethod::CompiledMethod(Method* method, const char* name, CompilerType type, int size, int header_size, CodeBuffer* cb, int frame_complete_offset, int frame_size, - OopMapSet* oop_maps, bool caller_must_gc_arguments) + OopMapSet* oop_maps, bool caller_must_gc_arguments, bool compiled) : CodeBlob(name, type, CodeBlobLayout((address) this, size, header_size, cb), cb, - frame_complete_offset, frame_size, oop_maps, caller_must_gc_arguments), + frame_complete_offset, frame_size, oop_maps, caller_must_gc_arguments, compiled), _mark_for_deoptimization_status(not_marked), _method(method), _gc_data(NULL) @@ -80,6 +83,7 @@ void CompiledMethod::init_defaults() { _has_unsafe_access = 0; _has_method_handle_invokes = 0; _has_wide_vectors = 0; + _has_monitors = 0; } bool CompiledMethod::is_method_handle_return(address return_pc) { @@ -114,9 +118,12 @@ const char* CompiledMethod::state() const { //----------------------------------------------------------------------------- void CompiledMethod::mark_for_deoptimization(bool inc_recompile_counts) { + // assert(can_be_deoptimized(), ""); // in some places we check before marking, in others not. MutexLocker ml(CompiledMethod_lock->owned_by_self() ? NULL : CompiledMethod_lock, Mutex::_no_safepoint_check_flag); - _mark_for_deoptimization_status = (inc_recompile_counts ? deoptimize : deoptimize_noupdate); + if (_mark_for_deoptimization_status != deoptimize_done) { // can't go backwards + _mark_for_deoptimization_status = (inc_recompile_counts ? deoptimize : deoptimize_noupdate); + } } //----------------------------------------------------------------------------- @@ -319,7 +326,7 @@ address CompiledMethod::oops_reloc_begin() const { // It is not safe to read oops concurrently using entry barriers, if their // location depend on whether the nmethod is entrant or not. - assert(BarrierSet::barrier_set()->barrier_set_nmethod() == NULL, "Not safe oop scan"); + // assert(BarrierSet::barrier_set()->barrier_set_nmethod() == NULL, "Not safe oop scan"); address low_boundary = verified_entry_point(); if (!is_in_use() && is_nmethod()) { @@ -357,14 +364,20 @@ int CompiledMethod::verify_icholder_relocations() { // Method that knows how to preserve outgoing arguments at call. This method must be // called with a frame corresponding to a Java invoke void CompiledMethod::preserve_callee_argument_oops(frame fr, const RegisterMap *reg_map, OopClosure* f) { - if (method() != NULL && !method()->is_native()) { + if (method() == NULL) { + return; + } + + // handle the case of an anchor explicitly set in continuation code that doesn't have a callee + JavaThread* thread = reg_map->thread(); + if (thread->has_last_Java_frame() && fr.sp() == thread->last_Java_sp()) { + return; + } + + if (!method()->is_native()) { address pc = fr.pc(); - SimpleScopeDesc ssd(this, pc); - if (ssd.is_optimized_linkToNative()) return; // call was replaced - Bytecode_invoke call(methodHandle(Thread::current(), ssd.method()), ssd.bci()); - bool has_receiver = call.has_receiver(); - bool has_appendix = call.has_appendix(); - Symbol* signature = call.signature(); + bool has_receiver, has_appendix; + Symbol* signature; // The method attached by JIT-compilers should be used, if present. // Bytecode can be inaccurate in such case. @@ -372,10 +385,25 @@ void CompiledMethod::preserve_callee_argument_oops(frame fr, const RegisterMap * if (callee != NULL) { has_receiver = !(callee->access_flags().is_static()); has_appendix = false; - signature = callee->signature(); + signature = callee->signature(); + } else { + SimpleScopeDesc ssd(this, pc); + if (ssd.is_optimized_linkToNative()) { + // call was replaced + return; + } + + Bytecode_invoke call(methodHandle(Thread::current(), ssd.method()), ssd.bci()); + has_receiver = call.has_receiver(); + has_appendix = call.has_appendix(); + signature = call.signature(); } fr.oops_compiled_arguments_do(signature, has_receiver, has_appendix, reg_map, f); + } else if (method()->is_continuation_enter_intrinsic()) { + // This method only calls Continuation.enter() + Symbol* signature = vmSymbols::continuationEnter_signature(); + fr.oops_compiled_arguments_do(signature, false, false, reg_map, f); } } @@ -587,7 +615,7 @@ void CompiledMethod::run_nmethod_entry_barrier() { // By calling this nmethod entry barrier, it plays along and acts // like any other nmethod found on the stack of a thread (fewer surprises). nmethod* nm = as_nmethod_or_null(); - if (nm != NULL) { + if (nm != NULL && bs_nm->is_armed(nm)) { bool alive = bs_nm->nmethod_entry_barrier(nm); assert(alive, "should be alive"); } @@ -604,10 +632,22 @@ void CompiledMethod::cleanup_inline_caches(bool clean_all) { } // Call this nmethod entry barrier from the sweeper. run_nmethod_entry_barrier(); + if (!clean_all) { + MutexLocker ml(CodeCache_lock, Mutex::_no_safepoint_check_flag); + CodeCache::Sweep::end(); + } InlineCacheBuffer::refill_ic_stubs(); + if (!clean_all) { + MutexLocker ml(CodeCache_lock, Mutex::_no_safepoint_check_flag); + CodeCache::Sweep::begin(); + } } } +address* CompiledMethod::orig_pc_addr(const frame* fr) { + return (address*) ((address)fr->unextended_sp() + orig_pc_offset()); +} + // Called to clean up after class unloading for live nmethods and from the sweeper // for all methods. bool CompiledMethod::cleanup_inline_caches_impl(bool unloading_occurred, bool clean_all) { diff --git a/src/hotspot/share/code/compiledMethod.hpp b/src/hotspot/share/code/compiledMethod.hpp index 6c979f67a1d..f4f40c5a27e 100644 --- a/src/hotspot/share/code/compiledMethod.hpp +++ b/src/hotspot/share/code/compiledMethod.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2015, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -144,10 +144,11 @@ class CompiledMethod : public CodeBlob { void init_defaults(); protected: - enum MarkForDeoptimizationStatus { + enum MarkForDeoptimizationStatus : u1 { not_marked, deoptimize, - deoptimize_noupdate + deoptimize_noupdate, + deoptimize_done }; MarkForDeoptimizationStatus _mark_for_deoptimization_status; // Used for stack deoptimization @@ -156,6 +157,7 @@ protected: unsigned int _has_unsafe_access:1; // May fault due to unsafe access. unsigned int _has_method_handle_invokes:1; // Has this method MethodHandle invokes? unsigned int _has_wide_vectors:1; // Preserve wide vectors at safepoints + unsigned int _has_monitors:1; // Fastpath monitor detection for continuations Method* _method; address _scopes_data_begin; @@ -172,16 +174,15 @@ protected: void* _gc_data; virtual void flush() = 0; + protected: - CompiledMethod(Method* method, const char* name, CompilerType type, const CodeBlobLayout& layout, int frame_complete_offset, int frame_size, ImmutableOopMapSet* oop_maps, bool caller_must_gc_arguments); - CompiledMethod(Method* method, const char* name, CompilerType type, int size, int header_size, CodeBuffer* cb, int frame_complete_offset, int frame_size, OopMapSet* oop_maps, bool caller_must_gc_arguments); + CompiledMethod(Method* method, const char* name, CompilerType type, const CodeBlobLayout& layout, int frame_complete_offset, int frame_size, ImmutableOopMapSet* oop_maps, bool caller_must_gc_arguments, bool compiled); + CompiledMethod(Method* method, const char* name, CompilerType type, int size, int header_size, CodeBuffer* cb, int frame_complete_offset, int frame_size, OopMapSet* oop_maps, bool caller_must_gc_arguments, bool compiled); public: // Only used by unit test. CompiledMethod() {} - virtual bool is_compiled() const { return true; } - template T* gc_data() const { return reinterpret_cast(_gc_data); } template @@ -190,6 +191,9 @@ public: bool has_unsafe_access() const { return _has_unsafe_access; } void set_has_unsafe_access(bool z) { _has_unsafe_access = z; } + bool has_monitors() const { return _has_monitors; } + void set_has_monitors(bool z) { _has_monitors = z; } + bool has_method_handle_invokes() const { return _has_method_handle_invokes; } void set_has_method_handle_invokes(bool z) { _has_method_handle_invokes = z; } @@ -241,11 +245,17 @@ public: bool is_marked_for_deoptimization() const { return _mark_for_deoptimization_status != not_marked; } void mark_for_deoptimization(bool inc_recompile_counts = true); + bool has_been_deoptimized() const { return _mark_for_deoptimization_status == deoptimize_done; } + void mark_deoptimized() { _mark_for_deoptimization_status = deoptimize_done; } + + virtual void make_deoptimized() { assert(false, "not supported"); }; + bool update_recompile_counts() const { // Update recompile counts when either the update is explicitly requested (deoptimize) // or the nmethod is not marked for deoptimization at all (not_marked). // The latter happens during uncommon traps when deoptimized nmethod is made not entrant. - return _mark_for_deoptimization_status != deoptimize_noupdate; + return _mark_for_deoptimization_status != deoptimize_noupdate && + _mark_for_deoptimization_status != deoptimize_done; } // tells whether frames described by this nmethod can be deoptimized @@ -296,7 +306,6 @@ public: virtual oop* oop_addr_at(int index) const = 0; virtual Metadata** metadata_addr_at(int index) const = 0; - virtual void set_original_pc(const frame* fr, address pc) = 0; protected: // Exception cache support @@ -318,13 +327,23 @@ public: address deopt_mh_handler_begin() const { return _deopt_mh_handler_begin; } address deopt_handler_begin() const { return _deopt_handler_begin; } - virtual address get_original_pc(const frame* fr) = 0; + address* deopt_handler_begin_addr() { return &_deopt_handler_begin; } // Deopt // Return true is the PC is one would expect if the frame is being deopted. inline bool is_deopt_pc(address pc); inline bool is_deopt_mh_entry(address pc); inline bool is_deopt_entry(address pc); + // Accessor/mutator for the original pc of a frame before a frame was deopted. + address get_original_pc(const frame* fr) { return *orig_pc_addr(fr); } + void set_original_pc(const frame* fr, address pc) { *orig_pc_addr(fr) = pc; } + + virtual int orig_pc_offset() = 0; + +private: + address* orig_pc_addr(const frame* fr); + +public: virtual bool can_convert_to_zombie() = 0; virtual const char* compile_kind() const = 0; virtual int get_state() const = 0; diff --git a/src/hotspot/share/code/nmethod.cpp b/src/hotspot/share/code/nmethod.cpp index 9a4c133e6dd..6c78d29a818 100644 --- a/src/hotspot/share/code/nmethod.cpp +++ b/src/hotspot/share/code/nmethod.cpp @@ -38,7 +38,9 @@ #include "compiler/compilerDirectives.hpp" #include "compiler/directivesParser.hpp" #include "compiler/disassembler.hpp" -#include "compiler/oopMap.hpp" +#include "compiler/oopMap.inline.hpp" +#include "gc/shared/barrierSet.hpp" +#include "gc/shared/barrierSetNMethod.hpp" #include "gc/shared/collectedHeap.hpp" #include "interpreter/bytecode.hpp" #include "logging/log.hpp" @@ -51,9 +53,11 @@ #include "oops/method.inline.hpp" #include "oops/methodData.hpp" #include "oops/oop.inline.hpp" +#include "oops/weakHandle.inline.hpp" #include "prims/jvmtiImpl.hpp" #include "prims/jvmtiThreadState.hpp" #include "prims/methodHandles.hpp" +#include "runtime/continuation.hpp" #include "runtime/atomic.hpp" #include "runtime/deoptimization.hpp" #include "runtime/flags/flagSetting.hpp" @@ -422,13 +426,14 @@ int nmethod::total_size() const { nul_chk_table_size(); } -address* nmethod::orig_pc_addr(const frame* fr) { - return (address*) ((address)fr->unextended_sp() + _orig_pc_offset); -} - const char* nmethod::compile_kind() const { if (is_osr_method()) return "osr"; - if (method() != NULL && is_native_method()) return "c2n"; + if (method() != NULL && is_native_method()) { + if (method()->is_continuation_enter_intrinsic()) { + return "cnt"; + } + return "c2n"; + } return NULL; } @@ -460,7 +465,8 @@ nmethod* nmethod::new_native_nmethod(const methodHandle& method, int frame_size, ByteSize basic_lock_owner_sp_offset, ByteSize basic_lock_sp_offset, - OopMapSet* oop_maps) { + OopMapSet* oop_maps, + int exception_handler) { code_buffer->finalize_oop_references(method); // create nmethod nmethod* nm = NULL; @@ -471,6 +477,9 @@ nmethod* nmethod::new_native_nmethod(const methodHandle& method, CodeOffsets offsets; offsets.set_value(CodeOffsets::Verified_Entry, vep_offset); offsets.set_value(CodeOffsets::Frame_Complete, frame_complete); + if (exception_handler != -1) { + offsets.set_value(CodeOffsets::Exceptions, exception_handler); + } nm = new (native_nmethod_size, CompLevel_none) nmethod(method(), compiler_none, native_nmethod_size, compile_id, &offsets, @@ -603,7 +612,7 @@ nmethod::nmethod( ByteSize basic_lock_owner_sp_offset, ByteSize basic_lock_sp_offset, OopMapSet* oop_maps ) - : CompiledMethod(method, "native nmethod", type, nmethod_size, sizeof(nmethod), code_buffer, offsets->value(CodeOffsets::Frame_Complete), frame_size, oop_maps, false), + : CompiledMethod(method, "native nmethod", type, nmethod_size, sizeof(nmethod), code_buffer, offsets->value(CodeOffsets::Frame_Complete), frame_size, oop_maps, false, true), _is_unloading_state(0), _native_receiver_sp_offset(basic_lock_owner_sp_offset), _native_basic_lock_sp_offset(basic_lock_sp_offset) @@ -622,12 +631,13 @@ nmethod::nmethod( // values something that will never match a pc like the nmethod vtable entry _exception_offset = 0; _orig_pc_offset = 0; + _gc_epoch = Continuations::gc_epoch(); _consts_offset = data_offset(); - _stub_offset = data_offset(); + _stub_offset = content_offset() + code_buffer->total_offset_of(code_buffer->stubs()); _oops_offset = data_offset(); - _metadata_offset = _oops_offset + align_up(code_buffer->total_oop_size(), oopSize); - scopes_data_offset = _metadata_offset + align_up(code_buffer->total_metadata_size(), wordSize); + _metadata_offset = _oops_offset + align_up(code_buffer->total_oop_size(), oopSize); + scopes_data_offset = _metadata_offset + align_up(code_buffer->total_metadata_size(), wordSize); _scopes_pcs_offset = scopes_data_offset; _dependencies_offset = _scopes_pcs_offset; _native_invokers_offset = _dependencies_offset; @@ -649,6 +659,8 @@ nmethod::nmethod( _pc_desc_container.reset_to(NULL); _hotness_counter = NMethodSweeper::hotness_counter_reset_val(); + _exception_offset = code_offset() + offsets->value(CodeOffsets::Exceptions); + _scopes_data_begin = (address) this + scopes_data_offset; _deopt_handler_begin = (address) this + deoptimize_offset; _deopt_mh_handler_begin = (address) this + deoptimize_mh_offset; @@ -662,6 +674,8 @@ nmethod::nmethod( debug_only(Universe::heap()->verify_nmethod(this)); CodeCache::commit(this); + + finalize_relocations(); } if (PrintNativeNMethods || PrintDebugInfo || PrintRelocations || PrintDependencies) { @@ -735,7 +749,7 @@ nmethod::nmethod( int jvmci_data_size #endif ) - : CompiledMethod(method, "nmethod", type, nmethod_size, sizeof(nmethod), code_buffer, offsets->value(CodeOffsets::Frame_Complete), frame_size, oop_maps, false), + : CompiledMethod(method, "nmethod", type, nmethod_size, sizeof(nmethod), code_buffer, offsets->value(CodeOffsets::Frame_Complete), frame_size, oop_maps, false, true), _is_unloading_state(0), _native_receiver_sp_offset(in_ByteSize(-1)), _native_basic_lock_sp_offset(in_ByteSize(-1)) @@ -754,6 +768,7 @@ nmethod::nmethod( _comp_level = comp_level; _orig_pc_offset = orig_pc_offset; _hotness_counter = NMethodSweeper::hotness_counter_reset_val(); + _gc_epoch = Continuations::gc_epoch(); // Section offsets _consts_offset = content_offset() + code_buffer->total_offset_of(code_buffer->consts()); @@ -839,6 +854,8 @@ nmethod::nmethod( CodeCache::commit(this); + finalize_relocations(); + // Copy contents of ExceptionHandlerTable to nmethod handler_table->copy_to(this); nul_chk_table->copy_to(this); @@ -1085,6 +1102,84 @@ void nmethod::fix_oop_relocations(address begin, address end, bool initialize_im } } +static void install_post_call_nop_displacement(nmethod* nm, address pc) { + NativePostCallNop* nop = nativePostCallNop_at((address) pc); + intptr_t cbaddr = (intptr_t) nm; + intptr_t offset = ((intptr_t) pc) - cbaddr; + + int oopmap_slot = nm->oop_maps()->find_slot_for_offset((intptr_t) pc - (intptr_t) nm->code_begin()); + if (oopmap_slot < 0) { // this can happen at asynchronous (non-safepoint) stackwalks + log_debug(codecache)("failed to find oopmap for cb: " INTPTR_FORMAT " offset: %d", cbaddr, (int) offset); + } else if (((oopmap_slot & 0xff) == oopmap_slot) && ((offset & 0xffffff) == offset)) { + jint value = (oopmap_slot << 24) | (jint) offset; + nop->patch(value); + } else { + log_debug(codecache)("failed to encode %d %d", oopmap_slot, (int) offset); + } +} + +void nmethod::finalize_relocations() { + NoSafepointVerifier nsv; + + // Make sure that post call nops fill in nmethod offsets eagerly so + // we don't have to race with deoptimization + RelocIterator iter(this); + while (iter.next()) { + if (iter.type() == relocInfo::post_call_nop_type) { + post_call_nop_Relocation* const reloc = iter.post_call_nop_reloc(); + address pc = reloc->addr(); + install_post_call_nop_displacement(this, pc); + } + } +} + +void nmethod::make_deoptimized() { + if (!Continuations::enabled()) { + return; + } + + assert(method() == NULL || can_be_deoptimized(), ""); + assert(!is_zombie(), ""); + + CompiledICLocker ml(this); + assert(CompiledICLocker::is_safe(this), "mt unsafe call"); + ResourceMark rm; + RelocIterator iter(this, oops_reloc_begin()); + + while (iter.next()) { + + switch (iter.type()) { + case relocInfo::virtual_call_type: + case relocInfo::opt_virtual_call_type: { + CompiledIC *ic = CompiledIC_at(&iter); + address pc = ic->end_of_call(); + NativePostCallNop* nop = nativePostCallNop_at(pc); + if (nop != NULL) { + nop->make_deopt(); + } + assert(NativeDeoptInstruction::is_deopt_at(pc), "check"); + break; + } + case relocInfo::static_call_type: { + CompiledStaticCall *csc = compiledStaticCall_at(iter.reloc()); + address pc = csc->end_of_call(); + NativePostCallNop* nop = nativePostCallNop_at(pc); + //tty->print_cr(" - static pc %p", pc); + if (nop != NULL) { + nop->make_deopt(); + } + // We can't assert here, there are some calls to stubs / runtime + // that have reloc data and doesn't have a post call NOP. + //assert(NativeDeoptInstruction::is_deopt_at(pc), "check"); + break; + } + default: + break; + } + } + // Don't deopt this again. + mark_deoptimized(); +} void nmethod::verify_clean_inline_caches() { assert(CompiledICLocker::is_safe(this), "mt unsafe call"); @@ -1135,6 +1230,21 @@ void nmethod::mark_as_seen_on_stack() { set_stack_traversal_mark(NMethodSweeper::traversal_count()); } +void nmethod::mark_as_maybe_on_continuation() { + assert(is_alive(), "Must be an alive method"); + _gc_epoch = Continuations::gc_epoch(); +} + +bool nmethod::is_maybe_on_continuation_stack() { + if (!Continuations::enabled()) { + return false; + } + + // If the condition below is true, it means that the nmethod was found to + // be alive the previous completed marking cycle. + return _gc_epoch >= Continuations::previous_completed_gc_marking_cycle(); +} + // Tell if a non-entrant method can be converted to a zombie (i.e., // there are no activations on the stack, not in use by the VM, // and not in use by the ServiceThread) @@ -1153,7 +1263,7 @@ bool nmethod::can_convert_to_zombie() { // If an is_unloading() nmethod is still not_entrant, then it is not safe to // convert it to zombie due to GC unloading interactions. However, if it // has become unloaded, then it is okay to convert such nmethods to zombie. - return stack_traversal_mark() + 1 < NMethodSweeper::traversal_count() && + return stack_traversal_mark()+1 < NMethodSweeper::traversal_count() && !is_maybe_on_continuation_stack() && !is_locked_by_vm() && (!is_unloading() || is_unloaded()); } @@ -1846,12 +1956,16 @@ void nmethod::do_unloading(bool unloading_occurred) { } else { guarantee(unload_nmethod_caches(unloading_occurred), "Should not need transition stubs"); + BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod(); + if (bs_nm != NULL) { + bs_nm->disarm(this); + } } } void nmethod::oops_do(OopClosure* f, bool allow_dead) { // make sure the oops ready to receive visitors - assert(allow_dead || is_alive(), "should not call follow on dead nmethod"); + assert(allow_dead || is_alive(), "should not call follow on dead nmethod: %d", _state); // Prevent extra code cache walk for platforms that don't have immediate oops. if (relocInfo::mustIterateImmediateOopsInCode()) { @@ -1880,6 +1994,20 @@ void nmethod::oops_do(OopClosure* f, bool allow_dead) { } } +void nmethod::follow_nmethod(OopIterateClosure* cl) { + // Process oops in the nmethod + oops_do(cl); + + // CodeCache sweeper support + mark_as_maybe_on_continuation(); + + BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod(); + bs_nm->disarm(this); + + // There's an assumption made that this function is not used by GCs that + // relocate objects, and therefore we don't call fix_oop_relocations. +} + nmethod* volatile nmethod::_oops_do_mark_nmethods; void nmethod::oops_do_log_change(const char* state) { diff --git a/src/hotspot/share/code/nmethod.hpp b/src/hotspot/share/code/nmethod.hpp index 866a0a37aeb..b8d9ea85d80 100644 --- a/src/hotspot/share/code/nmethod.hpp +++ b/src/hotspot/share/code/nmethod.hpp @@ -31,6 +31,7 @@ class DepChange; class DirectiveSet; class DebugInformationRecorder; class JvmtiThreadState; +class OopIterateClosure; // nmethods (native methods) are the compiled code versions of Java methods. // @@ -73,6 +74,8 @@ class nmethod : public CompiledMethod { // Shared fields for all nmethod's int _entry_bci; // != InvocationEntryBci if this nmethod is an on-stack replacement method + uint64_t _gc_epoch; + // To support simple linked-list chaining of nmethods: nmethod* _osr_link; // from InstanceKlass::osr_nmethods_head @@ -385,7 +388,8 @@ class nmethod : public CompiledMethod { int frame_size, ByteSize receiver_sp_offset, ByteSize basic_lock_sp_offset, - OopMapSet* oop_maps); + OopMapSet* oop_maps, + int exception_handler = -1); // type info bool is_nmethod() const { return true; } @@ -566,6 +570,8 @@ public: // See comment at definition of _last_seen_on_stack void mark_as_seen_on_stack(); + void mark_as_maybe_on_continuation(); + bool is_maybe_on_continuation_stack(); bool can_convert_to_zombie(); // Evolution support. We make old (discarded) compiled methods point to new Method*s. @@ -595,6 +601,9 @@ public: // nmethod. bool oops_do_try_claim(); + // Loom support for following nmethods on the stack + void follow_nmethod(OopIterateClosure* cl); + // Class containing callbacks for the oops_do_process_weak/strong() methods // below. class OopsDoProcessor { @@ -632,9 +641,7 @@ public: void copy_scopes_pcs(PcDesc* pcs, int count); void copy_scopes_data(address buffer, int size); - // Accessor/mutator for the original pc of a frame before a frame was deopted. - address get_original_pc(const frame* fr) { return *orig_pc_addr(fr); } - void set_original_pc(const frame* fr, address pc) { *orig_pc_addr(fr) = pc; } + int orig_pc_offset() { return _orig_pc_offset; } // jvmti support: void post_compiled_method_load_event(JvmtiThreadState* state = NULL); @@ -754,6 +761,9 @@ public: virtual CompiledStaticCall* compiledStaticCall_at(Relocation* call_site) const; virtual CompiledStaticCall* compiledStaticCall_at(address addr) const; virtual CompiledStaticCall* compiledStaticCall_before(address addr) const; + + virtual void make_deoptimized(); + void finalize_relocations(); }; // Locks an nmethod so its code will not get removed and it will not diff --git a/src/hotspot/share/code/relocInfo.hpp b/src/hotspot/share/code/relocInfo.hpp index 96f96daa99d..db5b55db5bc 100644 --- a/src/hotspot/share/code/relocInfo.hpp +++ b/src/hotspot/share/code/relocInfo.hpp @@ -268,7 +268,8 @@ class relocInfo { trampoline_stub_type = 13, // stub-entry for trampoline runtime_call_w_cp_type = 14, // Runtime call which may load its target from the constant pool data_prefix_tag = 15, // tag for a prefix (carries data arguments) - type_mask = 15 // A mask which selects only the above values + post_call_nop_type = 16, // A tag for post call nop relocations + type_mask = 31 // A mask which selects only the above values }; private: @@ -307,12 +308,13 @@ class relocInfo { visitor(poll_return) \ visitor(section_word) \ visitor(trampoline_stub) \ + visitor(post_call_nop) \ public: enum { value_width = sizeof(unsigned short) * BitsPerByte, - type_width = 4, // == log2(type_mask+1) + type_width = 5, // == log2(type_mask+1) nontype_width = value_width - type_width, datalen_width = nontype_width-1, datalen_tag = 1 << datalen_width, // or-ed into _value @@ -868,6 +870,19 @@ class DataRelocation : public Relocation { // added into the low-half immediate constant, and must not overflow it. }; +class post_call_nop_Relocation : public Relocation { + friend class RelocIterator; + +public: + post_call_nop_Relocation() : Relocation(relocInfo::post_call_nop_type) { } + + static RelocationHolder spec() { + RelocationHolder rh = newHolder(); + new(rh) post_call_nop_Relocation(); + return rh; + } +}; + // A CallRelocation always points at a call instruction. // It is PC-relative on most machines. class CallRelocation : public Relocation { diff --git a/src/hotspot/share/compiler/compilerOracle.hpp b/src/hotspot/share/compiler/compilerOracle.hpp index 303c52683f5..52ebce474df 100644 --- a/src/hotspot/share/compiler/compilerOracle.hpp +++ b/src/hotspot/share/compiler/compilerOracle.hpp @@ -135,6 +135,9 @@ class CompilerOracle : AllStatic { // Tells whether we want to disallow inlining of this method static bool should_not_inline(const methodHandle& method); + // Tells whether this method changes Thread.currentThread() + static bool changes_current_thread(const methodHandle& method); + // Tells whether we should print the assembly for this method static bool should_print(const methodHandle& method); diff --git a/src/hotspot/share/compiler/oopMap.cpp b/src/hotspot/share/compiler/oopMap.cpp index 2345bb4de32..e9cc5fe0f97 100644 --- a/src/hotspot/share/compiler/oopMap.cpp +++ b/src/hotspot/share/compiler/oopMap.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1998, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1998, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -27,13 +27,15 @@ #include "code/codeCache.hpp" #include "code/nmethod.hpp" #include "code/scopeDesc.hpp" -#include "compiler/oopMap.hpp" +#include "compiler/oopMap.inline.hpp" #include "gc/shared/collectedHeap.hpp" +#include "logging/log.hpp" +#include "logging/logStream.hpp" #include "memory/allocation.inline.hpp" #include "memory/iterator.hpp" #include "memory/resourceArea.hpp" -#include "memory/universe.hpp" #include "oops/compressedOops.hpp" +#include "runtime/atomic.hpp" #include "runtime/frame.inline.hpp" #include "runtime/handles.inline.hpp" #include "runtime/signature.hpp" @@ -70,15 +72,15 @@ static inline derived_pointer operator+(derived_pointer p, intptr_t offset) { // OopMapStream -OopMapStream::OopMapStream(OopMap* oop_map) { - _stream = new CompressedReadStream(oop_map->write_stream()->buffer()); +OopMapStream::OopMapStream(const OopMap* oop_map) + : _stream(oop_map->write_stream()->buffer()) { _size = oop_map->omv_count(); _position = 0; _valid_omv = false; } -OopMapStream::OopMapStream(const ImmutableOopMap* oop_map) { - _stream = new CompressedReadStream(oop_map->data_addr()); +OopMapStream::OopMapStream(const ImmutableOopMap* oop_map) + : _stream(oop_map->data_addr()) { _size = oop_map->count(); _position = 0; _valid_omv = false; @@ -86,7 +88,7 @@ OopMapStream::OopMapStream(const ImmutableOopMap* oop_map) { void OopMapStream::find_next() { if (_position++ < _size) { - _omv.read_from(_stream); + _omv.read_from(&_stream); _valid_omv = true; return; } @@ -102,6 +104,9 @@ OopMap::OopMap(int frame_size, int arg_count) { // OopMaps are usually quite so small, so pick a small initial size set_write_stream(new CompressedWriteStream(32)); set_omv_count(0); + _num_oops = 0; + _has_derived_oops = false; + _index = -1; #ifdef ASSERT _locs_length = VMRegImpl::stack2reg(0)->value() + frame_size + arg_count; @@ -117,6 +122,9 @@ OopMap::OopMap(OopMap::DeepCopyToken, OopMap* source) { set_write_stream(new CompressedWriteStream(source->omv_count() * 2)); set_omv_count(0); set_offset(source->offset()); + _num_oops = source->num_oops(); + _has_derived_oops = source->has_derived_oops(); + _index = -1; #ifdef ASSERT _locs_length = source->_locs_length; @@ -141,6 +149,166 @@ void OopMap::copy_data_to(address addr) const { memcpy(addr, write_stream()->buffer(), write_stream()->position()); } +class OopMapSort { +private: + const OopMap* _map; + OopMapValue* _values; + int _count; + +public: + OopMapSort(const OopMap* map) : _map(map), _count(0) { + _values = NEW_RESOURCE_ARRAY(OopMapValue, _map->omv_count()); + } + + void sort(); + + void print(); + + void write(CompressedWriteStream* stream) { + for (int i = 0; i < _count; ++i) { + _values[i].write_on(stream); + } + } + +private: + int find_derived_position(OopMapValue omv, int start) { + assert(omv.type() == OopMapValue::derived_oop_value, ""); + + VMReg base = omv.content_reg(); + int i = start; + + for (; i < _count; ++i) { + if (base == _values[i].reg()) { + + for (int n = i + 1; n < _count; ++n) { + if (_values[i].type() != OopMapValue::derived_oop_value || _values[i].content_reg() != base) { + return n; + } + + if (derived_cost(_values[i]) > derived_cost(omv)) { + return n; + } + } + return _count; + } + } + + assert(false, "failed to find base"); + return -1; + } + + int find_position(OopMapValue omv, int start) { + assert(omv.type() != OopMapValue::derived_oop_value, ""); + + int i = start; + for (; i < _count; ++i) { + if (omv_cost(_values[i]) > omv_cost(omv)) { + return i; + } + } + assert(i < _map->omv_count(), "bounds check"); + return i; + } + + void insert(OopMapValue value, int pos) { + assert(pos >= 0 && pos < _map->omv_count(), "bounds check"); + assert(pos <= _count, "sanity"); + + if (pos < _count) { + OopMapValue prev = _values[pos]; + + for (int i = pos; i < _count; ++i) { + OopMapValue tmp = _values[i+1]; + _values[i+1] = prev; + prev = tmp; + } + } + _values[pos] = value; + + ++_count; + } + + int omv_cost(OopMapValue omv) { + assert(omv.type() == OopMapValue::oop_value || omv.type() == OopMapValue::narrowoop_value, ""); + return reg_cost(omv.reg()); + } + + int reg_cost(VMReg reg) { + if (reg->is_reg()) { + return 0; + } + return reg->reg2stack() * VMRegImpl::stack_slot_size; + } + + int derived_cost(OopMapValue omv) { + return reg_cost(omv.reg()); + } +}; + +void OopMapSort::sort() { + for (OopMapStream oms(_map); !oms.is_done(); oms.next()) { + OopMapValue omv = oms.current(); + assert(omv.type() == OopMapValue::oop_value || omv.type() == OopMapValue::narrowoop_value || omv.type() == OopMapValue::derived_oop_value || omv.type() == OopMapValue::callee_saved_value, ""); + } + + for (OopMapStream oms(_map); !oms.is_done(); oms.next()) { + if (oms.current().type() == OopMapValue::callee_saved_value) { + insert(oms.current(), _count); + } + } + + int start = _count; + for (OopMapStream oms(_map); !oms.is_done(); oms.next()) { + OopMapValue omv = oms.current(); + if (omv.type() == OopMapValue::oop_value || omv.type() == OopMapValue::narrowoop_value) { + int pos = find_position(omv, start); + insert(omv, pos); + } + } + + for (OopMapStream oms(_map); !oms.is_done(); oms.next()) { + OopMapValue omv = oms.current(); + if (omv.type() == OopMapValue::derived_oop_value) { + int pos = find_derived_position(omv, start); + assert(pos > 0, ""); + insert(omv, pos); + } + } +} + +void OopMapSort::print() { + for (int i = 0; i < _count; ++i) { + OopMapValue omv = _values[i]; + if (omv.type() == OopMapValue::oop_value || omv.type() == OopMapValue::narrowoop_value) { + if (omv.reg()->is_reg()) { + tty->print_cr("[%c][%d] -> reg (" INTPTR_FORMAT ")", omv.type() == OopMapValue::narrowoop_value ? 'n' : 'o', i, omv.reg()->value()); + } else { + tty->print_cr("[%c][%d] -> stack (" INTPTR_FORMAT ")", omv.type() == OopMapValue::narrowoop_value ? 'n' : 'o', i, omv.reg()->reg2stack() * VMRegImpl::stack_slot_size); + } + } else { + if (omv.content_reg()->is_reg()) { + tty->print_cr("[d][%d] -> reg (" INTPTR_FORMAT ") stack (" INTPTR_FORMAT ")", i, omv.content_reg()->value(), omv.reg()->reg2stack() * VMRegImpl::stack_slot_size); + } else if (omv.reg()->is_reg()) { + tty->print_cr("[d][%d] -> stack (" INTPTR_FORMAT ") reg (" INTPTR_FORMAT ")", i, omv.content_reg()->reg2stack() * VMRegImpl::stack_slot_size, omv.reg()->value()); + } else { + int derived_offset = omv.reg()->reg2stack() * VMRegImpl::stack_slot_size; + int base_offset = omv.content_reg()->reg2stack() * VMRegImpl::stack_slot_size; + tty->print_cr("[d][%d] -> stack (%x) stack (%x)", i, base_offset, derived_offset); + } + } + } +} + +void OopMap::copy_and_sort_data_to(address addr) const { + OopMapSort sort(this); + sort.sort(); + CompressedWriteStream* stream = new CompressedWriteStream(_write_stream->position()); + sort.write(stream); + + assert(stream->position() == write_stream()->position(), ""); + memcpy(addr, stream->buffer(), stream->position()); +} + int OopMap::heap_size() const { int size = sizeof(OopMap); int align = sizeof(void *) - 1; @@ -161,6 +329,11 @@ void OopMap::set_xxx(VMReg reg, OopMapValue::oop_types x, VMReg optional) { OopMapValue o(reg, x, optional); o.write_on(write_stream()); increment_count(); + if (x == OopMapValue::oop_value || x == OopMapValue::narrowoop_value) { + increment_num_oops(); + } else if (x == OopMapValue::derived_oop_value) { + set_has_derived_oops(true); + } } @@ -192,7 +365,7 @@ void OopMap::set_derived_oop(VMReg reg, VMReg derived_from_local_register ) { OopMapSet::OopMapSet() : _list(MinOopMapAllocation) {} -void OopMapSet::add_gc_map(int pc_offset, OopMap *map ) { +int OopMapSet::add_gc_map(int pc_offset, OopMap *map ) { map->set_offset(pc_offset); #ifdef ASSERT @@ -208,32 +381,177 @@ void OopMapSet::add_gc_map(int pc_offset, OopMap *map ) { } #endif // ASSERT - add(map); + int index = add(map); + map->_index = index; + return index; } -static void add_derived_oop(oop* base, derived_pointer* derived, OopClosure* oop_fn) { +class AddDerivedOop : public DerivedOopClosure { + public: + enum { + SkipNull = true, NeedsLock = true + }; + + virtual void do_derived_oop(oop* base, derived_pointer* derived) { #if COMPILER2_OR_JVMCI - DerivedPointerTable::add(derived, base); + DerivedPointerTable::add(derived, base); #endif // COMPILER2_OR_JVMCI + } +}; + +class ProcessDerivedOop : public DerivedOopClosure { + OopClosure* _oop_cl; + +public: + ProcessDerivedOop(OopClosure* oop_cl) : + _oop_cl(oop_cl) {} + + enum { + SkipNull = true, NeedsLock = true + }; + + virtual void do_derived_oop(oop* base, derived_pointer* derived) { + // All derived pointers must be processed before the base pointer of any derived pointer is processed. + // Otherwise, if two derived pointers use the same base, the second derived pointer will get an obscured + // offset, if the base pointer is processed in the first derived pointer. + derived_pointer derived_base = to_derived_pointer(*base); + intptr_t offset = *derived - derived_base; + *derived = derived_base; + _oop_cl->do_oop((oop*)derived); + *derived = *derived + offset; + } +}; + +class IgnoreDerivedOop : public DerivedOopClosure { + OopClosure* _oop_cl; + +public: + enum { + SkipNull = true, NeedsLock = true + }; + + virtual void do_derived_oop(oop* base, derived_pointer* derived) {} +}; + +void OopMapSet::oops_do(const frame* fr, const RegisterMap* reg_map, OopClosure* f, DerivedPointerIterationMode mode) { + find_map(fr)->oops_do(fr, reg_map, f, mode); } -static void ignore_derived_oop(oop* base, derived_pointer* derived, OopClosure* oop_fn) { +void OopMapSet::oops_do(const frame *fr, const RegisterMap* reg_map, OopClosure* f, DerivedOopClosure* df) { + find_map(fr)->oops_do(fr, reg_map, f, df); } -static void process_derived_oop(oop* base, derived_pointer* derived, OopClosure* oop_fn) { - // All derived pointers must be processed before the base pointer of any derived pointer is processed. - // Otherwise, if two derived pointers use the same base, the second derived pointer will get an obscured - // offset, if the base pointer is processed in the first derived pointer. - derived_pointer derived_base = to_derived_pointer(*base); - intptr_t offset = *derived - derived_base; - *derived = derived_base; - oop_fn->do_oop((oop*)derived); - *derived = *derived + offset; +void ImmutableOopMap::oops_do(const frame *fr, const RegisterMap *reg_map, + OopClosure* oop_fn, DerivedOopClosure* derived_oop_fn) const { + assert(derived_oop_fn != NULL, "sanity"); + OopMapDo visitor(oop_fn, derived_oop_fn); + visitor.oops_do(fr, reg_map, this); } +void ImmutableOopMap::oops_do(const frame *fr, const RegisterMap *reg_map, + OopClosure* oop_fn, DerivedPointerIterationMode derived_mode) const { + ProcessDerivedOop process_cl(oop_fn); + AddDerivedOop add_cl; + IgnoreDerivedOop ignore_cl; + DerivedOopClosure* derived_cl; + switch (derived_mode) { + case DerivedPointerIterationMode::_directly: + derived_cl = &process_cl; + break; + case DerivedPointerIterationMode::_with_table: + derived_cl = &add_cl; + break; + case DerivedPointerIterationMode::_ignore: + derived_cl = &ignore_cl; + break; + default: + guarantee (false, "unreachable"); + } + OopMapDo visitor(oop_fn, derived_cl); + visitor.oops_do(fr, reg_map, this); +} + +void ImmutableOopMap::all_type_do(const frame *fr, OopMapClosure* fn) const { + OopMapValue omv; + for (OopMapStream oms(this); !oms.is_done(); oms.next()) { + omv = oms.current(); + if (fn->handle_type(omv.type())) { + fn->do_value(omv.reg(), omv.type()); + } + } +} + +void ImmutableOopMap::all_type_do(const frame *fr, OopMapValue::oop_types type, OopMapClosure* fn) const { + OopMapValue omv; + for (OopMapStream oms(this); !oms.is_done(); oms.next()) { + omv = oms.current(); + if (omv.type() == type) { + fn->do_value(omv.reg(), omv.type()); + } + } +} + +static void update_register_map1(const ImmutableOopMap* oopmap, const frame* fr, RegisterMap* reg_map) { + for (OopMapStream oms(oopmap); !oms.is_done(); oms.next()) { + OopMapValue omv = oms.current(); + if (omv.type() == OopMapValue::callee_saved_value) { + VMReg reg = omv.content_reg(); + address loc = fr->oopmapreg_to_location(omv.reg(), reg_map); + reg_map->set_location(reg, loc); + //DEBUG_ONLY(nof_callee++;) + } + } +} + +// Update callee-saved register info for the following frame +void ImmutableOopMap::update_register_map(const frame *fr, RegisterMap *reg_map) const { + CodeBlob* cb = fr->cb(); + assert(cb != NULL, "no codeblob"); + // Any reg might be saved by a safepoint handler (see generate_handler_blob). + assert( reg_map->_update_for_id == NULL || fr->is_older(reg_map->_update_for_id), + "already updated this map; do not 'update' it twice!" ); + debug_only(reg_map->_update_for_id = fr->id()); + + // Check if caller must update oop argument + assert((reg_map->include_argument_oops() || + !cb->caller_must_gc_arguments(reg_map->thread())), + "include_argument_oops should already be set"); + + // Scan through oopmap and find location of all callee-saved registers + // (we do not do update in place, since info could be overwritten) + + DEBUG_ONLY(int nof_callee = 0;) + update_register_map1(this, fr, reg_map); + + // Check that runtime stubs save all callee-saved registers +#ifdef COMPILER2 + assert(cb == NULL || cb->is_compiled_by_c1() || cb->is_compiled_by_jvmci() || !cb->is_runtime_stub() || + (nof_callee >= SAVED_ON_ENTRY_REG_COUNT || nof_callee >= C_SAVED_ON_ENTRY_REG_COUNT), + "must save all"); +#endif // COMPILER2 +} + +const ImmutableOopMap* OopMapSet::find_map(const frame *fr) { + return find_map(fr->cb(), fr->pc()); +} + +const ImmutableOopMap* OopMapSet::find_map(const CodeBlob* cb, address pc) { + assert(cb != NULL, "no codeblob"); + const ImmutableOopMap* map = cb->oop_map_for_return_address(pc); + assert(map != NULL, "no ptr map found"); + return map; +} + +// Update callee-saved register info for the following frame +void OopMapSet::update_register_map(const frame *fr, RegisterMap *reg_map) { + find_map(fr)->update_register_map(fr, reg_map); +} + +//============================================================================= +// Non-Product code #ifndef PRODUCT -static void trace_codeblob_maps(const frame *fr, const RegisterMap *reg_map) { +void OopMapSet::trace_codeblob_maps(const frame *fr, const RegisterMap *reg_map) { // Print oopmap and regmap tty->print_cr("------ "); CodeBlob* cb = fr->cb(); @@ -254,149 +572,14 @@ static void trace_codeblob_maps(const frame *fr, const RegisterMap *reg_map) { fr->print_on(tty); tty->print(" "); cb->print_value_on(tty); tty->cr(); - reg_map->print(); + if (reg_map != NULL) { + reg_map->print(); + } tty->print_cr("------ "); } #endif // PRODUCT -void OopMapSet::oops_do(const frame *fr, const RegisterMap* reg_map, OopClosure* f, DerivedPointerIterationMode mode) { - switch (mode) { - case DerivedPointerIterationMode::_directly: - all_do(fr, reg_map, f, process_derived_oop); - break; - case DerivedPointerIterationMode::_with_table: - all_do(fr, reg_map, f, add_derived_oop); - break; - case DerivedPointerIterationMode::_ignore: - all_do(fr, reg_map, f, ignore_derived_oop); - break; - } -} - - -void OopMapSet::all_do(const frame *fr, const RegisterMap *reg_map, - OopClosure* oop_fn, void derived_oop_fn(oop*, derived_pointer*, OopClosure*)) { - CodeBlob* cb = fr->cb(); - assert(cb != NULL, "no codeblob"); - - NOT_PRODUCT(if (TraceCodeBlobStacks) trace_codeblob_maps(fr, reg_map);) - - const ImmutableOopMap* map = cb->oop_map_for_return_address(fr->pc()); - assert(map != NULL, "no ptr map found"); - - // handle derived pointers first (otherwise base pointer may be - // changed before derived pointer offset has been collected) - { - for (OopMapStream oms(map); !oms.is_done(); oms.next()) { - OopMapValue omv = oms.current(); - if (omv.type() != OopMapValue::derived_oop_value) { - continue; - } - -#ifndef COMPILER2 - COMPILER1_PRESENT(ShouldNotReachHere();) -#if INCLUDE_JVMCI - if (UseJVMCICompiler) { - ShouldNotReachHere(); - } -#endif -#endif // !COMPILER2 - derived_pointer* derived_loc = (derived_pointer*)fr->oopmapreg_to_location(omv.reg(),reg_map); - guarantee(derived_loc != NULL, "missing saved register"); - oop* base_loc = fr->oopmapreg_to_oop_location(omv.content_reg(), reg_map); - // Ignore NULL oops and decoded NULL narrow oops which - // equal to CompressedOops::base() when a narrow oop - // implicit null check is used in compiled code. - // The narrow_oop_base could be NULL or be the address - // of the page below heap depending on compressed oops mode. - if (base_loc != NULL && *base_loc != NULL && !CompressedOops::is_base(*base_loc)) { - derived_oop_fn(base_loc, derived_loc, oop_fn); - } - } - } - - { - // We want coop and oop oop_types - for (OopMapStream oms(map); !oms.is_done(); oms.next()) { - OopMapValue omv = oms.current(); - oop* loc = fr->oopmapreg_to_oop_location(omv.reg(),reg_map); - // It should be an error if no location can be found for a - // register mentioned as contained an oop of some kind. Maybe - // this was allowed previously because value_value items might - // be missing? - guarantee(loc != NULL, "missing saved register"); - if ( omv.type() == OopMapValue::oop_value ) { - oop val = *loc; - if (val == NULL || CompressedOops::is_base(val)) { - // Ignore NULL oops and decoded NULL narrow oops which - // equal to CompressedOops::base() when a narrow oop - // implicit null check is used in compiled code. - // The narrow_oop_base could be NULL or be the address - // of the page below heap depending on compressed oops mode. - continue; - } - oop_fn->do_oop(loc); - } else if ( omv.type() == OopMapValue::narrowoop_value ) { - narrowOop *nl = (narrowOop*)loc; -#ifndef VM_LITTLE_ENDIAN - VMReg vmReg = omv.reg(); - if (!vmReg->is_stack()) { - // compressed oops in registers only take up 4 bytes of an - // 8 byte register but they are in the wrong part of the - // word so adjust loc to point at the right place. - nl = (narrowOop*)((address)nl + 4); - } -#endif - oop_fn->do_oop(nl); - } - } - } -} - - -// Update callee-saved register info for the following frame -void OopMapSet::update_register_map(const frame *fr, RegisterMap *reg_map) { - ResourceMark rm; - CodeBlob* cb = fr->cb(); - assert(cb != NULL, "no codeblob"); - - // Any reg might be saved by a safepoint handler (see generate_handler_blob). - assert( reg_map->_update_for_id == NULL || fr->is_older(reg_map->_update_for_id), - "already updated this map; do not 'update' it twice!" ); - debug_only(reg_map->_update_for_id = fr->id()); - - // Check if caller must update oop argument - assert((reg_map->include_argument_oops() || - !cb->caller_must_gc_arguments(reg_map->thread())), - "include_argument_oops should already be set"); - - // Scan through oopmap and find location of all callee-saved registers - // (we do not do update in place, since info could be overwritten) - - address pc = fr->pc(); - const ImmutableOopMap* map = cb->oop_map_for_return_address(pc); - assert(map != NULL, "no ptr map found"); - DEBUG_ONLY(int nof_callee = 0;) - - for (OopMapStream oms(map); !oms.is_done(); oms.next()) { - OopMapValue omv = oms.current(); - if (omv.type() == OopMapValue::callee_saved_value) { - VMReg reg = omv.content_reg(); - oop* loc = fr->oopmapreg_to_oop_location(omv.reg(), reg_map); - reg_map->set_location(reg, (address) loc); - DEBUG_ONLY(nof_callee++;) - } - } - - // Check that runtime stubs save all callee-saved registers -#ifdef COMPILER2 - assert(cb->is_compiled_by_c1() || cb->is_compiled_by_jvmci() || !cb->is_runtime_stub() || - (nof_callee >= SAVED_ON_ENTRY_REG_COUNT || nof_callee >= C_SAVED_ON_ENTRY_REG_COUNT), - "must save all"); -#endif // COMPILER2 -} - // Printing code is present in product build for -XX:+PrintAssembly. static @@ -507,6 +690,18 @@ bool OopMap::equals(const OopMap* other) const { return true; } +int ImmutableOopMapSet::find_slot_for_offset(int pc_offset) const { + // we might not have an oopmap at asynchronous (non-safepoint) stackwalks + ImmutableOopMapPair* pairs = get_pairs(); + for (int i = 0; i < _count; ++i) { + if (pairs[i].pc_offset() >= pc_offset) { + ImmutableOopMapPair* last = &pairs[i]; + return last->pc_offset() == pc_offset ? i : -1; + } + } + return -1; +} + const ImmutableOopMap* ImmutableOopMapSet::find_map_at_offset(int pc_offset) const { ImmutableOopMapPair* pairs = get_pairs(); ImmutableOopMapPair* last = NULL; @@ -524,13 +719,21 @@ const ImmutableOopMap* ImmutableOopMapSet::find_map_at_offset(int pc_offset) con return last->get_from(this); } -const ImmutableOopMap* ImmutableOopMapPair::get_from(const ImmutableOopMapSet* set) const { - return set->oopmap_at_offset(_oopmap_offset); +ImmutableOopMap::ImmutableOopMap(const OopMap* oopmap) + : _count(oopmap->count()), _num_oops(oopmap->num_oops()) { + _num_oops = oopmap->num_oops(); + _has_derived_oops = oopmap->has_derived_oops(); + address addr = data_addr(); + oopmap->copy_and_sort_data_to(addr); } -ImmutableOopMap::ImmutableOopMap(const OopMap* oopmap) : _count(oopmap->count()) { - address addr = data_addr(); - oopmap->copy_data_to(addr); +bool ImmutableOopMap::has_any(OopMapValue::oop_types type) const { + for (OopMapStream oms(this); !oms.is_done(); oms.next()) { + if (oms.current().type() == type) { + return true; + } + } + return false; } #ifdef ASSERT @@ -622,8 +825,8 @@ void ImmutableOopMapBuilder::fill(ImmutableOopMapSet* set, int sz) { fill_pair(&pairs[i], map, _mapping[i]._offset, set); } - const ImmutableOopMap* nv = set->find_map_at_offset(map->offset()); - assert(memcmp(map->data(), nv->data_addr(), map->data_size()) == 0, "check identity"); + //const ImmutableOopMap* nv = set->find_map_at_offset(map->offset()); + //assert(memcmp(map->data(), nv->data_addr(), map->data_size()) == 0, "check identity"); } } @@ -755,6 +958,8 @@ void DerivedPointerTable::update_pointers() { *derived_loc = derived_base + offset; assert(*derived_loc - derived_base == offset, "sanity check"); + // assert(offset >= 0 && offset <= (intptr_t)(base->size() << LogHeapWordSize), "offset: %ld base->size: %zu relative: %d", offset, base->size() << LogHeapWordSize, *(intptr_t*)derived_loc <= 0); + if (TraceDerivedPointers) { tty->print_cr("Updating derived pointer@" INTPTR_FORMAT " - Derived: " INTPTR_FORMAT " Base: " INTPTR_FORMAT " (Offset: " INTX_FORMAT ")", diff --git a/src/hotspot/share/compiler/oopMap.hpp b/src/hotspot/share/compiler/oopMap.hpp index 7244df25e72..d3c2acd0a28 100644 --- a/src/hotspot/share/compiler/oopMap.hpp +++ b/src/hotspot/share/compiler/oopMap.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1998, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1998, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -28,6 +28,7 @@ #include "code/compressedStream.hpp" #include "code/vmreg.hpp" #include "memory/allocation.hpp" +#include "memory/iterator.hpp" #include "oops/oopsHierarchy.hpp" #include "utilities/growableArray.hpp" @@ -44,6 +45,8 @@ enum class DerivedPointerIterationMode; class frame; class RegisterMap; class OopClosure; +class CodeBlob; +class ImmutableOopMap; enum class derived_pointer : intptr_t {}; @@ -147,9 +150,14 @@ public: class OopMap: public ResourceObj { friend class OopMapStream; friend class VMStructs; + friend class OopMapSet; + friend class OopMapSort; private: int _pc_offset; // offset in the code that this OopMap corresponds to int _omv_count; // number of OopMapValues in the stream + int _num_oops; // number of oops + int _index; // index in OopMapSet + bool _has_derived_oops; CompressedWriteStream* _write_stream; debug_only( OopMapValue::oop_types* _locs_used; int _locs_length;) @@ -158,10 +166,11 @@ class OopMap: public ResourceObj { int omv_count() const { return _omv_count; } void set_omv_count(int value) { _omv_count = value; } void increment_count() { _omv_count++; } + void increment_num_oops() { _num_oops++; } + void set_has_derived_oops(bool value) { _has_derived_oops = value; } CompressedWriteStream* write_stream() const { return _write_stream; } void set_write_stream(CompressedWriteStream* value) { _write_stream = value; } - private: enum DeepCopyToken { _deep_copy_token }; OopMap(DeepCopyToken, OopMap* source); // used only by deep_copy @@ -176,6 +185,9 @@ class OopMap: public ResourceObj { int count() const { return _omv_count; } int data_size() const { return write_stream()->position(); } address data() const { return write_stream()->buffer(); } + int num_oops() const { return _num_oops; } + bool has_derived_oops() const { return _has_derived_oops; } + int index() const { return _index; } // Construction // frame_size units are stack-slots (4 bytes) NOT intptr_t; we can name odd @@ -187,6 +199,7 @@ class OopMap: public ResourceObj { int heap_size() const; void copy_data_to(address addr) const; + void copy_and_sort_data_to(address addr) const; OopMap* deep_copy(); bool legal_vm_reg_name(VMReg local) { @@ -204,7 +217,7 @@ class OopMapSet : public ResourceObj { private: GrowableArray _list; - void add(OopMap* value) { _list.append(value); } + int add(OopMap* value) { return _list.append(value); } public: OopMapSet(); @@ -215,23 +228,29 @@ class OopMapSet : public ResourceObj { OopMap* at(int index) const { return _list.at(index); } // Collect OopMaps. - void add_gc_map(int pc, OopMap* map); + int add_gc_map(int pc, OopMap* map); // Methods oops_do() and all_do() filter out NULL oops and // oop == CompressedOops::base() before passing oops // to closures. + static const ImmutableOopMap* find_map(const CodeBlob* cb, address pc); + static const ImmutableOopMap* find_map(const frame *fr); + // Iterates through frame for a compiled method + static void oops_do (const frame* fr, + const RegisterMap* reg_map, + OopClosure* f, + DerivedOopClosure* df); static void oops_do (const frame* fr, const RegisterMap* reg_map, OopClosure* f, DerivedPointerIterationMode mode); static void update_register_map(const frame* fr, RegisterMap *reg_map); - // Iterates through frame for a compiled method for dead ones and values, too - static void all_do(const frame* fr, const RegisterMap* reg_map, - OopClosure* oop_fn, - void derived_oop_fn(oop* base, derived_pointer* derived, OopClosure* oop_fn)); +#ifndef PRODUCT + static void trace_codeblob_maps(const frame *fr, const RegisterMap *reg_map); +#endif // Printing void print_on(outputStream* st) const; @@ -240,24 +259,47 @@ class OopMapSet : public ResourceObj { class ImmutableOopMapBuilder; +class OopMapClosure : public Closure { + public: + virtual bool handle_type(OopMapValue::oop_types type) { return true; } + virtual void do_value(VMReg reg, OopMapValue::oop_types type) = 0; +}; + +template +class OopMapDo; + class ImmutableOopMap { friend class OopMapStream; friend class VMStructs; + template + friend class OopMapDo; #ifdef ASSERT friend class ImmutableOopMapBuilder; #endif private: int _count; // contains the number of entries in this OopMap + int _num_oops; + bool _has_derived_oops; address data_addr() const { return (address) this + sizeof(ImmutableOopMap); } public: ImmutableOopMap(const OopMap* oopmap); int count() const { return _count; } + int num_oops() const { return _num_oops; } + bool has_derived_oops() const { return _has_derived_oops; } + bool has_any(OopMapValue::oop_types type) const; + #ifdef ASSERT int nr_of_bytes() const; // this is an expensive operation, only used in debug builds #endif + void oops_do(const frame* fr, const RegisterMap* reg_map, OopClosure* f, DerivedOopClosure* df) const; + void oops_do(const frame* fr, const RegisterMap* reg_map, OopClosure* f, DerivedPointerIterationMode derived_mode) const; + void all_type_do(const frame *fr, OopMapValue::oop_types type, OopMapClosure* fn) const; + void all_type_do(const frame *fr, OopMapClosure* fn) const; + void update_register_map(const frame* fr, RegisterMap *reg_map) const; + // Printing void print_on(outputStream* st) const; void print() const; @@ -303,7 +345,9 @@ public: static ImmutableOopMapSet* build_from(const OopMapSet* oopmap_set); + int find_slot_for_offset(int pc_offset) const; const ImmutableOopMap* find_map_at_offset(int pc_offset) const; + const ImmutableOopMap* find_map_at_slot(int slot, int pc_offset) const; const ImmutableOopMapPair* pair_at(int index) const { assert(index >= 0 && index < _count, "check"); return &get_pairs()[index]; } @@ -316,7 +360,7 @@ public: class OopMapStream : public StackObj { private: - CompressedReadStream* _stream; + CompressedReadStream _stream; int _size; int _position; bool _valid_omv; @@ -324,13 +368,13 @@ class OopMapStream : public StackObj { void find_next(); public: - OopMapStream(OopMap* oop_map); + OopMapStream(const OopMap* oop_map); OopMapStream(const ImmutableOopMap* oop_map); bool is_done() { if(!_valid_omv) { find_next(); } return !_valid_omv; } void next() { find_next(); } OopMapValue current() { return _omv; } #ifdef ASSERT - int stream_position() const { return _stream->position(); } + int stream_position() const { return _stream.position(); } #endif }; @@ -403,6 +447,29 @@ private: void fill(ImmutableOopMapSet* set, int size); }; +class SkipNullValue { +public: + static inline bool should_skip(oop val); +}; + +class IncludeAllValues { +public: + static bool should_skip(oop value) { return false; } +}; + +template +class OopMapDo { +private: + OopFnT* _oop_fn; + DerivedOopFnT* _derived_oop_fn; +public: + OopMapDo(OopFnT* oop_fn, DerivedOopFnT* derived_oop_fn) : _oop_fn(oop_fn), _derived_oop_fn(derived_oop_fn) {} + template + void oops_do(const frame* fr, const RegisterMapT* reg_map, const ImmutableOopMap* oopmap); +private: + template + void iterate_oops_do(const frame *fr, const RegisterMapT *reg_map, const ImmutableOopMap* oopmap); +}; // Derived pointer support. This table keeps track of all derived points on a // stack. It is cleared before each scavenge/GC. During the traversal of all diff --git a/src/hotspot/share/compiler/oopMap.inline.hpp b/src/hotspot/share/compiler/oopMap.inline.hpp new file mode 100644 index 00000000000..d9cd5ae1ee8 --- /dev/null +++ b/src/hotspot/share/compiler/oopMap.inline.hpp @@ -0,0 +1,157 @@ +/* + * Copyright (c) 1998, 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_COMPILER_OOPMAP_INLINE_HPP +#define SHARE_VM_COMPILER_OOPMAP_INLINE_HPP + +#include "compiler/oopMap.hpp" + +#include "oops/compressedOops.hpp" +#include "runtime/frame.inline.hpp" +#include "runtime/globals.hpp" +#include "utilities/ostream.hpp" + +inline const ImmutableOopMap* ImmutableOopMapSet::find_map_at_slot(int slot, int pc_offset) const { + assert(slot >= 0 && slot < _count, "bounds count: %d slot: %d", _count, slot); + ImmutableOopMapPair* pairs = get_pairs(); + ImmutableOopMapPair* last = &pairs[slot]; + assert(last->pc_offset() == pc_offset, "oopmap not found"); + return last->get_from(this); +} + +inline const ImmutableOopMap* ImmutableOopMapPair::get_from(const ImmutableOopMapSet* set) const { + return set->oopmap_at_offset(_oopmap_offset); +} + +inline bool SkipNullValue::should_skip(oop val) { + return val == (oop)NULL || CompressedOops::is_base(val); +} + +template +template +void OopMapDo::iterate_oops_do(const frame *fr, const RegisterMapT *reg_map, const ImmutableOopMap* oopmap) { + NOT_PRODUCT(if (TraceCodeBlobStacks) OopMapSet::trace_codeblob_maps(fr, reg_map->as_RegisterMap());) + assert(fr != NULL, ""); + + // handle derived pointers first (otherwise base pointer may be + // changed before derived pointer offset has been collected) + if (_derived_oop_fn != nullptr) { + for (OopMapStream oms(oopmap); !oms.is_done(); oms.next()) { + OopMapValue omv = oms.current(); + if (omv.type() != OopMapValue::derived_oop_value) + continue; + + #ifndef COMPILER2 + COMPILER1_PRESENT(ShouldNotReachHere();) + #if INCLUDE_JVMCI + if (UseJVMCICompiler) { + ShouldNotReachHere(); + } + #endif + #endif // !COMPILER2 + + address loc = fr->oopmapreg_to_location(omv.reg(), reg_map); + + DEBUG_ONLY(if (loc == NULL && reg_map->should_skip_missing()) continue;) + + if (loc == NULL) { + tty->print("oops reg: "); omv.reg()->print_on(tty); tty->cr(); + fr->print_on(tty); + } + guarantee(loc != NULL, "missing saved register"); + derived_pointer* derived_loc = (derived_pointer*)loc; + oop* base_loc = fr->oopmapreg_to_oop_location(omv.content_reg(), reg_map); + // Ignore NULL oops and decoded NULL narrow oops which + // equal to CompressedOops::base() when a narrow oop + // implicit null check is used in compiled code. + // The narrow_oop_base could be NULL or be the address + // of the page below heap depending on compressed oops mode. + if (base_loc != NULL && *base_loc != (oop)NULL && !CompressedOops::is_base(*base_loc)) { + _derived_oop_fn->do_derived_oop(base_loc, derived_loc); + } + } + } + + // We want coop and oop oop_types + if (_oop_fn != nullptr) { + for (OopMapStream oms(oopmap); !oms.is_done(); oms.next()) { + OopMapValue omv = oms.current(); + if (omv.type() != OopMapValue::oop_value && omv.type() != OopMapValue::narrowoop_value) + continue; + oop* loc = fr->oopmapreg_to_oop_location(omv.reg(),reg_map); + // It should be an error if no location can be found for a + // register mentioned as contained an oop of some kind. Maybe + // this was allowed previously because value_value items might + // be missing? +#ifdef ASSERT + if (loc == NULL) { + if (reg_map->should_skip_missing()) + continue; + VMReg reg = omv.reg(); + tty->print_cr("missing saved register: reg: " INTPTR_FORMAT " %s loc: %p", reg->value(), reg->name(), loc); + fr->print_on(tty); + } +#endif + if (loc == NULL) { + tty->print("oops reg: "); omv.reg()->print_on(tty); tty->cr(); + fr->print_on(tty); + } + guarantee(loc != NULL, "missing saved register"); + if ( omv.type() == OopMapValue::oop_value ) { + oop val = *loc; + if (ValueFilterT::should_skip(val)) { // TODO: UGLY (basically used to decide if we're freezing/thawing continuation) + // Ignore NULL oops and decoded NULL narrow oops which + // equal to CompressedOops::base() when a narrow oop + // implicit null check is used in compiled code. + // The narrow_oop_base could be NULL or be the address + // of the page below heap depending on compressed oops mode. + continue; + } + _oop_fn->do_oop(loc); + } else if ( omv.type() == OopMapValue::narrowoop_value ) { + narrowOop *nl = (narrowOop*)loc; +#ifndef VM_LITTLE_ENDIAN + VMReg vmReg = omv.reg(); + if (!vmReg->is_stack()) { + // compressed oops in registers only take up 4 bytes of an + // 8 byte register but they are in the wrong part of the + // word so adjust loc to point at the right place. + nl = (narrowOop*)((address)nl + 4); + } +#endif + _oop_fn->do_oop(nl); + } + } + } +} + + +template +template +void OopMapDo::oops_do(const frame *fr, const RegisterMapT *reg_map, const ImmutableOopMap* oopmap) { + iterate_oops_do(fr, reg_map, oopmap); +} + +#endif // SHARE_VM_COMPILER_OOPMAP_INLINE_HPP + diff --git a/src/hotspot/share/gc/epsilon/epsilonHeap.hpp b/src/hotspot/share/gc/epsilon/epsilonHeap.hpp index 036593c72ae..54d35669496 100644 --- a/src/hotspot/share/gc/epsilon/epsilonHeap.hpp +++ b/src/hotspot/share/gc/epsilon/epsilonHeap.hpp @@ -82,6 +82,8 @@ public: return _space->is_in(p); } + virtual bool requires_barriers(stackChunkOop obj) const { return false; } + virtual bool is_maximal_no_gc() const { // No GC is going to happen. Return "we are at max", when we are about to fail. return used() == capacity(); diff --git a/src/hotspot/share/gc/g1/g1BarrierSet.cpp b/src/hotspot/share/gc/g1/g1BarrierSet.cpp index a04c7370518..bd0eaed9f99 100644 --- a/src/hotspot/share/gc/g1/g1BarrierSet.cpp +++ b/src/hotspot/share/gc/g1/g1BarrierSet.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2001, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -138,6 +138,7 @@ void G1BarrierSet::on_thread_destroy(Thread* thread) { } void G1BarrierSet::on_thread_attach(Thread* thread) { + BarrierSet::on_thread_attach(thread); SATBMarkQueue& queue = G1ThreadLocalData::satb_mark_queue(thread); assert(!queue.is_active(), "SATB queue should not be active"); assert(queue.buffer() == nullptr, "SATB queue should not have a buffer"); diff --git a/src/hotspot/share/gc/g1/g1CodeBlobClosure.cpp b/src/hotspot/share/gc/g1/g1CodeBlobClosure.cpp index f2dc595ef53..be9b82ba8f8 100644 --- a/src/hotspot/share/gc/g1/g1CodeBlobClosure.cpp +++ b/src/hotspot/share/gc/g1/g1CodeBlobClosure.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2015, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -29,6 +29,7 @@ #include "gc/g1/g1ConcurrentMark.inline.hpp" #include "gc/g1/heapRegion.hpp" #include "gc/g1/heapRegionRemSet.inline.hpp" +#include "gc/shared/barrierSetNMethod.hpp" #include "oops/access.inline.hpp" #include "oops/compressedOops.inline.hpp" #include "oops/oop.inline.hpp" @@ -75,12 +76,36 @@ void G1CodeBlobClosure::MarkingOopClosure::do_oop(narrowOop* o) { void G1CodeBlobClosure::do_evacuation_and_fixup(nmethod* nm) { _oc.set_nm(nm); + + // Evacuate objects pointed to by the nmethod nm->oops_do(&_oc); + + if (_strong) { + // CodeCache sweeper support + nm->mark_as_maybe_on_continuation(); + + BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod(); + if (bs_nm != NULL) { + bs_nm->disarm(nm); + } + } + nm->fix_oop_relocations(); } void G1CodeBlobClosure::do_marking(nmethod* nm) { + // Mark through oops in the nmethod nm->oops_do(&_marking_oc); + + // CodeCache sweeper support + nm->mark_as_maybe_on_continuation(); + + BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod(); + if (bs_nm != NULL) { + bs_nm->disarm(nm); + } + + // The oops were only marked, no need to update oop relocations. } class G1NmethodProcessor : public nmethod::OopsDoProcessor { diff --git a/src/hotspot/share/gc/g1/g1CollectedHeap.cpp b/src/hotspot/share/gc/g1/g1CollectedHeap.cpp index a2c6c226b02..b7ad2bc42fd 100644 --- a/src/hotspot/share/gc/g1/g1CollectedHeap.cpp +++ b/src/hotspot/share/gc/g1/g1CollectedHeap.cpp @@ -1493,6 +1493,9 @@ G1CollectedHeap::G1CollectedHeap() : // objects are created. _filler_array_max_size = _humongous_object_threshold_in_words; + // Override the default _stack_chunk_max_size so that no humongous stack chunks are created + _stack_chunk_max_size = _humongous_object_threshold_in_words; + uint n_queues = ParallelGCThreads; _task_queues = new G1ScannerTasksQueueSet(n_queues); @@ -1893,6 +1896,7 @@ bool G1CollectedHeap::should_do_concurrent_full_gc(GCCause::Cause cause) { case GCCause::_g1_humongous_allocation: return true; case GCCause::_g1_periodic_collection: return G1PeriodicGCInvokesConcurrent; case GCCause::_wb_breakpoint: return true; + case GCCause::_codecache_GC_threshold: return true; default: return is_user_requested_concurrent_full_gc(cause); } } @@ -3451,3 +3455,16 @@ void G1CollectedHeap::fill_with_dummy_object(HeapWord* start, HeapWord* end, boo HeapRegion* region = heap_region_containing(start); region->fill_with_dummy_object(start, pointer_delta(end, start), zap); } + +void G1CollectedHeap::start_codecache_marking_cycle_if_inactive() { + if (!Continuations::is_gc_marking_cycle_active()) { + // This is the normal case when we do not call collect when a + // concurrent mark is ongoing. We then start a new code marking + // cycle. If, on the other hand, a concurrent mark is ongoing, we + // will be conservative and use the last code marking cycle. Code + // caches marked between the two concurrent marks will live a bit + // longer than needed. + Continuations::on_gc_marking_cycle_start(); + Continuations::arm_all_nmethods(); + } +} diff --git a/src/hotspot/share/gc/g1/g1CollectedHeap.hpp b/src/hotspot/share/gc/g1/g1CollectedHeap.hpp index de2442c7ce5..7f1344953f1 100644 --- a/src/hotspot/share/gc/g1/g1CollectedHeap.hpp +++ b/src/hotspot/share/gc/g1/g1CollectedHeap.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2001, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -937,6 +937,8 @@ public: void fill_with_dummy_object(HeapWord* start, HeapWord* end, bool zap) override; + static void start_codecache_marking_cycle_if_inactive(); + // Apply the given closure on all cards in the Hot Card Cache, emptying it. void iterate_hcc_closure(G1CardTableEntryClosure* cl, uint worker_id); @@ -1180,6 +1182,7 @@ public: size_t unsafe_max_tlab_alloc(Thread* ignored) const override; inline bool is_in_young(const oop obj) const; + inline bool requires_barriers(stackChunkOop obj) const override; // Returns "true" iff the given word_size is "very large". static bool is_humongous(size_t word_size) { diff --git a/src/hotspot/share/gc/g1/g1CollectedHeap.inline.hpp b/src/hotspot/share/gc/g1/g1CollectedHeap.inline.hpp index 0cd8de23e56..55f0d2762a2 100644 --- a/src/hotspot/share/gc/g1/g1CollectedHeap.inline.hpp +++ b/src/hotspot/share/gc/g1/g1CollectedHeap.inline.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2001, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -214,6 +214,11 @@ inline bool G1CollectedHeap::is_in_young(const oop obj) const { return heap_region_containing(obj)->is_young(); } +inline bool G1CollectedHeap::requires_barriers(stackChunkOop obj) const { + assert(obj != NULL, ""); + return !heap_region_containing(obj)->is_young(); // is_in_young does an unnecessary NULL check +} + inline bool G1CollectedHeap::is_obj_dead(const oop obj, const HeapRegion* hr) const { return hr->is_obj_dead(obj, _cm->prev_mark_bitmap()); } diff --git a/src/hotspot/share/gc/g1/g1ConcurrentMark.cpp b/src/hotspot/share/gc/g1/g1ConcurrentMark.cpp index a8a8f1448de..c5c2d9ff329 100644 --- a/src/hotspot/share/gc/g1/g1ConcurrentMark.cpp +++ b/src/hotspot/share/gc/g1/g1ConcurrentMark.cpp @@ -66,6 +66,7 @@ #include "oops/access.inline.hpp" #include "oops/oop.inline.hpp" #include "runtime/atomic.hpp" +#include "runtime/continuation.hpp" #include "runtime/globals_extension.hpp" #include "runtime/handles.inline.hpp" #include "runtime/java.hpp" @@ -814,6 +815,8 @@ G1PreConcurrentStartTask::G1PreConcurrentStartTask(GCCause::Cause cause, G1Concu void G1ConcurrentMark::pre_concurrent_start(GCCause::Cause cause) { assert_at_safepoint_on_vm_thread(); + G1CollectedHeap::start_codecache_marking_cycle_if_inactive(); + G1PreConcurrentStartTask cl(cause, this); G1CollectedHeap::heap()->run_batch_task(&cl); @@ -1298,6 +1301,9 @@ void G1ConcurrentMark::remark() { report_object_count(mark_finished); } + Continuations::on_gc_marking_cycle_finish(); + Continuations::arm_all_nmethods(); + // Statistics double now = os::elapsedTime(); _remark_mark_times.add((mark_work_end - start) * 1000.0); @@ -1772,7 +1778,7 @@ class G1RemarkThreadsClosure : public ThreadClosure { G1RemarkThreadsClosure(G1CollectedHeap* g1h, G1CMTask* task) : _qset(G1BarrierSet::satb_mark_queue_set()), _cm_cl(g1h, task), - _code_cl(&_cm_cl, !CodeBlobToOopClosure::FixRelocations), + _code_cl(&_cm_cl, !CodeBlobToOopClosure::FixRelocations, true /* keepalive nmethods */), _claim_token(Threads::thread_claim_token()) {} void do_thread(Thread* thread) { diff --git a/src/hotspot/share/gc/g1/g1FullCollector.cpp b/src/hotspot/share/gc/g1/g1FullCollector.cpp index 7e07ff4757c..077687af898 100644 --- a/src/hotspot/share/gc/g1/g1FullCollector.cpp +++ b/src/hotspot/share/gc/g1/g1FullCollector.cpp @@ -44,6 +44,7 @@ #include "gc/shared/weakProcessor.inline.hpp" #include "gc/shared/workerPolicy.hpp" #include "logging/log.hpp" +#include "runtime/continuation.hpp" #include "runtime/handles.inline.hpp" #include "utilities/debug.hpp" @@ -187,6 +188,8 @@ void G1FullCollector::prepare_collection() { } void G1FullCollector::collect() { + G1CollectedHeap::start_codecache_marking_cycle_if_inactive(); + phase1_mark_live_objects(); verify_after_marking(); @@ -198,6 +201,9 @@ void G1FullCollector::collect() { phase3_adjust_pointers(); phase4_do_compaction(); + + Continuations::on_gc_marking_cycle_finish(); + Continuations::arm_all_nmethods(); } void G1FullCollector::complete_collection() { diff --git a/src/hotspot/share/gc/g1/g1FullGCCompactTask.cpp b/src/hotspot/share/gc/g1/g1FullGCCompactTask.cpp index f484ee5133c..97c0d44424c 100644 --- a/src/hotspot/share/gc/g1/g1FullGCCompactTask.cpp +++ b/src/hotspot/share/gc/g1/g1FullGCCompactTask.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -87,6 +87,8 @@ size_t G1FullGCCompactTask::G1CompactRegionClosure::apply(oop obj) { HeapWord* obj_addr = cast_from_oop(obj); assert(obj_addr != destination, "everything in this pass should be moving"); Copy::aligned_conjoint_words(obj_addr, destination, size); + + // There is no need to transform stack chunks - marking already did that. cast_to_oop(destination)->init_mark(); assert(cast_to_oop(destination)->klass() != NULL, "should have a class"); diff --git a/src/hotspot/share/gc/g1/g1FullGCMarkTask.cpp b/src/hotspot/share/gc/g1/g1FullGCMarkTask.cpp index e7054983f5b..83a73a0d843 100644 --- a/src/hotspot/share/gc/g1/g1FullGCMarkTask.cpp +++ b/src/hotspot/share/gc/g1/g1FullGCMarkTask.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -45,7 +45,7 @@ void G1FullGCMarkTask::work(uint worker_id) { Ticks start = Ticks::now(); ResourceMark rm; G1FullGCMarker* marker = collector()->marker(worker_id); - MarkingCodeBlobClosure code_closure(marker->mark_closure(), !CodeBlobToOopClosure::FixRelocations); + MarkingCodeBlobClosure code_closure(marker->mark_closure(), !CodeBlobToOopClosure::FixRelocations, true /* keepalive nmethods */); if (ClassUnloading) { _root_processor.process_strong_roots(marker->mark_closure(), diff --git a/src/hotspot/share/gc/g1/g1FullGCMarker.inline.hpp b/src/hotspot/share/gc/g1/g1FullGCMarker.inline.hpp index ce3013138ce..cfa0ef9e9d1 100644 --- a/src/hotspot/share/gc/g1/g1FullGCMarker.inline.hpp +++ b/src/hotspot/share/gc/g1/g1FullGCMarker.inline.hpp @@ -35,6 +35,7 @@ #include "gc/g1/g1FullGCOopClosures.inline.hpp" #include "gc/g1/g1RegionMarkStatsCache.hpp" #include "gc/g1/g1StringDedup.hpp" +#include "gc/shared/continuationGCSupport.inline.hpp" #include "gc/shared/preservedMarks.inline.hpp" #include "gc/shared/stringdedup/stringDedup.hpp" #include "oops/access.inline.hpp" @@ -67,6 +68,8 @@ inline bool G1FullGCMarker::mark_object(oop obj) { _string_dedup_requests.add(obj); } + ContinuationGCSupport::transform_stack_chunk(obj); + // Collect live words. _mark_stats_cache.add_live_words(obj); diff --git a/src/hotspot/share/gc/g1/g1FullGCOopClosures.hpp b/src/hotspot/share/gc/g1/g1FullGCOopClosures.hpp index 1690c99ea24..d3af21e4db0 100644 --- a/src/hotspot/share/gc/g1/g1FullGCOopClosures.hpp +++ b/src/hotspot/share/gc/g1/g1FullGCOopClosures.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -75,6 +75,8 @@ public: virtual bool do_metadata(); virtual void do_klass(Klass* k); virtual void do_cld(ClassLoaderData* cld); + virtual void do_method(Method* m); + virtual void do_nmethod(nmethod* nm); }; class G1AdjustClosure : public BasicOopIterateClosure { diff --git a/src/hotspot/share/gc/g1/g1FullGCOopClosures.inline.hpp b/src/hotspot/share/gc/g1/g1FullGCOopClosures.inline.hpp index f4b9d9e1176..738145afcab 100644 --- a/src/hotspot/share/gc/g1/g1FullGCOopClosures.inline.hpp +++ b/src/hotspot/share/gc/g1/g1FullGCOopClosures.inline.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -63,6 +63,14 @@ inline void G1MarkAndPushClosure::do_cld(ClassLoaderData* cld) { _marker->follow_cld(cld); } +inline void G1MarkAndPushClosure::do_method(Method* m) { + m->record_gc_epoch(); +} + +inline void G1MarkAndPushClosure::do_nmethod(nmethod* nm) { + nm->follow_nmethod(this); +} + template inline void G1AdjustClosure::adjust_pointer(T* p) { T heap_oop = RawAccess<>::oop_load(p); if (CompressedOops::is_null(heap_oop)) { diff --git a/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp b/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp index e5c323a06e4..2c0822cd5c9 100644 --- a/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp +++ b/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp @@ -33,6 +33,7 @@ #include "gc/g1/g1StringDedup.hpp" #include "gc/g1/g1Trace.hpp" #include "gc/g1/g1YoungGCEvacFailureInjector.inline.hpp" +#include "gc/shared/continuationGCSupport.inline.hpp" #include "gc/shared/partialArrayTaskStepper.inline.hpp" #include "gc/shared/preservedMarks.inline.hpp" #include "gc/shared/stringdedup/stringDedup.hpp" @@ -526,6 +527,8 @@ oop G1ParScanThreadState::do_copy_to_survivor_space(G1HeapRegionAttr const regio return obj; } + ContinuationGCSupport::transform_stack_chunk(obj); + // Check for deduplicating young Strings. if (G1StringDedup::is_candidate_from_evacuation(klass, region_attr, @@ -630,6 +633,9 @@ oop G1ParScanThreadState::handle_evacuation_failure_par(oop old, markWord m, siz } _preserved_marks->push_if_necessary(old, m); + + ContinuationGCSupport::transform_stack_chunk(old); + _evacuation_failed_info.register_copy_failure(word_sz); // For iterating objects that failed evacuation currently we can reuse the diff --git a/src/hotspot/share/gc/g1/g1Policy.cpp b/src/hotspot/share/gc/g1/g1Policy.cpp index 6de90942ff5..445a2794d6b 100644 --- a/src/hotspot/share/gc/g1/g1Policy.cpp +++ b/src/hotspot/share/gc/g1/g1Policy.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2001, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -1080,10 +1080,10 @@ void G1Policy::decide_on_concurrent_start_pause() { initiate_conc_mark(); log_debug(gc, ergo)("Initiate concurrent cycle (concurrent cycle initiation requested)"); } else if (_g1h->is_user_requested_concurrent_full_gc(cause) || + (cause == GCCause::_codecache_GC_threshold) || (cause == GCCause::_wb_breakpoint)) { - // Initiate a user requested concurrent start or run to a breakpoint. - // A concurrent start must be young only GC, so the collector state - // must be updated to reflect this. + // Initiate a concurrent start. A concurrent start must be a young only + // GC, so the collector state must be updated to reflect this. collector_state()->set_in_young_only_phase(true); collector_state()->set_in_young_gc_before_mixed(false); diff --git a/src/hotspot/share/gc/parallel/parallelScavengeHeap.cpp b/src/hotspot/share/gc/parallel/parallelScavengeHeap.cpp index e47730d47e1..2f701b2ed08 100644 --- a/src/hotspot/share/gc/parallel/parallelScavengeHeap.cpp +++ b/src/hotspot/share/gc/parallel/parallelScavengeHeap.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2001, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -231,6 +231,10 @@ bool ParallelScavengeHeap::is_in_reserved(const void* p) const { return young_gen()->is_in_reserved(p) || old_gen()->is_in_reserved(p); } +bool ParallelScavengeHeap::requires_barriers(stackChunkOop p) const { + return !is_in_young(p); +} + // There are two levels of allocation policy here. // // When an allocation request fails, the requesting thread must invoke a VM diff --git a/src/hotspot/share/gc/parallel/parallelScavengeHeap.hpp b/src/hotspot/share/gc/parallel/parallelScavengeHeap.hpp index 3ee16e5d8b9..cc0d65040bb 100644 --- a/src/hotspot/share/gc/parallel/parallelScavengeHeap.hpp +++ b/src/hotspot/share/gc/parallel/parallelScavengeHeap.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2001, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -186,6 +186,8 @@ class ParallelScavengeHeap : public CollectedHeap { bool is_in_young(const oop p) const; + virtual bool requires_barriers(stackChunkOop obj) const; + MemRegion reserved_region() const { return _reserved; } HeapWord* base() const { return _reserved.start(); } diff --git a/src/hotspot/share/gc/parallel/psParallelCompact.cpp b/src/hotspot/share/gc/parallel/psParallelCompact.cpp index 22b5bc41f34..4917c4e92a0 100644 --- a/src/hotspot/share/gc/parallel/psParallelCompact.cpp +++ b/src/hotspot/share/gc/parallel/psParallelCompact.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2005, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2005, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -75,6 +75,7 @@ #include "oops/objArrayKlass.inline.hpp" #include "oops/oop.inline.hpp" #include "runtime/atomic.hpp" +#include "runtime/continuation.hpp" #include "runtime/handles.inline.hpp" #include "runtime/java.hpp" #include "runtime/safepoint.hpp" @@ -960,6 +961,9 @@ void PSParallelCompact::pre_compact() // Increment the invocation count heap->increment_total_collections(true); + Continuations::on_gc_marking_cycle_start(); + Continuations::arm_all_nmethods(); + // We need to track unique mark sweep invocations as well. _total_invocations++; @@ -990,6 +994,9 @@ void PSParallelCompact::post_compact() GCTraceTime(Info, gc, phases) tm("Post Compact", &_gc_timer); ParCompactionManager::remove_all_shadow_regions(); + Continuations::on_gc_marking_cycle_finish(); + Continuations::arm_all_nmethods(); + for (unsigned int id = old_space_id; id < last_space_id; ++id) { // Clear the marking bitmap, summary data and split info. clear_data_covering_space(SpaceId(id)); @@ -1914,7 +1921,7 @@ public: ParCompactionManager* cm = ParCompactionManager::gc_thread_compaction_manager(_worker_id); PCMarkAndPushClosure mark_and_push_closure(cm); - MarkingCodeBlobClosure mark_and_push_in_blobs(&mark_and_push_closure, !CodeBlobToOopClosure::FixRelocations); + MarkingCodeBlobClosure mark_and_push_in_blobs(&mark_and_push_closure, !CodeBlobToOopClosure::FixRelocations, true /* keepalive nmethods */); thread->oops_do(&mark_and_push_closure, &mark_and_push_in_blobs); diff --git a/src/hotspot/share/gc/parallel/psParallelCompact.inline.hpp b/src/hotspot/share/gc/parallel/psParallelCompact.inline.hpp index 48c5a98fd1a..8f5b50f0405 100644 --- a/src/hotspot/share/gc/parallel/psParallelCompact.inline.hpp +++ b/src/hotspot/share/gc/parallel/psParallelCompact.inline.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2016, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -30,6 +30,7 @@ #include "gc/parallel/parallelScavengeHeap.hpp" #include "gc/parallel/parMarkBitMap.inline.hpp" #include "gc/shared/collectedHeap.hpp" +#include "gc/shared/continuationGCSupport.inline.hpp" #include "oops/access.inline.hpp" #include "oops/compressedOops.inline.hpp" #include "oops/klass.hpp" @@ -100,6 +101,7 @@ inline bool PSParallelCompact::mark_obj(oop obj) { const size_t obj_size = obj->size(); if (mark_bitmap()->mark_obj(obj, obj_size)) { _summary_data.add_obj(obj, obj_size); + ContinuationGCSupport::transform_stack_chunk(obj); return true; } else { return false; diff --git a/src/hotspot/share/gc/parallel/psPromotionManager.cpp b/src/hotspot/share/gc/parallel/psPromotionManager.cpp index 652342ae31f..c600361db46 100644 --- a/src/hotspot/share/gc/parallel/psPromotionManager.cpp +++ b/src/hotspot/share/gc/parallel/psPromotionManager.cpp @@ -29,6 +29,7 @@ #include "gc/parallel/psOldGen.hpp" #include "gc/parallel/psPromotionManager.inline.hpp" #include "gc/parallel/psScavenge.inline.hpp" +#include "gc/shared/continuationGCSupport.inline.hpp" #include "gc/shared/gcTrace.hpp" #include "gc/shared/preservedMarks.inline.hpp" #include "gc/shared/taskqueue.inline.hpp" @@ -336,6 +337,8 @@ oop PSPromotionManager::oop_promotion_failed(oop obj, markWord obj_mark) { _promotion_failed_info.register_copy_failure(obj->size()); + ContinuationGCSupport::transform_stack_chunk(obj); + push_contents(obj); // Save the markWord of promotion-failed objs in _preserved_marks for later diff --git a/src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp b/src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp index 77247a4bdf6..e016154a8e1 100644 --- a/src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp +++ b/src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp @@ -33,6 +33,7 @@ #include "gc/parallel/psPromotionLAB.inline.hpp" #include "gc/parallel/psScavenge.inline.hpp" #include "gc/parallel/psStringDedup.hpp" +#include "gc/shared/continuationGCSupport.inline.hpp" #include "gc/shared/taskqueue.inline.hpp" #include "gc/shared/tlab_globals.hpp" #include "logging/log.hpp" @@ -41,6 +42,7 @@ #include "oops/oop.inline.hpp" #include "runtime/orderAccess.hpp" #include "runtime/prefetch.inline.hpp" +#include "utilities/copy.hpp" inline PSPromotionManager* PSPromotionManager::manager_array(uint index) { assert(_manager_array != NULL, "access of NULL manager_array"); @@ -245,6 +247,10 @@ inline oop PSPromotionManager::copy_unmarked_to_survivor_space(oop o, // Copy obj Copy::aligned_disjoint_words(cast_from_oop(o), cast_from_oop(new_obj), new_obj_size); + // Parallel GC claims with a release - so other threads might access this object + // after claiming and they should see the "completed" object. + ContinuationGCSupport::transform_stack_chunk(new_obj); + // Now we have to CAS in the header. // Make copy visible to threads reading the forwardee. oop forwardee = o->forward_to_atomic(new_obj, test_mark, memory_order_release); diff --git a/src/hotspot/share/gc/parallel/psScavenge.cpp b/src/hotspot/share/gc/parallel/psScavenge.cpp index 049bacaf1a5..1bf4d7d0c52 100644 --- a/src/hotspot/share/gc/parallel/psScavenge.cpp +++ b/src/hotspot/share/gc/parallel/psScavenge.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2002, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2002, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -99,7 +99,7 @@ static void scavenge_roots_work(ParallelRootType::Value root_type, uint worker_i case ParallelRootType::code_cache: { - MarkingCodeBlobClosure code_closure(&roots_to_old_closure, CodeBlobToOopClosure::FixRelocations); + MarkingCodeBlobClosure code_closure(&roots_to_old_closure, CodeBlobToOopClosure::FixRelocations, true /* keepalive nmethods */); ScavengableNMethods::nmethods_do(&code_closure); } break; @@ -269,7 +269,7 @@ public: PSPromotionManager* pm = PSPromotionManager::gc_thread_promotion_manager(_worker_id); PSScavengeRootsClosure roots_closure(pm); - MarkingCodeBlobClosure roots_in_blobs(&roots_closure, CodeBlobToOopClosure::FixRelocations); + MarkingCodeBlobClosure roots_in_blobs(&roots_closure, CodeBlobToOopClosure::FixRelocations, true /* keepalive nmethods */); thread->oops_do(&roots_closure, &roots_in_blobs); diff --git a/src/hotspot/share/gc/serial/defNewGeneration.cpp b/src/hotspot/share/gc/serial/defNewGeneration.cpp index faec1096cfd..48067f7099d 100644 --- a/src/hotspot/share/gc/serial/defNewGeneration.cpp +++ b/src/hotspot/share/gc/serial/defNewGeneration.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2001, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -32,6 +32,7 @@ #include "gc/shared/ageTable.inline.hpp" #include "gc/shared/cardTableRS.hpp" #include "gc/shared/collectorCounters.hpp" +#include "gc/shared/continuationGCSupport.inline.hpp" #include "gc/shared/gcArguments.hpp" #include "gc/shared/gcHeapSummary.hpp" #include "gc/shared/gcLocker.hpp" @@ -696,6 +697,9 @@ void DefNewGeneration::handle_promotion_failure(oop old) { _promotion_failed = true; _promotion_failed_info.register_copy_failure(old->size()); _preserved_marks_set.get()->push_if_necessary(old, old->mark()); + + ContinuationGCSupport::transform_stack_chunk(old); + // forward to self old->forward_to(old); @@ -737,6 +741,8 @@ oop DefNewGeneration::copy_to_survivor_space(oop old) { // Copy obj Copy::aligned_disjoint_words(cast_from_oop(old), cast_from_oop(obj), s); + ContinuationGCSupport::transform_stack_chunk(obj); + // Increment age if obj still in new generation obj->incr_age(); age_table()->add(obj, s); diff --git a/src/hotspot/share/gc/serial/markSweep.hpp b/src/hotspot/share/gc/serial/markSweep.hpp index 03d9add1045..17e4692ce73 100644 --- a/src/hotspot/share/gc/serial/markSweep.hpp +++ b/src/hotspot/share/gc/serial/markSweep.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -38,6 +38,8 @@ class ReferenceProcessor; class DataLayout; +class Method; +class nmethod; class SerialOldTracer; class STWGCTimer; @@ -181,6 +183,8 @@ public: virtual bool do_metadata() { return true; } virtual void do_klass(Klass* k); virtual void do_cld(ClassLoaderData* cld); + virtual void do_method(Method* m); + virtual void do_nmethod(nmethod* nm); void set_ref_discoverer(ReferenceDiscoverer* rd) { set_ref_discoverer_internal(rd); diff --git a/src/hotspot/share/gc/serial/markSweep.inline.hpp b/src/hotspot/share/gc/serial/markSweep.inline.hpp index 27fb8bb2f29..71a61c7263b 100644 --- a/src/hotspot/share/gc/serial/markSweep.inline.hpp +++ b/src/hotspot/share/gc/serial/markSweep.inline.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2000, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -29,11 +29,14 @@ #include "classfile/classLoaderData.inline.hpp" #include "classfile/javaClasses.inline.hpp" +#include "code/nmethod.hpp" +#include "gc/shared/continuationGCSupport.inline.hpp" #include "gc/serial/serialStringDedup.hpp" #include "memory/universe.hpp" #include "oops/markWord.hpp" #include "oops/access.inline.hpp" #include "oops/compressedOops.inline.hpp" +#include "oops/method.hpp" #include "oops/oop.inline.hpp" #include "utilities/align.hpp" #include "utilities/stack.inline.hpp" @@ -50,6 +53,8 @@ inline void MarkSweep::mark_object(oop obj) { markWord mark = obj->mark(); obj->set_mark(markWord::prototype().set_marked()); + ContinuationGCSupport::transform_stack_chunk(obj); + if (obj->mark_must_be_preserved(mark)) { preserve_mark(obj, mark); } @@ -81,6 +86,8 @@ inline void MarkAndPushClosure::do_oop(oop* p) { do_oop_work(p); } inline void MarkAndPushClosure::do_oop(narrowOop* p) { do_oop_work(p); } inline void MarkAndPushClosure::do_klass(Klass* k) { MarkSweep::follow_klass(k); } inline void MarkAndPushClosure::do_cld(ClassLoaderData* cld) { MarkSweep::follow_cld(cld); } +inline void MarkAndPushClosure::do_method(Method* m) { m->record_gc_epoch(); } +inline void MarkAndPushClosure::do_nmethod(nmethod* nm) { nm->follow_nmethod(this); } template inline void MarkSweep::adjust_pointer(T* p) { T heap_oop = RawAccess<>::oop_load(p); diff --git a/src/hotspot/share/gc/serial/serialHeap.cpp b/src/hotspot/share/gc/serial/serialHeap.cpp index 0c506cd8d59..3089a76d8ad 100644 --- a/src/hotspot/share/gc/serial/serialHeap.cpp +++ b/src/hotspot/share/gc/serial/serialHeap.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -94,7 +94,7 @@ GrowableArray SerialHeap::memory_pools() { void SerialHeap::young_process_roots(OopIterateClosure* root_closure, OopIterateClosure* old_gen_closure, CLDClosure* cld_closure) { - MarkingCodeBlobClosure mark_code_closure(root_closure, CodeBlobToOopClosure::FixRelocations); + MarkingCodeBlobClosure mark_code_closure(root_closure, CodeBlobToOopClosure::FixRelocations, false /* keepalive nmethods */); process_roots(SO_ScavengeCodeCache, root_closure, cld_closure, cld_closure, &mark_code_closure); diff --git a/src/hotspot/share/gc/shared/barrierSet.cpp b/src/hotspot/share/gc/shared/barrierSet.cpp index 614011827f4..c7e1e9659a0 100644 --- a/src/hotspot/share/gc/shared/barrierSet.cpp +++ b/src/hotspot/share/gc/shared/barrierSet.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -25,6 +25,8 @@ #include "precompiled.hpp" #include "gc/shared/barrierSet.hpp" #include "gc/shared/barrierSetAssembler.hpp" +#include "gc/shared/barrierSetNMethod.hpp" +#include "runtime/continuation.hpp" #include "runtime/thread.hpp" #include "utilities/debug.hpp" #include "utilities/macros.hpp" @@ -49,6 +51,38 @@ void BarrierSet::set_barrier_set(BarrierSet* barrier_set) { _barrier_set->on_thread_create(Thread::current()); } +static BarrierSetNMethod* select_barrier_set_nmethod(BarrierSetNMethod* barrier_set_nmethod) { + if (barrier_set_nmethod != NULL) { + // The GC needs nmethod entry barriers to do concurrent GC + return barrier_set_nmethod; + } else if (Continuations::enabled()) { + // The GC needs nmethod entry barriers to deal with continuations + return new BarrierSetNMethod(); + } else { + // The GC does not need nmethod entry barriers + return NULL; + } +} + +BarrierSet::BarrierSet(BarrierSetAssembler* barrier_set_assembler, + BarrierSetC1* barrier_set_c1, + BarrierSetC2* barrier_set_c2, + BarrierSetNMethod* barrier_set_nmethod, + const FakeRtti& fake_rtti) : + _fake_rtti(fake_rtti), + _barrier_set_assembler(barrier_set_assembler), + _barrier_set_c1(barrier_set_c1), + _barrier_set_c2(barrier_set_c2), + _barrier_set_nmethod(select_barrier_set_nmethod(barrier_set_nmethod)) { +} + +void BarrierSet::on_thread_attach(Thread* thread) { + if (Continuations::enabled()) { + BarrierSetNMethod* bs_nm = barrier_set_nmethod(); + thread->set_nmethod_disarm_value(bs_nm->disarmed_value()); + } +} + // Called from init.cpp void gc_barrier_stubs_init() { BarrierSet* bs = BarrierSet::barrier_set(); diff --git a/src/hotspot/share/gc/shared/barrierSet.hpp b/src/hotspot/share/gc/shared/barrierSet.hpp index 51f2bfb5409..d9561ba20fc 100644 --- a/src/hotspot/share/gc/shared/barrierSet.hpp +++ b/src/hotspot/share/gc/shared/barrierSet.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2000, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -98,12 +98,7 @@ protected: BarrierSetC1* barrier_set_c1, BarrierSetC2* barrier_set_c2, BarrierSetNMethod* barrier_set_nmethod, - const FakeRtti& fake_rtti) : - _fake_rtti(fake_rtti), - _barrier_set_assembler(barrier_set_assembler), - _barrier_set_c1(barrier_set_c1), - _barrier_set_c2(barrier_set_c2), - _barrier_set_nmethod(barrier_set_nmethod) {} + const FakeRtti& fake_rtti); ~BarrierSet() { } template @@ -139,7 +134,7 @@ public: // caller. That locking ensures the operation is "atomic" with the list // modification wrto operations that hold the NJTList_lock and either also // hold the Threads_lock or are at a safepoint. - virtual void on_thread_attach(Thread* thread) {} + virtual void on_thread_attach(Thread* thread); virtual void on_thread_detach(Thread* thread) {} virtual void make_parsable(JavaThread* thread) {} diff --git a/src/hotspot/share/gc/shared/barrierSetNMethod.cpp b/src/hotspot/share/gc/shared/barrierSetNMethod.cpp index efe2ff7b9e3..d0af04b8a61 100644 --- a/src/hotspot/share/gc/shared/barrierSetNMethod.cpp +++ b/src/hotspot/share/gc/shared/barrierSetNMethod.cpp @@ -26,13 +26,25 @@ #include "code/codeCache.hpp" #include "code/nmethod.hpp" #include "gc/shared/barrierSet.hpp" +#include "gc/shared/barrierSetAssembler.hpp" #include "gc/shared/barrierSetNMethod.hpp" #include "logging/log.hpp" +#include "memory/iterator.hpp" +#include "oops/access.inline.hpp" +#include "oops/method.hpp" #include "runtime/frame.inline.hpp" #include "runtime/thread.hpp" #include "runtime/threadWXSetters.inline.hpp" #include "utilities/debug.hpp" +class LoadPhantomOopClosure : public OopClosure { +public: + virtual void do_oop(oop* p) { + NativeAccess::oop_load(p); + } + virtual void do_oop(narrowOop* p) { ShouldNotReachHere(); } +}; + int BarrierSetNMethod::disarmed_value() const { return *disarmed_value_address(); } @@ -42,6 +54,10 @@ bool BarrierSetNMethod::supports_entry_barrier(nmethod* nm) { return false; } + if (nm->method()->is_continuation_enter_intrinsic()) { + return false; + } + if (!nm->is_native_method() && !nm->is_compiled_by_c2() && !nm->is_compiled_by_c1()) { return false; } @@ -49,6 +65,57 @@ bool BarrierSetNMethod::supports_entry_barrier(nmethod* nm) { return true; } +bool BarrierSetNMethod::nmethod_entry_barrier(nmethod* nm) { + // If the nmethod is the only thing pointing to the oops, and we are using a + // SATB GC, then it is important that this code marks them live. This is done + // by the phantom load. + LoadPhantomOopClosure cl; + nm->oops_do(&cl); + + // CodeCache sweeper support + nm->mark_as_maybe_on_continuation(); + + disarm(nm); + + return true; +} + +int* BarrierSetNMethod::disarmed_value_address() const { + return (int*) &_current_phase; +} + +ByteSize BarrierSetNMethod::thread_disarmed_offset() const { + return Thread::nmethod_disarmed_offset(); +} + +class BarrierSetNMethodArmClosure : public ThreadClosure { +private: + int _disarm_value; + +public: + BarrierSetNMethodArmClosure(int disarm_value) : + _disarm_value(disarm_value) {} + + virtual void do_thread(Thread* thread) { + thread->set_nmethod_disarm_value(_disarm_value); + } +}; + +void BarrierSetNMethod::arm_all_nmethods() { + // Change to a new global GC phase. Doing this requires changing the thread-local + // disarm value for all threads, to reflect the new GC phase. + ++_current_phase; + if (_current_phase == 4) { + _current_phase = 1; + } + BarrierSetNMethodArmClosure cl(_current_phase); + Threads::threads_do(&cl); + + // We clear the patching epoch when disarming nmethods, so that + // the counter won't overflow. + AARCH64_PORT_ONLY(BarrierSetAssembler::clear_patching_epoch()); +} + int BarrierSetNMethod::nmethod_stub_entry_barrier(address* return_address_ptr) { // Enable WXWrite: the function is called directly from nmethod_entry_barrier // stub. diff --git a/src/hotspot/share/gc/shared/barrierSetNMethod.hpp b/src/hotspot/share/gc/shared/barrierSetNMethod.hpp index 572bbc2b216..81099d66fea 100644 --- a/src/hotspot/share/gc/shared/barrierSetNMethod.hpp +++ b/src/hotspot/share/gc/shared/barrierSetNMethod.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -32,20 +32,27 @@ class nmethod; class BarrierSetNMethod: public CHeapObj { +private: + int _current_phase; void deoptimize(nmethod* nm, address* return_addr_ptr); - int disarmed_value() const; public: + BarrierSetNMethod() : _current_phase(1) {} bool supports_entry_barrier(nmethod* nm); - virtual bool nmethod_entry_barrier(nmethod* nm) = 0; - virtual ByteSize thread_disarmed_offset() const = 0; - virtual int* disarmed_value_address() const = 0; + virtual bool nmethod_entry_barrier(nmethod* nm); + virtual ByteSize thread_disarmed_offset() const; + virtual int* disarmed_value_address() const; + + int disarmed_value() const; static int nmethod_stub_entry_barrier(address* return_address_ptr); bool nmethod_osr_entry_barrier(nmethod* nm); bool is_armed(nmethod* nm); void disarm(nmethod* nm); + void arm(nmethod* nm, int arm_value); + + void arm_all_nmethods(); }; diff --git a/src/hotspot/share/gc/shared/collectedHeap.cpp b/src/hotspot/share/gc/shared/collectedHeap.cpp index bf3447b3b56..0a3c6411ad7 100644 --- a/src/hotspot/share/gc/shared/collectedHeap.cpp +++ b/src/hotspot/share/gc/shared/collectedHeap.cpp @@ -62,6 +62,7 @@ class ClassLoaderData; Klass* CollectedHeap::_filler_object_klass = NULL; size_t CollectedHeap::_filler_array_max_size = 0; +size_t CollectedHeap::_stack_chunk_max_size = 0; class GCMessage : public FormatBuffer<1024> { public: @@ -279,6 +280,7 @@ void CollectedHeap::collect_as_vm_thread(GCCause::Cause cause) { assert(Heap_lock->is_locked(), "Precondition#2"); GCCauseSetter gcs(this, cause); switch (cause) { + case GCCause::_codecache_GC_threshold: case GCCause::_heap_inspection: case GCCause::_heap_dump: case GCCause::_metadata_GC_threshold : { diff --git a/src/hotspot/share/gc/shared/collectedHeap.hpp b/src/hotspot/share/gc/shared/collectedHeap.hpp index 853ee8b74e1..012e75acfc2 100644 --- a/src/hotspot/share/gc/shared/collectedHeap.hpp +++ b/src/hotspot/share/gc/shared/collectedHeap.hpp @@ -117,6 +117,8 @@ class CollectedHeap : public CHeapObj { // Used for filler objects (static, but initialized in ctor). static size_t _filler_array_max_size; + static size_t _stack_chunk_max_size; // 0 for no limit + // Last time the whole heap has been examined in support of RMI // MaxObjectInspectionAge. // This timestamp must be monotonically non-decreasing to avoid @@ -208,6 +210,10 @@ class CollectedHeap : public CHeapObj { return _filler_array_max_size; } + static inline size_t stack_chunk_max_size() { + return _stack_chunk_max_size; + } + static inline Klass* filler_object_klass() { return _filler_object_klass; } @@ -385,6 +391,15 @@ class CollectedHeap : public CHeapObj { size_t size, Metaspace::MetadataType mdtype); + // Return true, if accesses to the object would require barriers. + // This is used by continuations to copy chunks of a thread stack into StackChunk object or out of a StackChunk + // object back into the thread stack. These chunks may contain references to objects. It is crucial that + // the GC does not attempt to traverse the object while we modify it, because its structure (oopmap) is changed + // when stack chunks are stored into it. + // StackChunk objects may be reused, the GC must not assume that a StackChunk object is always a freshly + // allocated object. + virtual bool requires_barriers(stackChunkOop obj) const = 0; + // Returns "true" iff there is a stop-world GC in progress. (I assume // that it should answer "false" for the concurrent part of a concurrent // collector -- dld). diff --git a/src/hotspot/share/gc/shared/continuationGCSupport.hpp b/src/hotspot/share/gc/shared/continuationGCSupport.hpp new file mode 100644 index 00000000000..dda983e0bd8 --- /dev/null +++ b/src/hotspot/share/gc/shared/continuationGCSupport.hpp @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_GC_SHARED_CONTINUATIONGCSUPPORT_HPP +#define SHARE_GC_SHARED_CONTINUATIONGCSUPPORT_HPP + +#include "memory/allStatic.hpp" +#include "oops/oopsHierarchy.hpp" + +class ContinuationGCSupport : public AllStatic { +public: + // Relativize the given oop if it is a stack chunk. + static bool relativize_stack_chunk(oop obj); + // Relativize and transform to use a bitmap for future oop iteration for the + // given oop if it is a stack chunk. + static void transform_stack_chunk(oop obj); +}; + +#endif // SHARE_GC_SHARED_CONTINUATIONGCSUPPORT_HPP diff --git a/src/hotspot/share/gc/shared/continuationGCSupport.inline.hpp b/src/hotspot/share/gc/shared/continuationGCSupport.inline.hpp new file mode 100644 index 00000000000..72a828b7396 --- /dev/null +++ b/src/hotspot/share/gc/shared/continuationGCSupport.inline.hpp @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_GC_SHARED_CONTINUATIONGCSUPPORT_INLINE_HPP +#define SHARE_GC_SHARED_CONTINUATIONGCSUPPORT_INLINE_HPP + +#include "gc/shared/continuationGCSupport.hpp" + +#include "oops/instanceStackChunkKlass.hpp" +#include "oops/oop.inline.hpp" +#include "oops/stackChunkOop.inline.hpp" + +inline bool ContinuationGCSupport::relativize_stack_chunk(oop obj) { + if (!obj->is_stackChunk()) { + return false; + } + + stackChunkOop chunk = stackChunkOopDesc::cast(obj); + chunk->relativize_derived_pointers_concurrently(); + + return true; +} + +inline void ContinuationGCSupport::transform_stack_chunk(oop obj) { + if (!obj->is_stackChunk()) { + return; + } + + stackChunkOop chunk = stackChunkOopDesc::cast(obj); + if (!chunk->is_gc_mode()) { + chunk->transform(); + } +} + +#endif // SHARE_GC_SHARED_CONTINUATIONGCSUPPORT_INLINE_HPP diff --git a/src/hotspot/share/gc/shared/copyFailedInfo.hpp b/src/hotspot/share/gc/shared/copyFailedInfo.hpp index 675f34af65f..f4aeac16c34 100644 --- a/src/hotspot/share/gc/shared/copyFailedInfo.hpp +++ b/src/hotspot/share/gc/shared/copyFailedInfo.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2012, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -71,7 +71,7 @@ class PromotionFailedInfo : public CopyFailedInfo { void register_copy_failure(size_t size) { CopyFailedInfo::register_copy_failure(size); - _thread_trace_id = JFR_THREAD_ID(Thread::current()); + _thread_trace_id = JFR_JVM_THREAD_ID(Thread::current()); } void reset() { diff --git a/src/hotspot/share/gc/shared/gcCause.cpp b/src/hotspot/share/gc/shared/gcCause.cpp index 1d2c70b1f4d..939329dd31c 100644 --- a/src/hotspot/share/gc/shared/gcCause.cpp +++ b/src/hotspot/share/gc/shared/gcCause.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2002, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2002, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -75,6 +75,9 @@ const char* GCCause::to_string(GCCause::Cause cause) { case _tenured_generation_full: return "Tenured Generation Full"; + case _codecache_GC_threshold: + return "CodeCache GC Threshold"; + case _metadata_GC_threshold: return "Metadata GC Threshold"; diff --git a/src/hotspot/share/gc/shared/gcCause.hpp b/src/hotspot/share/gc/shared/gcCause.hpp index e9804547026..ec43b881b3e 100644 --- a/src/hotspot/share/gc/shared/gcCause.hpp +++ b/src/hotspot/share/gc/shared/gcCause.hpp @@ -64,6 +64,7 @@ class GCCause : public AllStatic { /* implementation specific */ _tenured_generation_full, + _codecache_GC_threshold, _metadata_GC_threshold, _metadata_GC_clear_soft_refs, diff --git a/src/hotspot/share/gc/shared/genCollectedHeap.cpp b/src/hotspot/share/gc/shared/genCollectedHeap.cpp index e7f7ef5dc99..fd71e75db2f 100644 --- a/src/hotspot/share/gc/shared/genCollectedHeap.cpp +++ b/src/hotspot/share/gc/shared/genCollectedHeap.cpp @@ -36,6 +36,7 @@ #include "gc/shared/cardTableRS.hpp" #include "gc/shared/collectedHeap.inline.hpp" #include "gc/shared/collectorCounters.hpp" +#include "gc/shared/continuationGCSupport.inline.hpp" #include "gc/shared/gcId.hpp" #include "gc/shared/gcLocker.hpp" #include "gc/shared/gcPolicyCounters.hpp" @@ -62,6 +63,7 @@ #include "memory/resourceArea.hpp" #include "memory/universe.hpp" #include "oops/oop.inline.hpp" +#include "runtime/continuation.hpp" #include "runtime/handles.hpp" #include "runtime/handles.inline.hpp" #include "runtime/java.hpp" @@ -606,6 +608,9 @@ void GenCollectedHeap::do_collection(bool full, increment_total_full_collections(); } + Continuations::on_gc_marking_cycle_start(); + Continuations::arm_all_nmethods(); + collect_generation(_old_gen, full, size, @@ -613,6 +618,9 @@ void GenCollectedHeap::do_collection(bool full, run_verification && VerifyGCLevel <= 1, do_clear_all_soft_refs); + Continuations::on_gc_marking_cycle_finish(); + Continuations::arm_all_nmethods(); + // Adjust generation sizes. _old_gen->compute_new_size(); _young_gen->compute_new_size(); @@ -790,7 +798,10 @@ void GenCollectedHeap::full_process_roots(bool is_adjust_phase, bool only_strong_roots, OopClosure* root_closure, CLDClosure* cld_closure) { - MarkingCodeBlobClosure mark_code_closure(root_closure, is_adjust_phase); + // Called from either the marking phase or the adjust phase. + const bool is_marking_phase = !is_adjust_phase; + + MarkingCodeBlobClosure mark_code_closure(root_closure, is_adjust_phase, is_marking_phase); CLDClosure* weak_cld_closure = only_strong_roots ? NULL : cld_closure; process_roots(so, root_closure, cld_closure, weak_cld_closure, &mark_code_closure); @@ -888,13 +899,17 @@ void GenCollectedHeap::do_full_collection(bool clear_all_soft_refs, } } -bool GenCollectedHeap::is_in_young(oop p) { +bool GenCollectedHeap::is_in_young(oop p) const { bool result = cast_from_oop(p) < _old_gen->reserved().start(); assert(result == _young_gen->is_in_reserved(p), "incorrect test - result=%d, p=" INTPTR_FORMAT, result, p2i((void*)p)); return result; } +bool GenCollectedHeap::requires_barriers(stackChunkOop obj) const { + return !is_in_young(obj); +} + // Returns "TRUE" iff "p" points into the committed areas of the heap. bool GenCollectedHeap::is_in(const void* p) const { return _young_gen->is_in(p) || _old_gen->is_in(p); @@ -1214,18 +1229,3 @@ void GenCollectedHeap::ensure_parsability(bool retire_tlabs) { GenEnsureParsabilityClosure ep_cl; generation_iterate(&ep_cl, false); } - -oop GenCollectedHeap::handle_failed_promotion(Generation* old_gen, - oop obj, - size_t obj_size) { - guarantee(old_gen == _old_gen, "We only get here with an old generation"); - assert(obj_size == obj->size(), "bad obj_size passed in"); - HeapWord* result = NULL; - - result = old_gen->expand_and_allocate(obj_size, false); - - if (result != NULL) { - Copy::aligned_disjoint_words(cast_from_oop(obj), result, obj_size); - } - return cast_to_oop(result); -} diff --git a/src/hotspot/share/gc/shared/genCollectedHeap.hpp b/src/hotspot/share/gc/shared/genCollectedHeap.hpp index 39012fa3cb1..3b4e26837d0 100644 --- a/src/hotspot/share/gc/shared/genCollectedHeap.hpp +++ b/src/hotspot/share/gc/shared/genCollectedHeap.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2000, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -205,7 +205,9 @@ public: // Returns true if the reference is to an object in the reserved space // for the young generation. // Assumes the the young gen address range is less than that of the old gen. - bool is_in_young(oop p); + bool is_in_young(oop p) const; + + virtual bool requires_barriers(stackChunkOop obj) const; #ifdef ASSERT bool is_in_partial_collection(const void* p); diff --git a/src/hotspot/share/gc/shared/genOopClosures.hpp b/src/hotspot/share/gc/shared/genOopClosures.hpp index e1e8104dfa5..05601199118 100644 --- a/src/hotspot/share/gc/shared/genOopClosures.hpp +++ b/src/hotspot/share/gc/shared/genOopClosures.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2001, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -33,6 +33,8 @@ class CardTableRS; class CardTableBarrierSet; class DefNewGeneration; class KlassRemSet; +class Method; +class nmethod; #if INCLUDE_SERIALGC @@ -116,6 +118,8 @@ class FilteringClosure: public OopIterateClosure { virtual bool do_metadata() { assert(!_cl->do_metadata(), "assumption broken, must change to 'return _cl->do_metadata()'"); return false; } virtual void do_klass(Klass*) { ShouldNotReachHere(); } virtual void do_cld(ClassLoaderData*) { ShouldNotReachHere(); } + virtual void do_method(Method*) { ShouldNotReachHere(); } + virtual void do_nmethod(nmethod*) { ShouldNotReachHere(); } }; #if INCLUDE_SERIALGC diff --git a/src/hotspot/share/gc/shared/generation.cpp b/src/hotspot/share/gc/shared/generation.cpp index a0a57dea115..181f7b164dd 100644 --- a/src/hotspot/share/gc/shared/generation.cpp +++ b/src/hotspot/share/gc/shared/generation.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -164,14 +164,24 @@ oop Generation::promote(oop obj, size_t obj_size) { } #endif // #ifndef PRODUCT + // Allocate new object. HeapWord* result = allocate(obj_size, false); - if (result != NULL) { - Copy::aligned_disjoint_words(cast_from_oop(obj), result, obj_size); - return cast_to_oop(result); - } else { - GenCollectedHeap* gch = GenCollectedHeap::heap(); - return gch->handle_failed_promotion(this, obj, obj_size); + if (result == NULL) { + // Promotion of obj into gen failed. Try to expand and allocate. + result = expand_and_allocate(obj_size, false); + if (result == NULL) { + return NULL; + } } + + // Copy to new location. + Copy::aligned_disjoint_words(cast_from_oop(obj), result, obj_size); + oop new_obj = cast_to_oop(result); + + // Transform object if it is a stack chunk. + ContinuationGCSupport::transform_stack_chunk(new_obj); + + return new_obj; } oop Generation::par_promote(int thread_num, diff --git a/src/hotspot/share/gc/shared/memAllocator.cpp b/src/hotspot/share/gc/shared/memAllocator.cpp index 1aacb030884..4e99cea7422 100644 --- a/src/hotspot/share/gc/shared/memAllocator.cpp +++ b/src/hotspot/share/gc/shared/memAllocator.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -23,7 +23,7 @@ */ #include "precompiled.hpp" -#include "classfile/javaClasses.hpp" +#include "classfile/javaClasses.inline.hpp" #include "gc/shared/allocTracer.hpp" #include "gc/shared/collectedHeap.hpp" #include "gc/shared/memAllocator.hpp" @@ -269,7 +269,7 @@ HeapWord* MemAllocator::allocate_inside_tlab(Allocation& allocation) const { assert(UseTLAB, "should use UseTLAB"); // Try allocating from an existing TLAB. - HeapWord* mem = _thread->tlab().allocate(_word_size); + HeapWord* mem = allocate_inside_tlab_fast(); if (mem != NULL) { return mem; } @@ -278,6 +278,10 @@ HeapWord* MemAllocator::allocate_inside_tlab(Allocation& allocation) const { return allocate_inside_tlab_slow(allocation); } +HeapWord* MemAllocator::allocate_inside_tlab_fast() const { + return _thread->tlab().allocate(_word_size); +} + HeapWord* MemAllocator::allocate_inside_tlab_slow(Allocation& allocation) const { HeapWord* mem = NULL; ThreadLocalAllocBuffer& tlab = _thread->tlab(); @@ -372,6 +376,21 @@ oop MemAllocator::allocate() const { return obj; } +oop MemAllocator::try_allocate_in_existing_tlab() { + oop obj = NULL; + { + HeapWord* mem = allocate_inside_tlab_fast(); + if (mem != NULL) { + obj = initialize(mem); + } else { + // The unhandled oop detector will poison local variable obj, + // so reset it to NULL if mem is NULL. + obj = NULL; + } + } + return obj; +} + void MemAllocator::mem_clear(HeapWord* mem) const { assert(mem != NULL, "cannot initialize NULL object"); const size_t hs = oopDesc::header_size(); @@ -426,3 +445,20 @@ oop ClassAllocator::initialize(HeapWord* mem) const { java_lang_Class::set_oop_size(mem, _word_size); return finish(mem); } + +// Does the minimal amount of initialization needed for a TLAB allocation. +// We don't need to do a full initialization, as such an allocation need not be immediately walkable. +oop StackChunkAllocator::initialize(HeapWord* mem) const { + assert(_stack_size > 0, ""); + assert(_stack_size <= max_jint, ""); + assert(_word_size > _stack_size, ""); + + // zero out fields (but not the stack) + const size_t hs = oopDesc::header_size(); + Copy::fill_to_aligned_words(mem + hs, vmClasses::StackChunk_klass()->size_helper() - hs); + + jdk_internal_vm_StackChunk::set_size(mem, (int)_stack_size); + jdk_internal_vm_StackChunk::set_sp(mem, (int)_stack_size); + + return finish(mem); +} diff --git a/src/hotspot/share/gc/shared/memAllocator.hpp b/src/hotspot/share/gc/shared/memAllocator.hpp index 6df76dca2ff..320331f5ae7 100644 --- a/src/hotspot/share/gc/shared/memAllocator.hpp +++ b/src/hotspot/share/gc/shared/memAllocator.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -45,6 +45,7 @@ protected: private: // Allocate from the current thread's TLAB, with broken-out slow path. HeapWord* allocate_inside_tlab(Allocation& allocation) const; + HeapWord* allocate_inside_tlab_fast() const; HeapWord* allocate_inside_tlab_slow(Allocation& allocation) const; HeapWord* allocate_outside_tlab(Allocation& allocation) const; @@ -72,6 +73,7 @@ protected: public: oop allocate() const; + oop try_allocate_in_existing_tlab(); virtual oop initialize(HeapWord* mem) const = 0; }; @@ -104,4 +106,14 @@ public: virtual oop initialize(HeapWord* mem) const; }; +class StackChunkAllocator : public MemAllocator { + const size_t _stack_size; + +public: + StackChunkAllocator(Klass* klass, size_t word_size, size_t stack_size, Thread* thread = Thread::current()) + : MemAllocator(klass, word_size, thread), + _stack_size(stack_size) {} + virtual oop initialize(HeapWord* mem) const; +}; + #endif // SHARE_GC_SHARED_MEMALLOCATOR_HPP diff --git a/src/hotspot/share/gc/shared/space.cpp b/src/hotspot/share/gc/shared/space.cpp index 67d76173614..230eea7f7de 100644 --- a/src/hotspot/share/gc/shared/space.cpp +++ b/src/hotspot/share/gc/shared/space.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -555,8 +555,12 @@ void CompactibleSpace::compact() { // copy object and reinit its mark assert(cur_obj != compaction_top, "everything in this pass should be moving"); Copy::aligned_conjoint_words(cur_obj, compaction_top, size); - cast_to_oop(compaction_top)->init_mark(); - assert(cast_to_oop(compaction_top)->klass() != NULL, "should have a class"); + oop new_obj = cast_to_oop(compaction_top); + + ContinuationGCSupport::transform_stack_chunk(new_obj); + + new_obj->init_mark(); + assert(new_obj->klass() != NULL, "should have a class"); debug_only(prev_obj = cur_obj); cur_obj += size; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp index 893e8238aaa..da12de337d9 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp @@ -25,6 +25,7 @@ #include "precompiled.hpp" #include "compiler/oopMap.hpp" +#include "gc/shared/continuationGCSupport.hpp" #include "gc/shared/gcTraceTime.inline.hpp" #include "gc/shared/preservedMarks.inline.hpp" #include "gc/shared/tlab_globals.hpp" @@ -740,6 +741,8 @@ public: void do_oop(oop* p) { do_oop_work(p); } void do_oop(narrowOop* p) { do_oop_work(p); } + void do_method(Method* m) {} + void do_nmethod(nmethod* nm) {} }; class ShenandoahAdjustPointersObjectClosure : public ObjectClosure { @@ -840,6 +843,8 @@ public: HeapWord* compact_to = cast_from_oop(p->forwardee()); Copy::aligned_conjoint_words(compact_from, compact_to, size); oop new_obj = cast_to_oop(compact_to); + + ContinuationGCSupport::relativize_stack_chunk(new_obj); new_obj->init_mark(); } } @@ -951,9 +956,8 @@ void ShenandoahFullGC::compact_humongous_objects() { assert(old_start != new_start, "must be real move"); assert(r->is_stw_move_allowed(), "Region " SIZE_FORMAT " should be movable", r->index()); - Copy::aligned_conjoint_words(heap->get_region(old_start)->bottom(), - heap->get_region(new_start)->bottom(), - words_size); + Copy::aligned_conjoint_words(r->bottom(), heap->get_region(new_start)->bottom(), words_size); + ContinuationGCSupport::relativize_stack_chunk(cast_to_oop(r->bottom())); oop new_obj = cast_to_oop(heap->get_region(new_start)->bottom()); new_obj->init_mark(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index f72d31e1f69..ce37e8ab873 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -39,6 +39,7 @@ #include "gc/shenandoah/shenandoahCollectionSet.hpp" #include "gc/shenandoah/shenandoahCollectorPolicy.hpp" #include "gc/shenandoah/shenandoahConcurrentMark.hpp" +#include "gc/shenandoah/shenandoahMarkingContext.inline.hpp" #include "gc/shenandoah/shenandoahControlThread.hpp" #include "gc/shenandoah/shenandoahFreeSet.hpp" #include "gc/shenandoah/shenandoahPhaseTimings.hpp" @@ -2305,3 +2306,12 @@ void ShenandoahHeap::flush_liveness_cache(uint worker_id) { } } } + +bool ShenandoahHeap::requires_barriers(stackChunkOop obj) const { + ShenandoahHeapRegion* region = heap_region_containing(obj); + bool allocated_after_mark_start = marking_context()->allocated_after_mark_start(obj); + bool requires_concmark_barriers = is_concurrent_mark_in_progress() && !allocated_after_mark_start; + bool requires_loadref_barriers = has_forwarded_objects() && cast_from_oop(obj) < heap_region_containing(obj)->get_update_watermark(); + bool requires_deep_loadref_barriers = allocated_after_mark_start && has_forwarded_objects(); + return requires_concmark_barriers || requires_loadref_barriers || requires_deep_loadref_barriers; +} diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp index 1ec9d7051cf..ec65a424d35 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp @@ -470,6 +470,8 @@ public: bool is_in(const void* p) const; + bool requires_barriers(stackChunkOop obj) const; + MemRegion reserved_region() const { return _reserved; } bool is_in_reserved(const void* addr) const { return _reserved.contains(addr); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp index d925f6c1891..b58d97d68a6 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp @@ -30,6 +30,7 @@ #include "classfile/javaClasses.inline.hpp" #include "gc/shared/markBitMap.inline.hpp" #include "gc/shared/threadLocalAllocBuffer.inline.hpp" +#include "gc/shared/continuationGCSupport.inline.hpp" #include "gc/shared/suspendibleThreadSet.hpp" #include "gc/shared/tlab_globals.hpp" #include "gc/shenandoah/shenandoahAsserts.hpp" @@ -334,6 +335,8 @@ inline oop ShenandoahHeap::evacuate_object(oop p, Thread* thread) { // Try to install the new forwarding pointer. oop copy_val = cast_to_oop(copy); + ContinuationGCSupport::relativize_stack_chunk(copy_val); + oop result = ShenandoahForwarding::try_update_forwardee(p, copy_val); if (result == copy_val) { // Successfully evacuated. Our copy is now the public one! diff --git a/src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.inline.hpp index 5677f3daa60..98a8c6522a0 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.inline.hpp @@ -152,7 +152,7 @@ public: // we risk executing that code cache blob, and crashing. template void ShenandoahSTWRootScanner::roots_do(T* oops, uint worker_id) { - MarkingCodeBlobClosure blobs_cl(oops, !CodeBlobToOopClosure::FixRelocations); + MarkingCodeBlobClosure blobs_cl(oops, !CodeBlobToOopClosure::FixRelocations, true /*FIXME*/); CLDToOopClosure clds(oops, ClassLoaderData::_claim_strong); ResourceMark rm; diff --git a/src/hotspot/share/gc/z/zBarrier.cpp b/src/hotspot/share/gc/z/zBarrier.cpp index 4c4af10d27f..469e5fce935 100644 --- a/src/hotspot/share/gc/z/zBarrier.cpp +++ b/src/hotspot/share/gc/z/zBarrier.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2015, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -178,12 +178,14 @@ uintptr_t ZBarrier::keep_alive_barrier_on_oop_slow_path(uintptr_t addr) { } uintptr_t ZBarrier::keep_alive_barrier_on_weak_oop_slow_path(uintptr_t addr) { + assert(ZResurrection::is_blocked(), "This operation is only valid when resurrection is blocked"); const uintptr_t good_addr = weak_load_barrier_on_oop_slow_path(addr); assert(ZHeap::heap()->is_object_strongly_live(good_addr), "Should be live"); return good_addr; } uintptr_t ZBarrier::keep_alive_barrier_on_phantom_oop_slow_path(uintptr_t addr) { + assert(ZResurrection::is_blocked(), "This operation is only valid when resurrection is blocked"); const uintptr_t good_addr = weak_load_barrier_on_oop_slow_path(addr); assert(ZHeap::heap()->is_object_live(good_addr), "Should be live"); return good_addr; diff --git a/src/hotspot/share/gc/z/zBarrier.inline.hpp b/src/hotspot/share/gc/z/zBarrier.inline.hpp index 0f9f8dd2a20..3303e23ccea 100644 --- a/src/hotspot/share/gc/z/zBarrier.inline.hpp +++ b/src/hotspot/share/gc/z/zBarrier.inline.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2015, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -26,11 +26,13 @@ #include "gc/z/zBarrier.hpp" +#include "code/codeCache.hpp" #include "gc/z/zAddress.inline.hpp" #include "gc/z/zOop.inline.hpp" #include "gc/z/zResurrection.inline.hpp" #include "oops/oop.hpp" #include "runtime/atomic.hpp" +#include "runtime/continuation.hpp" // A self heal must always "upgrade" the address metadata bits in // accordance with the metadata bits state machine, which has the @@ -330,22 +332,26 @@ inline bool ZBarrier::is_alive_barrier_on_phantom_oop(oop o) { // Keep alive barrier // inline void ZBarrier::keep_alive_barrier_on_weak_oop_field(volatile oop* p) { - // This operation is only valid when resurrection is blocked. - assert(ZResurrection::is_blocked(), "Invalid phase"); + assert(ZResurrection::is_blocked(), "This operation is only valid when resurrection is blocked"); const oop o = Atomic::load(p); barrier(p, o); } inline void ZBarrier::keep_alive_barrier_on_phantom_oop_field(volatile oop* p) { - // This operation is only valid when resurrection is blocked. - assert(ZResurrection::is_blocked(), "Invalid phase"); + assert(ZResurrection::is_blocked(), "This operation is only valid when resurrection is blocked"); const oop o = Atomic::load(p); barrier(p, o); } inline void ZBarrier::keep_alive_barrier_on_phantom_root_oop_field(oop* p) { - // This operation is only valid when resurrection is blocked. - assert(ZResurrection::is_blocked(), "Invalid phase"); + // The keep alive operation is only valid when resurrection is blocked. + // + // Except with Loom, where we intentionally trigger arms nmethods after + // unlinking, to get a sense of what nmethods are alive. This will trigger + // the keep alive barriers, but the oops are healed and the slow-paths + // will not trigger. We have stronger checks in the slow-paths. + assert(ZResurrection::is_blocked() || (Continuations::enabled() && CodeCache::contains((void*)p)), + "This operation is only valid when resurrection is blocked"); const oop o = *p; root_barrier(p, o); } diff --git a/src/hotspot/share/gc/z/zBarrierSetNMethod.cpp b/src/hotspot/share/gc/z/zBarrierSetNMethod.cpp index 9916178cc2c..0e3557778a8 100644 --- a/src/hotspot/share/gc/z/zBarrierSetNMethod.cpp +++ b/src/hotspot/share/gc/z/zBarrierSetNMethod.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -58,6 +58,10 @@ bool ZBarrierSetNMethod::nmethod_entry_barrier(nmethod* nm) { // Heal oops ZNMethod::nmethod_oops_barrier(nm); + + // CodeCache sweeper support + nm->mark_as_maybe_on_continuation(); + // Disarm disarm(nm); diff --git a/src/hotspot/share/gc/z/zCollectedHeap.cpp b/src/hotspot/share/gc/z/zCollectedHeap.cpp index ea883450da0..bd2bb59714d 100644 --- a/src/hotspot/share/gc/z/zCollectedHeap.cpp +++ b/src/hotspot/share/gc/z/zCollectedHeap.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2015, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -23,6 +23,7 @@ #include "precompiled.hpp" #include "classfile/classLoaderData.hpp" +#include "classfile/javaClasses.hpp" #include "gc/shared/gcHeapSummary.hpp" #include "gc/shared/suspendibleThreadSet.hpp" #include "gc/z/zCollectedHeap.hpp" @@ -122,6 +123,27 @@ bool ZCollectedHeap::is_in(const void* p) const { return _heap.is_in((uintptr_t)p); } +bool ZCollectedHeap::requires_barriers(stackChunkOop obj) const { + uintptr_t* cont_addr = obj->field_addr(jdk_internal_vm_StackChunk::cont_offset()); + + if (!_heap.is_allocating(cast_from_oop(obj))) { + // An object that isn't allocating, is visible from GC tracing. Such + // stack chunks require barriers. + return true; + } + + if (!ZAddress::is_good_or_null(*cont_addr)) { + // If a chunk is allocated after a GC started, but before relocate start + // we can have an allocating chunk that isn't deeply good. That means that + // the contained oops might be bad and require GC barriers. + return true; + } + + // The chunk is allocating and its pointers are good. This chunk needs no + // GC barriers + return false; +} + uint32_t ZCollectedHeap::hash_oop(oop obj) const { return _heap.hash_oop(ZOop::to_address(obj)); } diff --git a/src/hotspot/share/gc/z/zCollectedHeap.hpp b/src/hotspot/share/gc/z/zCollectedHeap.hpp index 12f6902a025..b679a86f635 100644 --- a/src/hotspot/share/gc/z/zCollectedHeap.hpp +++ b/src/hotspot/share/gc/z/zCollectedHeap.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2015, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -72,6 +72,7 @@ public: virtual bool is_maximal_no_gc() const; virtual bool is_in(const void* p) const; + virtual bool requires_barriers(stackChunkOop obj) const; virtual uint32_t hash_oop(oop obj) const; diff --git a/src/hotspot/share/gc/z/zDriver.cpp b/src/hotspot/share/gc/z/zDriver.cpp index 30fdeb6804f..97e8ca21bcc 100644 --- a/src/hotspot/share/gc/z/zDriver.cpp +++ b/src/hotspot/share/gc/z/zDriver.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2015, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -229,6 +229,7 @@ void ZDriver::collect(const ZDriverRequest& request) { case GCCause::_scavenge_alot: case GCCause::_jvmti_force_gc: case GCCause::_metadata_GC_clear_soft_refs: + case GCCause::_codecache_GC_threshold: // Start synchronous GC _gc_cycle_port.send_sync(request); break; diff --git a/src/hotspot/share/gc/z/zHeap.cpp b/src/hotspot/share/gc/z/zHeap.cpp index b937a645f62..36afe843640 100644 --- a/src/hotspot/share/gc/z/zHeap.cpp +++ b/src/hotspot/share/gc/z/zHeap.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2015, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -433,6 +433,11 @@ void ZHeap::relocate() { ZStatHeap::set_at_relocate_end(_page_allocator.stats(), _object_allocator.relocated()); } +bool ZHeap::is_allocating(uintptr_t addr) const { + const ZPage* const page = _page_table.get(addr); + return page->is_allocating(); +} + void ZHeap::object_iterate(ObjectClosure* cl, bool visit_weaks) { assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint"); ZHeapIterator iter(1 /* nworkers */, visit_weaks); diff --git a/src/hotspot/share/gc/z/zHeap.hpp b/src/hotspot/share/gc/z/zHeap.hpp index 723ec52d39c..c30f3a02968 100644 --- a/src/hotspot/share/gc/z/zHeap.hpp +++ b/src/hotspot/share/gc/z/zHeap.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2015, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -139,6 +139,9 @@ public: uintptr_t remap_object(uintptr_t addr); void relocate(); + // Continuations + bool is_allocating(uintptr_t addr) const; + // Iteration void object_iterate(ObjectClosure* cl, bool visit_weaks); ParallelObjectIteratorImpl* parallel_object_iterator(uint nworkers, bool visit_weaks); diff --git a/src/hotspot/share/gc/z/zHeapIterator.cpp b/src/hotspot/share/gc/z/zHeapIterator.cpp index 89ff2680c0b..c7adc7a0a61 100644 --- a/src/hotspot/share/gc/z/zHeapIterator.cpp +++ b/src/hotspot/share/gc/z/zHeapIterator.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -190,6 +190,10 @@ public: NativeAccessClosure cl(_context); cld->oops_do(&cl, ClassLoaderData::_claim_other); } + + // Don't follow loom stack metadata; it's already followed in other ways through CLDs + virtual void do_nmethod(nmethod* nm) {} + virtual void do_method(Method* m) {} }; ZHeapIterator::ZHeapIterator(uint nworkers, bool visit_weaks) : diff --git a/src/hotspot/share/gc/z/zMark.cpp b/src/hotspot/share/gc/z/zMark.cpp index d73c2832854..b5e27f74b9d 100644 --- a/src/hotspot/share/gc/z/zMark.cpp +++ b/src/hotspot/share/gc/z/zMark.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2015, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -26,6 +26,7 @@ #include "classfile/classLoaderDataGraph.hpp" #include "classfile/javaClasses.inline.hpp" #include "code/nmethod.hpp" +#include "gc/shared/continuationGCSupport.inline.hpp" #include "gc/shared/gc_globals.hpp" #include "gc/shared/stringdedup/stringDedup.hpp" #include "gc/shared/suspendibleThreadSet.hpp" @@ -55,6 +56,7 @@ #include "oops/objArrayOop.inline.hpp" #include "oops/oop.inline.hpp" #include "runtime/atomic.hpp" +#include "runtime/continuation.hpp" #include "runtime/handshake.hpp" #include "runtime/prefetch.inline.hpp" #include "runtime/safepointMechanism.hpp" @@ -108,6 +110,12 @@ void ZMark::start() { // marking information for all pages. ZGlobalSeqNum++; + // Tell the sweeper that we start a marking cycle. + // Unlike other GCs, the color switch implicitly changes the nmethods + // to be armed, and the thread-local disarm values are lazily updated + // when JavaThreads wake up from safepoints. + Continuations::on_gc_marking_cycle_start(); + // Reset flush/continue counters _nproactiveflush = 0; _nterminateflush = 0; @@ -255,6 +263,11 @@ public: virtual void do_oop(narrowOop* p) { ShouldNotReachHere(); } + + virtual void do_nmethod(nmethod* nm) { + assert(!finalizable, "Can't handle finalizable marking of nmethods"); + nm->run_nmethod_entry_barrier(); + } }; void ZMark::follow_array_object(objArrayOop obj, bool finalizable) { @@ -273,6 +286,14 @@ void ZMark::follow_array_object(objArrayOop obj, bool finalizable) { } void ZMark::follow_object(oop obj, bool finalizable) { + if (ContinuationGCSupport::relativize_stack_chunk(obj)) { + // Loom doesn't support mixing of finalizable marking and strong marking of + // stack chunks. See: RelativizeDerivedOopClosure. + ZMarkBarrierOopClosure cl; + obj->oop_iterate(&cl); + return; + } + if (finalizable) { ZMarkBarrierOopClosure cl; obj->oop_iterate(&cl); @@ -679,6 +700,10 @@ public: if (ZNMethod::is_armed(nm)) { ZNMethod::nmethod_oops_do_inner(nm, _cl); + + // CodeCache sweeper support + nm->mark_as_maybe_on_continuation(); + ZNMethod::disarm(nm); } } @@ -800,6 +825,11 @@ bool ZMark::end() { // Update statistics ZStatMark::set_at_mark_end(_nproactiveflush, _nterminateflush, _ntrycomplete, _ncontinue); + // Tell the sweeper that we finished a marking cycle. + // Unlike other GCs, we do not arm the nmethods + // when marking terminates. + Continuations::on_gc_marking_cycle_finish(); + // Mark completed return true; } diff --git a/src/hotspot/share/gc/z/zNMethod.cpp b/src/hotspot/share/gc/z/zNMethod.cpp index 71f510c2e81..f35b06e6669 100644 --- a/src/hotspot/share/gc/z/zNMethod.cpp +++ b/src/hotspot/share/gc/z/zNMethod.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -43,6 +43,7 @@ #include "memory/universe.hpp" #include "oops/oop.inline.hpp" #include "runtime/atomic.hpp" +#include "runtime/continuation.hpp" #include "utilities/debug.hpp" static ZNMethodData* gc_data(const nmethod* nm) { @@ -206,6 +207,13 @@ void ZNMethod::disarm(nmethod* nm) { bs->disarm(nm); } +void ZNMethod::arm(nmethod* nm, int arm_value) { + BarrierSetNMethod* const bs = BarrierSet::barrier_set()->barrier_set_nmethod(); + if (bs != NULL) { + bs->arm(nm, arm_value); + } +} + void ZNMethod::nmethod_oops_do(nmethod* nm, OopClosure* cl) { ZLocker locker(ZNMethod::lock_for_nmethod(nm)); if (!nm->is_alive()) { @@ -331,7 +339,14 @@ public: if (ZNMethod::is_armed(nm)) { // Heal oops and disarm ZNMethod::nmethod_oops_barrier(nm); - ZNMethod::disarm(nm); + + if (Continuations::enabled()) { + // Loom needs to know about visited nmethods. Arm the nmethods to get + // mark_as_maybe_on_continuation() callbacks when they are used again. + ZNMethod::arm(nm, 0); + } else { + ZNMethod::disarm(nm); + } } // Clear compiled ICs and exception caches diff --git a/src/hotspot/share/gc/z/zNMethod.hpp b/src/hotspot/share/gc/z/zNMethod.hpp index 117c5e34f4e..8969c807bf3 100644 --- a/src/hotspot/share/gc/z/zNMethod.hpp +++ b/src/hotspot/share/gc/z/zNMethod.hpp @@ -47,6 +47,7 @@ public: static bool is_armed(nmethod* nm); static void disarm(nmethod* nm); + static void arm(nmethod* nm, int arm_value); static void nmethod_oops_do(nmethod* nm, OopClosure* cl); static void nmethod_oops_do_inner(nmethod* nm, OopClosure* cl); diff --git a/src/hotspot/share/gc/z/zVerify.cpp b/src/hotspot/share/gc/z/zVerify.cpp index 0dcd20fe4ae..bdf4d0eb372 100644 --- a/src/hotspot/share/gc/z/zVerify.cpp +++ b/src/hotspot/share/gc/z/zVerify.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -203,6 +203,10 @@ public: virtual ReferenceIterationMode reference_iteration_mode() { return _verify_weaks ? DO_FIELDS : DO_FIELDS_EXCEPT_REFERENT; } + + // Don't follow this metadata when verifying oops + virtual void do_method(Method* m) {} + virtual void do_nmethod(nmethod* nm) {} }; typedef ClaimingCLDToOopClosure ZVerifyCLDClosure; diff --git a/src/hotspot/share/include/jvm.h b/src/hotspot/share/include/jvm.h index e2da800f3a5..e12e16f22e3 100644 --- a/src/hotspot/share/include/jvm.h +++ b/src/hotspot/share/include/jvm.h @@ -168,6 +168,9 @@ JVM_IsSupportedJNIVersion(jint version); JNIEXPORT jobjectArray JNICALL JVM_GetVmArguments(JNIEnv *env); +JNIEXPORT jboolean JNICALL +JVM_IsPreviewEnabled(JNIEnv* env); + JNIEXPORT void JNICALL JVM_InitializeFromArchive(JNIEnv* env, jclass cls); @@ -219,7 +222,7 @@ JVM_FillInStackTrace(JNIEnv *env, jobject throwable); * java.lang.StackTraceElement */ JNIEXPORT void JNICALL -JVM_InitStackTraceElementArray(JNIEnv *env, jobjectArray elements, jobject throwable); +JVM_InitStackTraceElementArray(JNIEnv *env, jobjectArray elements, jobject backtrace, jint depth); JNIEXPORT void JNICALL JVM_InitStackTraceElement(JNIEnv* env, jobject element, jobject stackFrameInfo); @@ -243,14 +246,17 @@ enum { JNIEXPORT jobject JNICALL JVM_CallStackWalk(JNIEnv *env, jobject stackStream, jlong mode, - jint skip_frames, jint frame_count, jint start_index, - jobjectArray frames); + jint skip_frames, jobject contScope, jobject cont, + jint frame_count, jint start_index, jobjectArray frames); JNIEXPORT jint JNICALL JVM_MoreStackWalk(JNIEnv *env, jobject stackStream, jlong mode, jlong anchor, jint frame_count, jint start_index, jobjectArray frames); +JNIEXPORT void JNICALL +JVM_SetStackWalkContinuation(JNIEnv *env, jobject stackStream, jlong anchor, jobjectArray frames, jobject cont); + /* * java.lang.Thread */ @@ -278,15 +284,24 @@ JVM_Yield(JNIEnv *env, jclass threadClass); JNIEXPORT void JNICALL JVM_Sleep(JNIEnv *env, jclass threadClass, jlong millis); +JNIEXPORT jobject JNICALL +JVM_CurrentCarrierThread(JNIEnv *env, jclass threadClass); + JNIEXPORT jobject JNICALL JVM_CurrentThread(JNIEnv *env, jclass threadClass); +JNIEXPORT void JNICALL +JVM_SetCurrentThread(JNIEnv *env, jobject thisThread, jobject theThread); + JNIEXPORT void JNICALL JVM_Interrupt(JNIEnv *env, jobject thread); JNIEXPORT jboolean JNICALL JVM_HoldsLock(JNIEnv *env, jclass threadClass, jobject obj); +JNIEXPORT jobject JNICALL +JVM_GetStackTrace(JNIEnv *env, jobject thread); + JNIEXPORT void JNICALL JVM_DumpAllStacks(JNIEnv *env, jclass unused); @@ -300,6 +315,21 @@ JVM_SetNativeThreadName(JNIEnv *env, jobject jthread, jstring name); JNIEXPORT jobjectArray JNICALL JVM_DumpThreads(JNIEnv *env, jclass threadClass, jobjectArray threads); +JNIEXPORT jobject JNICALL +JVM_ExtentLocalCache(JNIEnv *env, jclass threadClass); + +JNIEXPORT void JNICALL +JVM_SetExtentLocalCache(JNIEnv *env, jclass threadClass, jobject theCache); + +JNIEXPORT jlong JNICALL +JVM_GetNextThreadIdOffset(JNIEnv *env, jclass threadClass); + +/* + * jdk.internal.vm.Continuation + */ +JNIEXPORT void JNICALL +JVM_RegisterContinuationMethods(JNIEnv *env, jclass cls); + /* * java.lang.SecurityManager */ @@ -1111,6 +1141,21 @@ JVM_GetTemporaryDirectory(JNIEnv *env); JNIEXPORT jobjectArray JNICALL JVM_GetEnclosingMethodInfo(JNIEnv* env, jclass ofClass); +/* + * Virtual thread support. + */ +JNIEXPORT void JNICALL +JVM_VirtualThreadMountBegin(JNIEnv* env, jobject vthread, jboolean first_mount); + +JNIEXPORT void JNICALL +JVM_VirtualThreadMountEnd(JNIEnv* env, jobject vthread, jboolean first_mount); + +JNIEXPORT void JNICALL +JVM_VirtualThreadUnmountBegin(JNIEnv* env, jobject vthread, jboolean last_unmount); + +JNIEXPORT void JNICALL +JVM_VirtualThreadUnmountEnd(JNIEnv* env, jobject vthread, jboolean last_unmount); + /* * This structure is used by the launcher to get the default thread * stack size from the VM using JNI_GetDefaultJavaVMInitArgs() with a diff --git a/src/hotspot/share/interpreter/abstractInterpreter.cpp b/src/hotspot/share/interpreter/abstractInterpreter.cpp index b94a30545b1..111f7a579d7 100644 --- a/src/hotspot/share/interpreter/abstractInterpreter.cpp +++ b/src/hotspot/share/interpreter/abstractInterpreter.cpp @@ -151,6 +151,7 @@ AbstractInterpreter::MethodKind AbstractInterpreter::method_kind(const methodHan case vmIntrinsics::_dsqrt: return java_lang_math_sqrt; case vmIntrinsics::_dsqrt_strict: return native; case vmIntrinsics::_Reference_get: return java_lang_ref_reference_get; + case vmIntrinsics::_Continuation_doYield: return java_lang_continuation_doYield; case vmIntrinsics::_Object_init: if (RegisterFinalizersAtInit && m->code_size() == 1) { // We need to execute the special return bytecode to check for @@ -164,6 +165,9 @@ AbstractInterpreter::MethodKind AbstractInterpreter::method_kind(const methodHan // Native method? if (m->is_native()) { + if (m->is_continuation_enter_intrinsic()) { + return zerolocals; + } assert(!m->is_method_handle_intrinsic(), "overlapping bits here, watch out"); return m->is_synchronized() ? native_synchronized : native; } diff --git a/src/hotspot/share/interpreter/abstractInterpreter.hpp b/src/hotspot/share/interpreter/abstractInterpreter.hpp index 26b3fffd2b9..58e3d41af6b 100644 --- a/src/hotspot/share/interpreter/abstractInterpreter.hpp +++ b/src/hotspot/share/interpreter/abstractInterpreter.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -81,6 +81,7 @@ class AbstractInterpreter: AllStatic { java_lang_math_fmaF, // implementation of java.lang.Math.fma (x, y, z) java_lang_math_fmaD, // implementation of java.lang.Math.fma (x, y, z) java_lang_ref_reference_get, // implementation of java.lang.ref.Reference.get() + java_lang_continuation_doYield, // implementation of jdk.internal.vm.Continuation.doYield() java_util_zip_CRC32_update, // implementation of java.util.zip.CRC32.update() java_util_zip_CRC32_updateBytes, // implementation of java.util.zip.CRC32.updateBytes() java_util_zip_CRC32_updateByteBuffer, // implementation of java.util.zip.CRC32.updateByteBuffer() @@ -157,7 +158,9 @@ class AbstractInterpreter: AllStatic { case vmIntrinsics::_dexp : // fall thru case vmIntrinsics::_fmaD : // fall thru case vmIntrinsics::_fmaF : // fall thru + case vmIntrinsics::_Continuation_doYield : // fall thru return false; + default: return true; } diff --git a/src/hotspot/share/interpreter/bytecodeTracer.cpp b/src/hotspot/share/interpreter/bytecodeTracer.cpp index d2ec5e1235c..421a8cd67ec 100644 --- a/src/hotspot/share/interpreter/bytecodeTracer.cpp +++ b/src/hotspot/share/interpreter/bytecodeTracer.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -170,6 +170,10 @@ BytecodeClosure* BytecodeTracer::std_closure() { void BytecodeTracer::trace(const methodHandle& method, address bcp, uintptr_t tos, uintptr_t tos2, outputStream* st) { + if (_closure == NULL) { + return; + } + if (TraceBytecodes && BytecodeCounter::counter_value() >= TraceBytecodesAt) { ttyLocker ttyl; // 5065316: keep the following output coherent // The ttyLocker also prevents races between two threads @@ -189,6 +193,10 @@ void BytecodeTracer::trace(const methodHandle& method, address bcp, uintptr_t to } void BytecodeTracer::trace(const methodHandle& method, address bcp, outputStream* st) { + if (_closure == NULL) { + return; + } + ttyLocker ttyl; // 5065316: keep the following output coherent _closure->trace(method, bcp, st); } diff --git a/src/hotspot/share/interpreter/interpreterRuntime.cpp b/src/hotspot/share/interpreter/interpreterRuntime.cpp index d4242595754..025e10f095e 100644 --- a/src/hotspot/share/interpreter/interpreterRuntime.cpp +++ b/src/hotspot/share/interpreter/interpreterRuntime.cpp @@ -55,6 +55,7 @@ #include "prims/methodHandles.hpp" #include "prims/nativeLookup.hpp" #include "runtime/atomic.hpp" +#include "runtime/continuation.hpp" #include "runtime/deoptimization.hpp" #include "runtime/fieldDescriptor.inline.hpp" #include "runtime/frame.inline.hpp" @@ -1118,6 +1119,10 @@ JRT_ENTRY(void, InterpreterRuntime::at_safepoint(JavaThread* current)) // JRT_END does an implicit safepoint check, hence we are guaranteed to block // if this is called during a safepoint + if (java_lang_VirtualThread::notify_jvmti_events()) { + JvmtiExport::check_vthread_and_suspend_at_safepoint(current); + } + if (JvmtiExport::should_post_single_step()) { // This function is called by the interpreter when single stepping. Such single // stepping could unwind a frame. Then, it is important that we process any frames @@ -1239,7 +1244,7 @@ JRT_END JRT_LEAF(int, InterpreterRuntime::interpreter_contains(address pc)) { - return (Interpreter::contains(pc) ? 1 : 0); + return (Interpreter::contains(Continuation::get_top_return_pc_post_barrier(JavaThread::current(), pc)) ? 1 : 0); } JRT_END diff --git a/src/hotspot/share/interpreter/linkResolver.cpp b/src/hotspot/share/interpreter/linkResolver.cpp index 63d000f134f..4391226f0d2 100644 --- a/src/hotspot/share/interpreter/linkResolver.cpp +++ b/src/hotspot/share/interpreter/linkResolver.cpp @@ -46,7 +46,7 @@ #include "oops/cpCache.inline.hpp" #include "oops/instanceKlass.inline.hpp" #include "oops/klass.inline.hpp" -#include "oops/method.hpp" +#include "oops/method.inline.hpp" #include "oops/objArrayKlass.hpp" #include "oops/objArrayOop.hpp" #include "oops/oop.inline.hpp" @@ -56,6 +56,7 @@ #include "runtime/handles.inline.hpp" #include "runtime/reflection.hpp" #include "runtime/safepointVerifiers.hpp" +#include "runtime/sharedRuntime.hpp" #include "runtime/signature.hpp" #include "runtime/thread.inline.hpp" #include "runtime/vmThread.hpp" @@ -592,6 +593,16 @@ void LinkResolver::check_method_accessability(Klass* ref_klass, } } +void LinkResolver::resolve_continuation_enter(CallInfo& callinfo, TRAPS) { + Klass* resolved_klass = vmClasses::Continuation_klass(); + Symbol* method_name = vmSymbols::enter_name(); + Symbol* method_signature = vmSymbols::continuationEnter_signature(); + Klass* current_klass = resolved_klass; + LinkInfo link_info(resolved_klass, method_name, method_signature, current_klass); + Method* resolved_method = resolve_method(link_info, Bytecodes::_invokestatic, CHECK); + callinfo.set_static(resolved_klass, methodHandle(THREAD, resolved_method), CHECK); +} + Method* LinkResolver::resolve_method_statically(Bytecodes::Code code, const constantPoolHandle& pool, int index, TRAPS) { // This method is used only @@ -1071,6 +1082,14 @@ void LinkResolver::resolve_static_call(CallInfo& result, resolved_method = linktime_resolve_static_method(new_info, CHECK); } + if (resolved_method->is_continuation_enter_intrinsic()) { + if (!resolved_method->has_compiled_code()) { + methodHandle mh(THREAD, resolved_method); + // Generate a compiled form of the enterSpecial intrinsic. + AdapterHandlerLibrary::create_native_wrapper(mh); + } + } + // setup result result.set_static(resolved_klass, methodHandle(THREAD, resolved_method), CHECK); } diff --git a/src/hotspot/share/interpreter/linkResolver.hpp b/src/hotspot/share/interpreter/linkResolver.hpp index e0b2c0b4378..b1a7274c6bc 100644 --- a/src/hotspot/share/interpreter/linkResolver.hpp +++ b/src/hotspot/share/interpreter/linkResolver.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -284,6 +284,8 @@ class LinkResolver: AllStatic { const constantPoolHandle& pool, int index, TRAPS); + static void resolve_continuation_enter(CallInfo& callinfo, TRAPS); + static void resolve_field_access(fieldDescriptor& result, const constantPoolHandle& pool, int index, diff --git a/src/hotspot/share/interpreter/oopMapCache.cpp b/src/hotspot/share/interpreter/oopMapCache.cpp index 8677cd0412d..cf491884060 100644 --- a/src/hotspot/share/interpreter/oopMapCache.cpp +++ b/src/hotspot/share/interpreter/oopMapCache.cpp @@ -202,6 +202,7 @@ void InterpreterOopMap::initialize() { _mask_size = USHRT_MAX; // This value should cause a failure quickly _bci = 0; _expression_stack_size = 0; + _num_oops = 0; for (int i = 0; i < N; i++) _bit_mask[i] = 0; } @@ -358,6 +359,7 @@ void OopMapCacheEntry::set_mask(CellTypeState *vars, CellTypeState *stack, int s uintptr_t value = 0; uintptr_t mask = 1; + _num_oops = 0; CellTypeState* cell = vars; for (int entry_index = 0; entry_index < n_entries; entry_index++, mask <<= bits_per_entry, cell++) { // store last word @@ -375,6 +377,7 @@ void OopMapCacheEntry::set_mask(CellTypeState *vars, CellTypeState *stack, int s // set oop bit if ( cell->is_reference()) { value |= (mask << oop_bit_number ); + _num_oops++; } // set dead bit @@ -407,6 +410,7 @@ void InterpreterOopMap::resource_copy(OopMapCacheEntry* from) { set_bci(from->bci()); set_mask_size(from->mask_size()); set_expression_stack_size(from->expression_stack_size()); + _num_oops = from->num_oops(); // Is the bit mask contained in the entry? if (from->mask_size() <= small_mask_limit) { diff --git a/src/hotspot/share/interpreter/oopMapCache.hpp b/src/hotspot/share/interpreter/oopMapCache.hpp index 75409371a5e..80efe351437 100644 --- a/src/hotspot/share/interpreter/oopMapCache.hpp +++ b/src/hotspot/share/interpreter/oopMapCache.hpp @@ -92,6 +92,7 @@ class InterpreterOopMap: ResourceObj { // "protected" so that sub classes can // access it without using trickery in // method bit_mask(). + int _num_oops; #ifdef ASSERT bool _resource_allocate_bit_mask; #endif @@ -139,6 +140,7 @@ class InterpreterOopMap: ResourceObj { int number_of_entries() const { return mask_size() / bits_per_entry; } bool is_dead(int offset) const { return (entry_at(offset) & (1 << dead_bit_number)) != 0; } bool is_oop (int offset) const { return (entry_at(offset) & (1 << oop_bit_number )) != 0; } + int num_oops() const { return _num_oops; } int expression_stack_size() const { return _expression_stack_size; } diff --git a/src/hotspot/share/interpreter/templateInterpreterGenerator.cpp b/src/hotspot/share/interpreter/templateInterpreterGenerator.cpp index c318882dcca..019b7f05ddf 100644 --- a/src/hotspot/share/interpreter/templateInterpreterGenerator.cpp +++ b/src/hotspot/share/interpreter/templateInterpreterGenerator.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -177,8 +177,8 @@ void TemplateInterpreterGenerator::generate_all() { -#define method_entry(kind) \ - { CodeletMark cm(_masm, "method entry point (kind = " #kind ")"); \ +#define method_entry(kind) \ + { CodeletMark cm(_masm, "method entry point (kind = " #kind ")"); \ Interpreter::_entry_table[Interpreter::kind] = generate_method_entry(Interpreter::kind); \ } @@ -223,6 +223,8 @@ void TemplateInterpreterGenerator::generate_all() { method_entry(java_lang_Double_longBitsToDouble); method_entry(java_lang_Double_doubleToRawLongBits); + method_entry(java_lang_continuation_doYield) + #undef method_entry // Bytecodes @@ -423,6 +425,8 @@ address TemplateInterpreterGenerator::generate_method_entry( case Interpreter::java_lang_math_fmaF : entry_point = generate_math_entry(kind); break; case Interpreter::java_lang_ref_reference_get : entry_point = generate_Reference_get_entry(); break; + case Interpreter::java_lang_continuation_doYield + : entry_point = generate_Continuation_doYield_entry(); break; case Interpreter::java_util_zip_CRC32_update : native = true; entry_point = generate_CRC32_update_entry(); break; case Interpreter::java_util_zip_CRC32_updateBytes diff --git a/src/hotspot/share/interpreter/templateInterpreterGenerator.hpp b/src/hotspot/share/interpreter/templateInterpreterGenerator.hpp index a5cfb9bbad0..2edcd91161a 100644 --- a/src/hotspot/share/interpreter/templateInterpreterGenerator.hpp +++ b/src/hotspot/share/interpreter/templateInterpreterGenerator.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -91,6 +91,7 @@ class TemplateInterpreterGenerator: public AbstractInterpreterGenerator { address generate_abstract_entry(void); address generate_math_entry(AbstractInterpreter::MethodKind kind); address generate_Reference_get_entry(); + address generate_Continuation_doYield_entry(); address generate_CRC32_update_entry(); address generate_CRC32_updateBytes_entry(AbstractInterpreter::MethodKind kind); address generate_CRC32C_updateBytes_entry(AbstractInterpreter::MethodKind kind); diff --git a/src/hotspot/share/jfr/jfr.cpp b/src/hotspot/share/jfr/jfr.cpp index 6f5704bab96..f6ead3b2ac7 100644 --- a/src/hotspot/share/jfr/jfr.cpp +++ b/src/hotspot/share/jfr/jfr.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -24,17 +24,15 @@ #include "precompiled.hpp" #include "jfr/jfr.hpp" +#include "jfr/jni/jfrJavaSupport.hpp" #include "jfr/leakprofiler/leakProfiler.hpp" -#include "jfr/recorder/checkpoint/types/traceid/jfrTraceIdLoadBarrier.inline.hpp" #include "jfr/recorder/jfrRecorder.hpp" #include "jfr/recorder/checkpoint/jfrCheckpointManager.hpp" #include "jfr/recorder/repository/jfrEmergencyDump.hpp" #include "jfr/recorder/service/jfrOptionSet.hpp" #include "jfr/recorder/repository/jfrRepository.hpp" #include "jfr/support/jfrThreadLocal.hpp" -#include "runtime/interfaceSupport.inline.hpp" #include "runtime/java.hpp" -#include "runtime/thread.hpp" bool Jfr::is_enabled() { return JfrRecorder::is_enabled(); @@ -72,6 +70,18 @@ void Jfr::on_unloading_classes() { } } +bool Jfr::is_excluded(Thread* t) { + return JfrJavaSupport::is_excluded(t); +} + +void Jfr::include_thread(Thread* t) { + JfrJavaSupport::include(t); +} + +void Jfr::exclude_thread(Thread* t) { + JfrJavaSupport::exclude(t); +} + void Jfr::on_thread_start(Thread* t) { JfrThreadLocal::on_start(t); } @@ -80,16 +90,12 @@ void Jfr::on_thread_exit(Thread* t) { JfrThreadLocal::on_exit(t); } -void Jfr::exclude_thread(Thread* t) { - JfrThreadLocal::exclude(t); +void Jfr::on_java_thread_start(JavaThread* starter, JavaThread* startee) { + JfrThreadLocal::on_java_thread_start(starter, startee); } -void Jfr::include_thread(Thread* t) { - JfrThreadLocal::include(t); -} - -bool Jfr::is_excluded(Thread* t) { - return t != NULL && t->jfr_thread_local()->is_excluded(); +void Jfr::on_set_current_thread(JavaThread* jt, oop thread) { + JfrThreadLocal::on_set_current_thread(jt, thread); } void Jfr::on_vm_shutdown(bool exception_handler) { @@ -111,16 +117,3 @@ bool Jfr::on_flight_recorder_option(const JavaVMOption** option, char* delimiter bool Jfr::on_start_flight_recording_option(const JavaVMOption** option, char* delimiter) { return JfrOptionSet::parse_start_flight_recording_option(option, delimiter); } - -JRT_LEAF(void, Jfr::get_class_id_intrinsic(const Klass* klass)) - assert(klass != NULL, "sanity"); - JfrTraceIdLoadBarrier::load_barrier(klass); -JRT_END - -address Jfr::epoch_address() { - return JfrTraceIdEpoch::epoch_address(); -} - -address Jfr::signal_address() { - return JfrTraceIdEpoch::signal_address(); -} diff --git a/src/hotspot/share/jfr/jfr.hpp b/src/hotspot/share/jfr/jfr.hpp index b4b1ae78adc..520fd71e4df 100644 --- a/src/hotspot/share/jfr/jfr.hpp +++ b/src/hotspot/share/jfr/jfr.hpp @@ -27,12 +27,12 @@ #include "jni.h" #include "memory/allStatic.hpp" +#include "oops/oopsHierarchy.hpp" #include "utilities/globalDefinitions.hpp" class JavaThread; -class Thread; -class Klass; class outputStream; +class Thread; extern "C" void JNICALL jfr_register_natives(JNIEnv*, jclass); @@ -48,20 +48,17 @@ class Jfr : AllStatic { static void on_create_vm_2(); static void on_create_vm_3(); static void on_unloading_classes(); - static void on_thread_start(Thread* thread); - static void on_thread_exit(Thread* thread); - static void on_vm_shutdown(bool exception_handler = false); - static bool on_flight_recorder_option(const JavaVMOption** option, char* delimiter); - static bool on_start_flight_recording_option(const JavaVMOption** option, char* delimiter); - static void on_vm_error_report(outputStream* st); - static void exclude_thread(Thread* thread); static bool is_excluded(Thread* thread); static void include_thread(Thread* thread); - - // intrinsic support - static void get_class_id_intrinsic(const Klass* klass); - static address epoch_address(); - static address signal_address(); + static void exclude_thread(Thread* thread); + static void on_thread_start(Thread* thread); + static void on_thread_exit(Thread* thread); + static void on_java_thread_start(JavaThread* starter, JavaThread* startee); + static void on_set_current_thread(JavaThread* jt, oop thread); + static void on_vm_shutdown(bool exception_handler = false); + static void on_vm_error_report(outputStream* st); + static bool on_flight_recorder_option(const JavaVMOption** option, char* delimiter); + static bool on_start_flight_recording_option(const JavaVMOption** option, char* delimiter); }; #endif // SHARE_JFR_JFR_HPP diff --git a/src/hotspot/share/jfr/jni/jfrJavaSupport.cpp b/src/hotspot/share/jfr/jni/jfrJavaSupport.cpp index ff15a68753d..d00b27a2fff 100644 --- a/src/hotspot/share/jfr/jni/jfrJavaSupport.cpp +++ b/src/hotspot/share/jfr/jni/jfrJavaSupport.cpp @@ -30,7 +30,10 @@ #include "classfile/vmSymbols.hpp" #include "jfr/jni/jfrJavaCall.hpp" #include "jfr/jni/jfrJavaSupport.hpp" -#include "jfr/support/jfrThreadId.hpp" +#include "jfr/recorder/checkpoint/jfrCheckpointManager.hpp" +#include "jfr/recorder/checkpoint/types/traceid/jfrOopTraceId.inline.hpp" +#include "jfr/recorder/checkpoint/types/traceid/jfrTraceIdEpoch.hpp" +#include "jfr/support/jfrThreadId.inline.hpp" #include "logging/log.hpp" #include "memory/resourceArea.hpp" #include "oops/instanceOop.hpp" @@ -50,20 +53,24 @@ #include "classfile/vmSymbols.hpp" #ifdef ASSERT -void JfrJavaSupport::check_java_thread_in_vm(JavaThread* t) { +static void check_java_thread_state(JavaThread* t, JavaThreadState state) { assert(t != NULL, "invariant"); - assert(t->thread_state() == _thread_in_vm, "invariant"); + assert(t->is_Java_thread(), "invariant"); + assert(t->thread_state() == state, "invariant"); +} + +void JfrJavaSupport::check_java_thread_in_vm(JavaThread* t) { + check_java_thread_state(t, _thread_in_vm); } void JfrJavaSupport::check_java_thread_in_native(JavaThread* t) { - assert(t != NULL, "invariant"); - assert(t->thread_state() == _thread_in_native, "invariant"); + check_java_thread_state(t, _thread_in_native); } -static void check_new_unstarted_java_thread(JavaThread* t) { - assert(t != NULL, "invariant"); - assert(t->thread_state() == _thread_new, "invariant"); +void JfrJavaSupport::check_java_thread_in_java(JavaThread* t) { + check_java_thread_state(t, _thread_in_Java); } + #endif /* @@ -118,6 +125,10 @@ oop JfrJavaSupport::resolve_non_null(jobject obj) { return JNIHandles::resolve_non_null(obj); } +oop JfrJavaSupport::resolve(jobject obj) { + return JNIHandles::resolve(obj); +} + /* * Method invocation */ @@ -389,6 +400,7 @@ static bool find_field(const InstanceKlass* ik, fieldDescriptor* fd, bool is_static = false, bool allow_super = false) { + assert(ik != NULL, "invariant"); if (allow_super || is_static) { return ik->find_field(name_symbol, signature_symbol, is_static, fd) != NULL; } @@ -638,98 +650,7 @@ bool JfrJavaSupport::is_jdk_jfr_module_available(outputStream* stream, TRAPS) { return true; } -class ThreadExclusionListAccess : public StackObj { - private: - static Semaphore _mutex_semaphore; - public: - ThreadExclusionListAccess() { _mutex_semaphore.wait(); } - ~ThreadExclusionListAccess() { _mutex_semaphore.signal(); } -}; - -Semaphore ThreadExclusionListAccess::_mutex_semaphore(1); -static GrowableArray* exclusion_list = NULL; - -static bool equals(const jweak excluded_thread, Handle target_thread) { - return JfrJavaSupport::resolve_non_null(excluded_thread) == target_thread(); -} - -static int find_exclusion_thread_idx(Handle thread) { - if (exclusion_list != NULL) { - for (int i = 0; i < exclusion_list->length(); ++i) { - if (equals(exclusion_list->at(i), thread)) { - return i; - } - } - } - return -1; -} - -static Handle as_handle(jobject thread) { - return Handle(Thread::current(), JfrJavaSupport::resolve_non_null(thread)); -} - -static bool thread_is_not_excluded(Handle thread) { - return -1 == find_exclusion_thread_idx(thread); -} - -static bool thread_is_not_excluded(jobject thread) { - return thread_is_not_excluded(as_handle(thread)); -} - -static bool is_thread_excluded(jobject thread) { - return !thread_is_not_excluded(thread); -} - -#ifdef ASSERT -static bool is_thread_excluded(Handle thread) { - return !thread_is_not_excluded(thread); -} -#endif // ASSERT - -static int add_thread_to_exclusion_list(jobject thread) { - ThreadExclusionListAccess lock; - if (exclusion_list == NULL) { - exclusion_list = new (ResourceObj::C_HEAP, mtTracing) GrowableArray(10, mtTracing); - } - assert(exclusion_list != NULL, "invariant"); - assert(thread_is_not_excluded(thread), "invariant"); - jweak ref = JfrJavaSupport::global_weak_jni_handle(thread, JavaThread::current()); - const int idx = exclusion_list->append(ref); - assert(is_thread_excluded(thread), "invariant"); - return idx; -} - -static void remove_thread_from_exclusion_list(Handle thread) { - assert(exclusion_list != NULL, "invariant"); - assert(is_thread_excluded(thread), "invariant"); - assert(exclusion_list != NULL, "invariant"); - const int idx = find_exclusion_thread_idx(thread); - assert(idx >= 0, "invariant"); - assert(idx < exclusion_list->length(), "invariant"); - JfrJavaSupport::destroy_global_weak_jni_handle(exclusion_list->at(idx)); - exclusion_list->delete_at(idx); - assert(thread_is_not_excluded(thread), "invariant"); - if (0 == exclusion_list->length()) { - delete exclusion_list; - exclusion_list = NULL; - } -} - -static void remove_thread_from_exclusion_list(jobject thread) { - ThreadExclusionListAccess lock; - remove_thread_from_exclusion_list(as_handle(thread)); -} - -// includes removal -static bool check_exclusion_state_on_thread_start(JavaThread* jt) { - Handle h_obj(jt, jt->threadObj()); - ThreadExclusionListAccess lock; - if (thread_is_not_excluded(h_obj)) { - return false; - } - remove_thread_from_exclusion_list(h_obj); - return true; -} +typedef JfrOopTraceId AccessThreadTraceId; static JavaThread* get_native(jobject thread) { ThreadsListHandle tlh; @@ -738,34 +659,107 @@ static JavaThread* get_native(jobject thread) { return native_thread; } -jlong JfrJavaSupport::jfr_thread_id(jobject thread) { - JavaThread* native_thread = get_native(thread); - return native_thread != NULL ? JFR_THREAD_ID(native_thread) : 0; +static bool is_virtual_thread(oop ref) { + const Klass* const k = ref->klass(); + assert(k != nullptr, "invariant"); + return k->is_subclass_of(vmClasses::VirtualThread_klass()); } -void JfrJavaSupport::exclude(jobject thread) { - JavaThread* native_thread = get_native(thread); - if (native_thread != NULL) { - JfrThreadLocal::exclude(native_thread); - } else { - // not started yet, track the thread oop - add_thread_to_exclusion_list(thread); +jlong JfrJavaSupport::jfr_thread_id(JavaThread* jt, jobject thread) { + assert(jt != nullptr, "invariant"); + oop ref = resolve(thread); + if (ref == nullptr) { + return 0; + } + const traceid tid = AccessThreadTraceId::id(ref); + if (is_virtual_thread(ref)) { + const u2 epoch = JfrTraceIdEpoch::epoch_generation(); + if (AccessThreadTraceId::epoch(ref) != epoch) { + AccessThreadTraceId::set_epoch(ref, epoch); + JfrCheckpointManager::write_checkpoint(jt, tid, ref); + } + } + return static_cast(tid); +} + +void JfrJavaSupport::exclude(JavaThread* jt, oop ref, jobject thread) { + if (ref != nullptr) { + AccessThreadTraceId::exclude(ref); + if (is_virtual_thread(ref)) { + if (ref == jt->vthread()) { + JfrThreadLocal::exclude_vthread(jt); + } + return; + } + } + jt = get_native(thread); + if (jt != nullptr) { + JfrThreadLocal::exclude_jvm_thread(jt); } } -void JfrJavaSupport::include(jobject thread) { - JavaThread* native_thread = get_native(thread); - if (native_thread != NULL) { - JfrThreadLocal::include(native_thread); - } else { - // not started yet, untrack the thread oop - remove_thread_from_exclusion_list(thread); +void JfrJavaSupport::include(JavaThread* jt, oop ref, jobject thread) { + if (ref != nullptr) { + AccessThreadTraceId::include(ref); + if (is_virtual_thread(ref)) { + if (ref == jt->vthread()) { + JfrThreadLocal::include_vthread(jt); + } + return; + } } + jt = get_native(thread); + if (jt != nullptr) { + JfrThreadLocal::include_jvm_thread(jt); + } +} + +void JfrJavaSupport::exclude(Thread* thread) { + assert(thread != nullptr, "invariant"); + if (thread->is_Java_thread()) { + JavaThread* const jt = JavaThread::cast(thread); + exclude(jt, jt->threadObj(), nullptr); + return; + } + JfrThreadLocal::exclude_jvm_thread(thread); +} + +void JfrJavaSupport::include(Thread* thread) { + assert(thread != nullptr, "invariant"); + if (thread->is_Java_thread()) { + JavaThread* const jt = JavaThread::cast(thread); + include(jt, jt->threadObj(), nullptr); + return; + } + JfrThreadLocal::include_jvm_thread(thread); +} + +void JfrJavaSupport::exclude(JavaThread* jt, jobject thread) { + oop ref = resolve(thread); + assert(ref != nullptr, "invariant"); + exclude(jt, ref, thread); +} + +void JfrJavaSupport::include(JavaThread* jt, jobject thread) { + oop ref = resolve(thread); + assert(ref != nullptr, "invariant"); + include(jt, ref, thread); } bool JfrJavaSupport::is_excluded(jobject thread) { - JavaThread* native_thread = get_native(thread); - return native_thread != NULL ? native_thread->jfr_thread_local()->is_excluded() : is_thread_excluded(thread); + oop ref = resolve(thread); + assert(ref != nullptr, "invariant"); + return AccessThreadTraceId::is_excluded(ref); +} + +bool JfrJavaSupport::is_excluded(Thread* thread) { + assert(thread != nullptr, "invariant"); + if (thread->is_Java_thread()) { + JavaThread* const jt = JavaThread::cast(thread); + oop ref = jt->threadObj(); + return ref != nullptr ? AccessThreadTraceId::is_excluded(ref) : false; + } + return JfrThreadLocal::is_jvm_thread_excluded(thread); } static const Klass* get_handler_field_descriptor(const Handle& h_mirror, fieldDescriptor* descriptor, TRAPS) { @@ -822,15 +816,32 @@ bool JfrJavaSupport::set_handler(jobject clazz, jobject handler, TRAPS) { return true; } -void JfrJavaSupport::on_thread_start(Thread* t) { +bool JfrJavaSupport::on_thread_start(Thread* t) { assert(t != NULL, "invariant"); assert(Thread::current() == t, "invariant"); if (!t->is_Java_thread()) { - return; + return true; } - DEBUG_ONLY(check_new_unstarted_java_thread(JavaThread::cast(t));) - HandleMark hm(t); - if (check_exclusion_state_on_thread_start(JavaThread::cast(t))) { - JfrThreadLocal::exclude(t); + JavaThread* const jt = JavaThread::cast(t); + assert(!JfrThreadLocal::is_vthread(jt), "invariant"); + if (is_excluded(jt)) { + JfrThreadLocal::exclude_jvm_thread(jt); + return false; } + return true; +} + +bool JfrJavaSupport::compute_field_offset(int &dest_offset, + Klass* klass, + Symbol* name_symbol, + Symbol* signature_symbol, + bool is_static, + bool allow_super) { + fieldDescriptor fd; + const InstanceKlass* const ik = InstanceKlass::cast(klass); + if (!find_field(ik, name_symbol, signature_symbol, &fd, is_static, allow_super)) { + return false; + } + dest_offset = fd.offset(); + return true; } diff --git a/src/hotspot/share/jfr/jni/jfrJavaSupport.hpp b/src/hotspot/share/jfr/jni/jfrJavaSupport.hpp index ee608387c51..71ddf4544b7 100644 --- a/src/hotspot/share/jfr/jni/jfrJavaSupport.hpp +++ b/src/hotspot/share/jfr/jni/jfrJavaSupport.hpp @@ -45,6 +45,7 @@ class JfrJavaSupport : public AllStatic { static jweak global_weak_jni_handle(const jobject handle, JavaThread* t); static void destroy_global_weak_jni_handle(jweak handle); + static oop resolve(jobject obj); static oop resolve_non_null(jobject obj); static void notify_all(jobject obj, TRAPS); static void set_array_element(jobjectArray arr, jobject element, int index, JavaThread* t); @@ -74,6 +75,14 @@ class JfrJavaSupport : public AllStatic { static jobject new_java_lang_Integer(jint value, TRAPS); static jobject new_java_lang_Long(jlong value, TRAPS); + // fields + static bool compute_field_offset(int &dest_offset, + Klass* klass, + Symbol* name_symbol, + Symbol* signature_symbol, + bool is_static = false, + bool allow_super = false); + // misc static Klass* klass(const jobject handle); static const char* c_str(jstring string, Thread* thread, bool c_heap = false); @@ -90,11 +99,16 @@ class JfrJavaSupport : public AllStatic { static bool is_jdk_jfr_module_available(); static bool is_jdk_jfr_module_available(outputStream* stream, TRAPS); - static jlong jfr_thread_id(jobject thread); - static void exclude(jobject thread); - static void include(jobject thread); + static jlong jfr_thread_id(JavaThread* jt, jobject thread); + static void exclude(JavaThread* jt, jobject thread); + static void include(JavaThread* jt, jobject thread); static bool is_excluded(jobject thread); - static void on_thread_start(Thread* t); + static void exclude(Thread* thread); + static void exclude(JavaThread* jt, oop ref, jobject thread); + static void include(Thread* thread); + static void include(JavaThread* jt, oop ref, jobject thread); + static bool is_excluded(Thread* thread); + static bool on_thread_start(Thread* t); static jobject get_handler(jobject clazz, TRAPS); static bool set_handler(jobject clazz, jobject handler, TRAPS); @@ -107,6 +121,7 @@ class JfrJavaSupport : public AllStatic { // asserts DEBUG_ONLY(static void check_java_thread_in_vm(JavaThread* t);) DEBUG_ONLY(static void check_java_thread_in_native(JavaThread* t);) + DEBUG_ONLY(static void check_java_thread_in_java(JavaThread* t);); enum CAUSE { VM_ERROR, diff --git a/src/hotspot/share/jfr/jni/jfrJniMethod.cpp b/src/hotspot/share/jfr/jni/jfrJniMethod.cpp index afbfaec6a96..a2a4d59e24d 100644 --- a/src/hotspot/share/jfr/jni/jfrJniMethod.cpp +++ b/src/hotspot/share/jfr/jni/jfrJniMethod.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2014, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -289,7 +289,7 @@ JVM_END // trace thread id for a thread object JVM_ENTRY_NO_ENV(jlong, jfr_id_for_thread(JNIEnv* env, jobject jvm, jobject t)) - return JfrJavaSupport::jfr_thread_id(t); + return JfrJavaSupport::jfr_thread_id(thread, t); JVM_END JVM_ENTRY_NO_ENV(jobject, jfr_get_event_writer(JNIEnv* env, jclass cls)) @@ -351,11 +351,11 @@ JVM_ENTRY_NO_ENV(void, jfr_emit_old_object_samples(JNIEnv* env, jobject jvm, jlo JVM_END JVM_ENTRY_NO_ENV(void, jfr_exclude_thread(JNIEnv* env, jobject jvm, jobject t)) - JfrJavaSupport::exclude(t); + JfrJavaSupport::exclude(thread, t); JVM_END JVM_ENTRY_NO_ENV(void, jfr_include_thread(JNIEnv* env, jobject jvm, jobject t)) - JfrJavaSupport::include(t); + JfrJavaSupport::include(thread, t); JVM_END JVM_ENTRY_NO_ENV(jboolean, jfr_is_thread_excluded(JNIEnv* env, jobject jvm, jobject t)) diff --git a/src/hotspot/share/jfr/jni/jfrJniMethodRegistration.cpp b/src/hotspot/share/jfr/jni/jfrJniMethodRegistration.cpp index d71e2b4f25a..afadb1fbb85 100644 --- a/src/hotspot/share/jfr/jni/jfrJniMethodRegistration.cpp +++ b/src/hotspot/share/jfr/jni/jfrJniMethodRegistration.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2016, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -69,7 +69,7 @@ JfrJniMethodRegistration::JfrJniMethodRegistration(JNIEnv* env) { (char*)"isAvailable", (char*)"()Z", (void*)jfr_is_available, (char*)"getTimeConversionFactor", (char*)"()D", (void*)jfr_time_conv_factor, (char*)"getTypeId", (char*)"(Ljava/lang/Class;)J", (void*)jfr_type_id, - (char*)"getEventWriter", (char*)"()Ljava/lang/Object;", (void*)jfr_get_event_writer, + (char*)"getEventWriter", (char*)"()Ljdk/jfr/internal/EventWriter;", (void*)jfr_get_event_writer, (char*)"newEventWriter", (char*)"()Ljdk/jfr/internal/EventWriter;", (void*)jfr_new_event_writer, (char*)"flush", (char*)"(Ljdk/jfr/internal/EventWriter;II)Z", (void*)jfr_event_writer_flush, (char*)"flush", (char*)"()V", (void*)jfr_flush, diff --git a/src/hotspot/share/jfr/leakprofiler/checkpoint/eventEmitter.cpp b/src/hotspot/share/jfr/leakprofiler/checkpoint/eventEmitter.cpp index 5ba92d7f641..aa8f6a098b6 100644 --- a/src/hotspot/share/jfr/leakprofiler/checkpoint/eventEmitter.cpp +++ b/src/hotspot/share/jfr/leakprofiler/checkpoint/eventEmitter.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2021, Datadog, Inc. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * @@ -44,13 +44,12 @@ EventEmitter::EventEmitter(const JfrTicks& start_time, const JfrTicks& end_time) _start_time(start_time), _end_time(end_time), _thread(Thread::current()), - _jfr_thread_local(_thread->jfr_thread_local()), - _thread_id(_thread->jfr_thread_local()->thread_id()) {} + _jfr_thread_local(_thread->jfr_thread_local()) {} EventEmitter::~EventEmitter() { // restore / reset thread local stack trace and thread id - _jfr_thread_local->set_thread_id(_thread_id); _jfr_thread_local->clear_cached_stack_trace(); + JfrThreadLocal::stop_impersonating(_thread); } void EventEmitter::emit(ObjectSampler* sampler, int64_t cutoff_ticks, bool emit_all, bool skip_bfs) { @@ -166,6 +165,6 @@ void EventEmitter::write_event(const ObjectSample* sample, EdgeStore* edge_store // supplying information from where the actual sampling occurred. _jfr_thread_local->set_cached_stack_trace_id(sample->stack_trace_id()); assert(sample->has_thread(), "invariant"); - _jfr_thread_local->set_thread_id(sample->thread_id()); + JfrThreadLocal::impersonate(_thread, sample->thread_id()); e.commit(); } diff --git a/src/hotspot/share/jfr/leakprofiler/checkpoint/objectSampleCheckpoint.cpp b/src/hotspot/share/jfr/leakprofiler/checkpoint/objectSampleCheckpoint.cpp index 8e2a1060528..870f5153b91 100644 --- a/src/hotspot/share/jfr/leakprofiler/checkpoint/objectSampleCheckpoint.cpp +++ b/src/hotspot/share/jfr/leakprofiler/checkpoint/objectSampleCheckpoint.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2014, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -84,10 +84,10 @@ static void add_to_unloaded_thread_set(traceid tid) { JfrMutablePredicate::test(unloaded_thread_id_set, tid); } -void ObjectSampleCheckpoint::on_thread_exit(JavaThread* jt) { - assert(jt != NULL, "invariant"); +void ObjectSampleCheckpoint::on_thread_exit(traceid tid) { + assert(tid != 0, "invariant"); if (LeakProfiler::is_running()) { - add_to_unloaded_thread_set(jt->jfr_thread_local()->thread_id()); + add_to_unloaded_thread_set(tid); } } @@ -329,7 +329,7 @@ static void write_type_set_blob(const ObjectSample* sample, JfrCheckpointWriter& static void write_thread_blob(const ObjectSample* sample, JfrCheckpointWriter& writer, bool reset) { assert(sample->has_thread(), "invariant"); - if (has_thread_exited(sample->thread_id())) { + if (sample->is_virtual_thread() || has_thread_exited(sample->thread_id())) { write_blob(sample->thread(), writer, reset); } } diff --git a/src/hotspot/share/jfr/leakprofiler/checkpoint/objectSampleCheckpoint.hpp b/src/hotspot/share/jfr/leakprofiler/checkpoint/objectSampleCheckpoint.hpp index f9f928699e2..eb887f2db9a 100644 --- a/src/hotspot/share/jfr/leakprofiler/checkpoint/objectSampleCheckpoint.hpp +++ b/src/hotspot/share/jfr/leakprofiler/checkpoint/objectSampleCheckpoint.hpp @@ -30,7 +30,6 @@ class EdgeStore; class InstanceKlass; -class JavaThread; class JfrCheckpointWriter; class JfrStackTrace; class Klass; @@ -53,7 +52,7 @@ class ObjectSampleCheckpoint : AllStatic { public: static void on_type_set(JfrCheckpointWriter& writer); static void on_type_set_unload(JfrCheckpointWriter& writer); - static void on_thread_exit(JavaThread* jt); + static void on_thread_exit(traceid tid); static void on_rotation(const ObjectSampler* sampler); }; diff --git a/src/hotspot/share/jfr/leakprofiler/sampling/objectSample.cpp b/src/hotspot/share/jfr/leakprofiler/sampling/objectSample.cpp index 725eb0b672e..b9b6aca862a 100644 --- a/src/hotspot/share/jfr/leakprofiler/sampling/objectSample.cpp +++ b/src/hotspot/share/jfr/leakprofiler/sampling/objectSample.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -32,6 +32,7 @@ void ObjectSample::reset() { set_stack_trace_id(0); set_stack_trace_hash(0); release_references(); + _virtual_thread = false; } const oop ObjectSample::object() const { diff --git a/src/hotspot/share/jfr/leakprofiler/sampling/objectSample.hpp b/src/hotspot/share/jfr/leakprofiler/sampling/objectSample.hpp index 9401c77113a..359657d60fc 100644 --- a/src/hotspot/share/jfr/leakprofiler/sampling/objectSample.hpp +++ b/src/hotspot/share/jfr/leakprofiler/sampling/objectSample.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2014, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -58,6 +58,7 @@ class ObjectSample : public JfrCHeapObj { size_t _allocated; size_t _heap_used_at_last_gc; unsigned int _stack_trace_hash; + bool _virtual_thread; void release_references() { _stacktrace.~JfrBlobHandle(); @@ -80,7 +81,8 @@ class ObjectSample : public JfrCHeapObj { _span(0), _allocated(0), _heap_used_at_last_gc(0), - _stack_trace_hash(0) {} + _stack_trace_hash(0), + _virtual_thread(false) {} ObjectSample* next() const { return _next; @@ -143,6 +145,11 @@ class ObjectSample : public JfrCHeapObj { _allocation_time = Ticks(time.value()); } + bool is_alive_and_older_than(jlong time_stamp) const { + return !is_dead() && (JfrTime::is_ft_enabled() ? + _allocation_time.ft_value() : _allocation_time.value()) < time_stamp; + } + void set_heap_used_at_last_gc(size_t heap_used) { _heap_used_at_last_gc = heap_used; } @@ -171,19 +178,6 @@ class ObjectSample : public JfrCHeapObj { _stack_trace_hash = hash; } - traceid thread_id() const { - return _thread_id; - } - - void set_thread_id(traceid id) { - _thread_id = id; - } - - bool is_alive_and_older_than(jlong time_stamp) const { - return !is_dead() && (JfrTime::is_ft_enabled() ? - _allocation_time.ft_value() : _allocation_time.value()) < time_stamp; - } - const JfrBlobHandle& stacktrace() const { return _stacktrace; } @@ -200,20 +194,37 @@ class ObjectSample : public JfrCHeapObj { } } - const JfrBlobHandle& thread() const { - return _thread; - } - bool has_thread() const { return _thread.valid(); } + const JfrBlobHandle& thread() const { + return _thread; + } + void set_thread(const JfrBlobHandle& ref) { if (_thread != ref) { _thread = ref; } } + traceid thread_id() const { + return _thread_id; + } + + void set_thread_id(traceid id) { + _thread_id = id; + } + + bool is_virtual_thread() const { + return _virtual_thread; + } + + void set_thread_is_virtual() { + assert(!_virtual_thread, "invariant"); + _virtual_thread = true; + } + const JfrBlobHandle& type_set() const { return _type_set; } diff --git a/src/hotspot/share/jfr/leakprofiler/sampling/objectSampler.cpp b/src/hotspot/share/jfr/leakprofiler/sampling/objectSampler.cpp index 5466d0162ad..acdf8959213 100644 --- a/src/hotspot/share/jfr/leakprofiler/sampling/objectSampler.cpp +++ b/src/hotspot/share/jfr/leakprofiler/sampling/objectSampler.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -144,8 +144,9 @@ void ObjectSampler::release() { _lock = 0; } -static traceid get_thread_id(JavaThread* thread) { +static traceid get_thread_id(JavaThread* thread, bool* virtual_thread) { assert(thread != NULL, "invariant"); + assert(virtual_thread != NULL, "invariant"); if (thread->threadObj() == NULL) { return 0; } @@ -154,11 +155,25 @@ static traceid get_thread_id(JavaThread* thread) { if (tl->is_excluded()) { return 0; } - if (!tl->has_thread_blob()) { - JfrCheckpointManager::create_thread_blob(thread); + *virtual_thread = tl->is_vthread(thread); + return JfrThreadLocal::thread_id(thread); +} + +static JfrBlobHandle get_thread_blob(JavaThread* thread, traceid tid, bool virtual_thread) { + assert(thread != NULL, "invariant"); + JfrThreadLocal* const tl = thread->jfr_thread_local(); + assert(tl != NULL, "invariant"); + assert(!tl->is_excluded(), "invariant"); + if (virtual_thread) { + // TODO: blob cache for virtual threads + return JfrCheckpointManager::create_thread_blob(thread, tid, thread->vthread()); } - assert(tl->has_thread_blob(), "invariant"); - return tl->thread_id(); + if (!tl->has_thread_blob()) { + // for regular threads, the blob is cached in the thread local data structure + tl->set_thread_blob(JfrCheckpointManager::create_thread_blob(thread, tid)); + assert(tl->has_thread_blob(), "invariant"); + } + return tl->thread_blob(); } class RecordStackTrace { @@ -182,10 +197,13 @@ class RecordStackTrace { void ObjectSampler::sample(HeapWord* obj, size_t allocated, JavaThread* thread) { assert(thread != NULL, "invariant"); assert(is_created(), "invariant"); - const traceid thread_id = get_thread_id(thread); + bool virtual_thread = false; + const traceid thread_id = get_thread_id(thread, &virtual_thread); if (thread_id == 0) { return; } + const JfrBlobHandle bh = get_thread_blob(thread, thread_id, virtual_thread); + assert(bh.valid(), "invariant"); RecordStackTrace rst(thread); // try enter critical section JfrTryLock tryLock(&_lock); @@ -193,14 +211,13 @@ void ObjectSampler::sample(HeapWord* obj, size_t allocated, JavaThread* thread) log_trace(jfr, oldobject, sampling)("Skipping old object sample due to lock contention"); return; } - instance().add(obj, allocated, thread_id, thread); + instance().add(obj, allocated, thread_id, virtual_thread, bh, thread); } -void ObjectSampler::add(HeapWord* obj, size_t allocated, traceid thread_id, JavaThread* thread) { +void ObjectSampler::add(HeapWord* obj, size_t allocated, traceid thread_id, bool virtual_thread, const JfrBlobHandle& bh, JavaThread* thread) { assert(obj != NULL, "invariant"); assert(thread_id != 0, "invariant"); assert(thread != NULL, "invariant"); - assert(thread->jfr_thread_local()->has_thread_blob(), "invariant"); if (Atomic::load(&_dead_samples)) { // There's a small race where a GC scan might reset this to true, potentially @@ -226,10 +243,12 @@ void ObjectSampler::add(HeapWord* obj, size_t allocated, traceid thread_id, Java assert(sample != NULL, "invariant"); sample->set_thread_id(thread_id); + if (virtual_thread) { + sample->set_thread_is_virtual(); + } + sample->set_thread(bh); const JfrThreadLocal* const tl = thread->jfr_thread_local(); - sample->set_thread(tl->thread_blob()); - const unsigned int stacktrace_hash = tl->cached_stack_trace_hash(); if (stacktrace_hash != 0) { sample->set_stack_trace_id(tl->cached_stack_trace_id()); diff --git a/src/hotspot/share/jfr/leakprofiler/sampling/objectSampler.hpp b/src/hotspot/share/jfr/leakprofiler/sampling/objectSampler.hpp index ccf40d17a9e..3789227b600 100644 --- a/src/hotspot/share/jfr/leakprofiler/sampling/objectSampler.hpp +++ b/src/hotspot/share/jfr/leakprofiler/sampling/objectSampler.hpp @@ -25,6 +25,7 @@ #ifndef SHARE_JFR_LEAKPROFILER_SAMPLING_OBJECTSAMPLER_HPP #define SHARE_JFR_LEAKPROFILER_SAMPLING_OBJECTSAMPLER_HPP +#include "jfr/utilities/jfrBlob.hpp" #include "memory/allocation.hpp" typedef u8 traceid; @@ -60,7 +61,7 @@ class ObjectSampler : public CHeapObj { // Sampling static void sample(HeapWord* object, size_t size, JavaThread* thread); - void add(HeapWord* object, size_t size, traceid thread_id, JavaThread* thread); + void add(HeapWord* object, size_t size, traceid thread_id, bool virtual_thread, const JfrBlobHandle& bh, JavaThread* thread); void scavenge(); void remove_dead(ObjectSample* sample); diff --git a/src/hotspot/share/jfr/metadata/metadata.xml b/src/hotspot/share/jfr/metadata/metadata.xml index 39d7c34a717..e6d474d8406 100644 --- a/src/hotspot/share/jfr/metadata/metadata.xml +++ b/src/hotspot/share/jfr/metadata/metadata.xml @@ -84,10 +84,6 @@ - - - - @@ -119,6 +115,44 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -1175,6 +1209,7 @@ + diff --git a/src/hotspot/share/jfr/periodic/jfrPeriodic.cpp b/src/hotspot/share/jfr/periodic/jfrPeriodic.cpp index 5b0ce8e2328..b6232756298 100644 --- a/src/hotspot/share/jfr/periodic/jfrPeriodic.cpp +++ b/src/hotspot/share/jfr/periodic/jfrPeriodic.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2012, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -44,7 +44,6 @@ #include "jfr/periodic/jfrThreadDumpEvent.hpp" #include "jfr/periodic/jfrNetworkUtilization.hpp" #include "jfr/recorder/jfrRecorder.hpp" -#include "jfr/support/jfrThreadId.hpp" #include "jfr/utilities/jfrThreadIterator.hpp" #include "jfr/utilities/jfrTime.hpp" #include "jfrfiles/jfrPeriodic.hpp" @@ -430,7 +429,7 @@ TRACE_REQUEST_FUNC(ThreadAllocationStatistics) { JavaThread* const jt = iter.next(); assert(jt != NULL, "invariant"); allocated.append(jt->cooked_allocated_bytes()); - thread_ids.append(JFR_THREAD_ID(jt)); + thread_ids.append(JFR_JVM_THREAD_ID(jt)); } // Write allocation statistics to buffer. diff --git a/src/hotspot/share/jfr/periodic/jfrThreadCPULoadEvent.cpp b/src/hotspot/share/jfr/periodic/jfrThreadCPULoadEvent.cpp index 96e117aa870..367bc0847f5 100644 --- a/src/hotspot/share/jfr/periodic/jfrThreadCPULoadEvent.cpp +++ b/src/hotspot/share/jfr/periodic/jfrThreadCPULoadEvent.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -26,12 +26,11 @@ #include "logging/log.hpp" #include "jfr/jfrEvents.hpp" #include "jfr/periodic/jfrThreadCPULoadEvent.hpp" -#include "jfr/support/jfrThreadId.hpp" -#include "jfr/support/jfrThreadLocal.hpp" #include "jfr/utilities/jfrThreadIterator.hpp" #include "jfr/utilities/jfrTime.hpp" #include "utilities/globalDefinitions.hpp" #include "runtime/os.hpp" +#include "runtime/thread.inline.hpp" jlong JfrThreadCPULoadEvent::get_wallclock_time() { return os::javaTimeNanos(); @@ -107,8 +106,7 @@ bool JfrThreadCPULoadEvent::update_event(EventThreadCPULoad& event, JavaThread* void JfrThreadCPULoadEvent::send_events() { Thread* periodic_thread = Thread::current(); - JfrThreadLocal* const periodic_thread_tl = periodic_thread->jfr_thread_local(); - traceid periodic_thread_id = periodic_thread_tl->thread_id(); + traceid periodic_thread_id = JfrThreadLocal::thread_id(periodic_thread); const int processor_count = JfrThreadCPULoadEvent::get_processor_count(); JfrTicks event_time = JfrTicks::now(); jlong cur_wallclock_time = JfrThreadCPULoadEvent::get_wallclock_time(); @@ -124,17 +122,16 @@ void JfrThreadCPULoadEvent::send_events() { event.set_starttime(event_time); if (jt != periodic_thread) { // Commit reads the thread id from this thread's trace data, so put it there temporarily - periodic_thread_tl->set_thread_id(JFR_THREAD_ID(jt)); + JfrThreadLocal::impersonate(periodic_thread, JFR_JVM_THREAD_ID(jt)); } else { - periodic_thread_tl->set_thread_id(periodic_thread_id); + JfrThreadLocal::impersonate(periodic_thread, periodic_thread_id); } event.commit(); } } log_trace(jfr)("Measured CPU usage for %d threads in %.3f milliseconds", number_of_threads, (double)(JfrTicks::now() - event_time).milliseconds()); - // Restore this thread's thread id - periodic_thread_tl->set_thread_id(periodic_thread_id); + JfrThreadLocal::stop_impersonating(periodic_thread); } void JfrThreadCPULoadEvent::send_event_for_thread(JavaThread* jt) { diff --git a/src/hotspot/share/jfr/periodic/sampling/jfrThreadSampler.cpp b/src/hotspot/share/jfr/periodic/sampling/jfrThreadSampler.cpp index c09c634e550..13460983364 100644 --- a/src/hotspot/share/jfr/periodic/sampling/jfrThreadSampler.cpp +++ b/src/hotspot/share/jfr/periodic/sampling/jfrThreadSampler.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2012, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -31,7 +31,6 @@ #include "jfr/recorder/service/jfrOptionSet.hpp" #include "jfr/recorder/stacktrace/jfrStackTraceRepository.hpp" #include "jfr/recorder/storage/jfrBuffer.hpp" -#include "jfr/support/jfrThreadId.hpp" #include "jfr/support/jfrThreadLocal.hpp" #include "jfr/utilities/jfrTime.hpp" #include "jfrfiles/jfrEventClasses.hpp" @@ -177,15 +176,15 @@ void OSThreadSampler::do_task(const os::SuspendedThreadTaskContext& context) { * using a signal handler / __try block. Don't take locks, rely on destructors or * leave memory (in case of signal / exception) in an inconsistent state. */ void OSThreadSampler::protected_task(const os::SuspendedThreadTaskContext& context) { - JavaThread* jth = JavaThread::cast(context.thread()); + JavaThread* const jt = JavaThread::cast(context.thread()); // Skip sample if we signaled a thread that moved to other state - if (!thread_state_in_java(jth)) { + if (!thread_state_in_java(jt)) { return; } - JfrGetCallTrace trace(true, jth); + JfrGetCallTrace trace(true, jt); frame topframe; if (trace.get_topframe(context.ucontext(), topframe)) { - if (_stacktrace.record_thread(*jth, topframe)) { + if (_stacktrace.record_async(jt, topframe)) { /* If we managed to get a topframe and a stacktrace, create an event * and put it into our array. We can't call Jfr::_stacktraces.add() * here since it would allocate memory using malloc. Doing so while @@ -194,7 +193,7 @@ void OSThreadSampler::protected_task(const os::SuspendedThreadTaskContext& conte EventExecutionSample *ev = _closure.next_event(); ev->set_starttime(_suspend_time); ev->set_endtime(_suspend_time); // fake to not take an end time - ev->set_sampledThread(JFR_THREAD_ID(jth)); + ev->set_sampledThread(JfrThreadLocal::thread_id(jt)); ev->set_state(static_cast(java_lang_Thread::get_thread_status(_thread_oop))); } } @@ -224,7 +223,7 @@ class JfrNativeSamplerCallback : public os::CrashProtectionCallback { static void write_native_event(JfrThreadSampleClosure& closure, JavaThread* jt, oop thread_oop) { EventNativeMethodSample *ev = closure.next_event_native(); ev->set_starttime(JfrTicks::now()); - ev->set_sampledThread(JFR_THREAD_ID(jt)); + ev->set_sampledThread(JfrThreadLocal::thread_id(jt)); ev->set_state(static_cast(java_lang_Thread::get_thread_status(thread_oop))); } @@ -245,7 +244,7 @@ void JfrNativeSamplerCallback::call() { return; } topframe = first_java_frame; - _success = _stacktrace.record_thread(*_jt, topframe); + _success = _stacktrace.record_async(_jt, topframe); if (_success) { write_native_event(_closure, _jt, _thread_oop); } diff --git a/src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointManager.cpp b/src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointManager.cpp index 0bd6c3355dd..a10e77bfc39 100644 --- a/src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointManager.cpp +++ b/src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointManager.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2016, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -39,12 +39,12 @@ #include "jfr/recorder/storage/jfrMemorySpace.inline.hpp" #include "jfr/recorder/storage/jfrStorageUtils.inline.hpp" #include "jfr/support/jfrKlassUnloading.hpp" +#include "jfr/support/jfrThreadLocal.hpp" #include "jfr/utilities/jfrBigEndian.hpp" #include "jfr/utilities/jfrIterator.hpp" #include "jfr/utilities/jfrLinkedList.inline.hpp" #include "jfr/utilities/jfrSignal.hpp" #include "jfr/utilities/jfrThreadIterator.hpp" -#include "jfr/utilities/jfrTypes.hpp" #include "jfr/writers/jfrJavaEventWriter.hpp" #include "logging/log.hpp" #include "memory/iterator.hpp" @@ -91,8 +91,8 @@ JfrCheckpointManager::~JfrCheckpointManager() { static const size_t global_buffer_prealloc_count = 2; static const size_t global_buffer_size = 512 * K; -static const size_t thread_local_buffer_prealloc_count = 16; -static const size_t thread_local_buffer_size = 128; +static const size_t thread_local_buffer_prealloc_count = 32; +static const size_t thread_local_buffer_size = 4 * K; bool JfrCheckpointManager::initialize() { assert(_global_mspace == NULL, "invariant"); @@ -162,7 +162,7 @@ BufferPtr JfrCheckpointManager::lease(Thread* thread, bool previous_epoch /* fal const u1 thread_local_context = 1; -static bool is_thread_local(JfrBuffer* buffer) { +static bool is_thread_local(const JfrBuffer* buffer) { assert(buffer != NULL, "invariant"); return buffer->context() == thread_local_context; } @@ -179,13 +179,27 @@ static void retire(JfrBuffer* buffer) { */ static void release(JfrBuffer* buffer) { DEBUG_ONLY(assert_release(buffer);) - if (is_thread_local(buffer)) { - retire(buffer); - } else { + if (!is_thread_local(buffer)) { buffer->clear_lease(); buffer->release(); } } + +BufferPtr JfrCheckpointManager::get_thread_local(Thread* thread) { + assert(thread != NULL, "invariant"); + return JfrTraceIdEpoch::epoch() ? thread->jfr_thread_local()->_checkpoint_buffer_epoch_1 : + thread->jfr_thread_local()->_checkpoint_buffer_epoch_0; +} + +void JfrCheckpointManager::set_thread_local(Thread* thread, BufferPtr buffer) { + assert(thread != NULL, "invariant"); + if (JfrTraceIdEpoch::epoch()) { + thread->jfr_thread_local()->_checkpoint_buffer_epoch_1 = buffer; + } else { + thread->jfr_thread_local()->_checkpoint_buffer_epoch_0 = buffer; + } +} + BufferPtr JfrCheckpointManager::acquire_thread_local(size_t size, Thread* thread) { assert(thread != NULL, "invariant"); JfrBuffer* const buffer = instance()._thread_local_mspace->acquire(size, thread); @@ -194,12 +208,21 @@ BufferPtr JfrCheckpointManager::acquire_thread_local(size_t size, Thread* thread buffer->set_context(thread_local_context); assert(is_thread_local(buffer), "invariant"); buffer->set_lease(); + set_thread_local(thread, buffer); return buffer; } BufferPtr JfrCheckpointManager::lease_thread_local(Thread* thread, size_t size /* 0 */) { - JfrBuffer* const buffer = acquire_thread_local(size, thread); + JfrBuffer* buffer = get_thread_local(thread); + if (buffer == NULL) { + buffer = acquire_thread_local(size, thread); + } else if (buffer->free_size() < size) { + retire(buffer); + buffer = acquire_thread_local(size, thread); + } DEBUG_ONLY(assert_lease(buffer);) + assert(buffer->free_size() >= size, "invariant"); + assert(get_thread_local(thread) == buffer, "invariant"); return buffer; } @@ -250,8 +273,8 @@ static jlong duration(const u1* data) { return read_data(data + duration_offset); } -static u1 checkpoint_type(const u1* data) { - return read_data(data + checkpoint_type_offset); +static juint checkpoint_type(const u1* data) { + return read_data(data + checkpoint_type_offset); } static juint number_of_types(const u1* data) { @@ -273,6 +296,17 @@ static void write_checkpoint_content(JfrChunkWriter& cw, const u1* data, size_t cw.write_unbuffered(data + payload_offset, size - sizeof(JfrCheckpointEntry)); } +static size_t write_thread_checkpoint_content(JfrChunkWriter& cw, const u1* data) { + assert(data != NULL, "invariant"); + const size_t size = total_size(data); + assert(size > sizeof(JfrCheckpointEntry), "invariant"); + assert(checkpoint_type(data) == THREADS, "invariant"); + assert(number_of_types(data) == 1, "invariant"); + // Thread checkpoints are small so write them buffered to cache as much as possible before flush. + cw.write_buffered(data + payload_offset, size - sizeof(JfrCheckpointEntry)); + return size; +} + static size_t write_checkpoint_event(JfrChunkWriter& cw, const u1* data) { assert(data != NULL, "invariant"); const int64_t event_begin = cw.current_offset(); @@ -303,6 +337,23 @@ static size_t write_checkpoints(JfrChunkWriter& cw, const u1* data, size_t size) return processed; } +static size_t write_thread_checkpoint_payloads(JfrChunkWriter& cw, const u1* data, size_t size, u4& elements) { + assert(cw.is_valid(), "invariant"); + assert(data != NULL, "invariant"); + assert(size > 0, "invariant"); + const u1* const limit = data + size; + const u1* next = data; + size_t processed_total = 0; + while (next < limit) { + const size_t processed = write_thread_checkpoint_content(cw, next); + next += processed; + processed_total += processed; + ++elements; + } + assert(next == limit, "invariant"); + return processed_total; +} + template class CheckpointWriteOp { private: @@ -318,10 +369,60 @@ class CheckpointWriteOp { size_t processed() const { return _processed; } }; + +// This op will collapse all individual vthread checkpoints into a single checkpoint. +template +class ThreadLocalCheckpointWriteOp { + private: + JfrChunkWriter& _cw; + int64_t _begin_offset; + int64_t _elements_offset; + size_t _processed; + u4 _elements; + public: + typedef T Type; + ThreadLocalCheckpointWriteOp(JfrChunkWriter& cw) : _cw(cw), _begin_offset(cw.current_offset()), _elements_offset(0), _processed(0), _elements(0) { + const int64_t last_checkpoint = cw.last_checkpoint_offset(); + const int64_t delta = last_checkpoint == 0 ? 0 : last_checkpoint - _begin_offset; + cw.reserve(sizeof(u4)); + cw.write(EVENT_CHECKPOINT); + cw.write(JfrTicks::now().value()); + cw.write(0); + cw.write(delta); + cw.write(THREADS); // Thread checkpoint type. + cw.write(1); // Number of types in this checkpoint, only one, TYPE_THREAD. + cw.write(TYPE_THREAD); // Constant pool type. + _elements_offset = cw.current_offset(); // Offset for the number of entries in the TYPE_THREAD constant pool. + cw.reserve(sizeof(u4)); + } + + ~ThreadLocalCheckpointWriteOp() { + if (_elements == 0) { + // Rewind. + _cw.seek(_begin_offset); + return; + } + const int64_t event_size = _cw.current_offset() - _begin_offset; + _cw.write_padded_at_offset(_elements, _elements_offset); + _cw.write_padded_at_offset(event_size, _begin_offset); + _cw.set_last_checkpoint_offset(_begin_offset); + } + + bool write(Type* t, const u1* data, size_t size) { + _processed += write_thread_checkpoint_payloads(_cw, data, size, _elements); + return true; + } + size_t elements() const { return _elements; } + size_t processed() const { return _processed; } +}; + typedef CheckpointWriteOp WriteOperation; +typedef ThreadLocalCheckpointWriteOp ThreadLocalWriteOperation; typedef MutexedWriteOp MutexedWriteOperation; +typedef MutexedWriteOp ThreadLocalMutexedWriteOperation; typedef ReleaseWithExcisionOp ReleaseOperation; typedef CompositeOperation WriteReleaseOperation; +typedef CompositeOperation ThreadLocalWriteReleaseOperation; void JfrCheckpointManager::begin_epoch_shift() { assert(SafepointSynchronize::is_at_safepoint(), "invariant"); @@ -339,12 +440,15 @@ size_t JfrCheckpointManager::write() { DEBUG_ONLY(JfrJavaSupport::check_java_thread_in_native(JavaThread::current())); WriteOperation wo(_chunkwriter); MutexedWriteOperation mwo(wo); - _thread_local_mspace->iterate(mwo, true); // previous epoch list assert(_global_mspace->free_list_is_empty(), "invariant"); ReleaseOperation ro(_global_mspace, _global_mspace->live_list(true)); WriteReleaseOperation wro(&mwo, &ro); process_live_list(wro, _global_mspace, true); // previous epoch list - return wo.processed(); + // Do thread local list after global. Careful, the tlwo destructor writes to chunk. + ThreadLocalWriteOperation tlwo(_chunkwriter); + ThreadLocalMutexedWriteOperation tlmwo(tlwo); + _thread_local_mspace->iterate(tlmwo, true); // previous epoch list + return wo.processed() + tlwo.processed(); } typedef DiscardOp > DiscardOperation; @@ -456,21 +560,27 @@ size_t JfrCheckpointManager::flush_type_set() { } } if (_new_checkpoint.is_signaled_with_reset()) { + assert(_global_mspace->live_list_is_nonempty(), "invariant"); WriteOperation wo(_chunkwriter); MutexedWriteOperation mwo(wo); - _thread_local_mspace->iterate(mwo); // current epoch list - assert(_global_mspace->live_list_is_nonempty(), "invariant"); process_live_list(mwo, _global_mspace); // current epoch list + // Do thread local list after global. Careful, the tlwo destructor writes to chunk. + ThreadLocalWriteOperation tlwo(_chunkwriter); + ThreadLocalMutexedWriteOperation tlmwo(tlwo); + _thread_local_mspace->iterate(tlmwo); // current epoch list } return elements; } -void JfrCheckpointManager::create_thread_blob(Thread* thread) { - JfrTypeManager::create_thread_blob(thread); +JfrBlobHandle JfrCheckpointManager::create_thread_blob(JavaThread* jt, traceid tid /* 0 */, oop vthread /* nullptr */) { + assert(jt != NULL, "invariant"); + assert(Thread::current() == jt, "invariant"); + DEBUG_ONLY(JfrJavaSupport::check_java_thread_in_vm(jt)); + return JfrTypeManager::create_thread_blob(jt, tid, vthread); } -void JfrCheckpointManager::write_thread_checkpoint(Thread* thread) { - JfrTypeManager::write_thread_checkpoint(thread); +void JfrCheckpointManager::write_checkpoint(Thread* thread, traceid tid /* 0 */, oop vthread /* nullptr */) { + JfrTypeManager::write_checkpoint(thread, tid, vthread); } class JfrNotifyClosure : public ThreadClosure { diff --git a/src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointManager.hpp b/src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointManager.hpp index c28b92138d5..7c8dc7d617a 100644 --- a/src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointManager.hpp +++ b/src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointManager.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2016, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -29,7 +29,9 @@ #include "jfr/recorder/storage/jfrEpochStorage.hpp" #include "jfr/recorder/storage/jfrMemorySpace.hpp" #include "jfr/recorder/storage/jfrMemorySpaceRetrieval.hpp" +#include "jfr/utilities/jfrBlob.hpp" #include "jfr/utilities/jfrLinkedList.hpp" +#include "jfr/utilities/jfrTypes.hpp" class JfrCheckpointManager; class JfrChunkWriter; @@ -67,10 +69,12 @@ class JfrCheckpointManager : public JfrCHeapObj { bool initialize(); static void destroy(); + static BufferPtr get_thread_local(Thread* thread); + static void set_thread_local(Thread* thread, BufferPtr buffer); + static BufferPtr acquire_thread_local(size_t size, Thread* thread); + static BufferPtr lease(Thread* thread, bool previous_epoch = false, size_t size = 0); static BufferPtr lease(BufferPtr old, Thread* thread, size_t size); - - static BufferPtr acquire_thread_local(size_t size, Thread* thread); static BufferPtr lease_thread_local(Thread* thread, size_t size = 0); static BufferPtr flush(BufferPtr old, size_t used, size_t requested, Thread* thread); @@ -95,9 +99,9 @@ class JfrCheckpointManager : public JfrCHeapObj { void register_full(BufferPtr buffer, Thread* thread); public: + static JfrBlobHandle create_thread_blob(JavaThread* jt, traceid tid = 0, oop vthread = NULL); + static void write_checkpoint(Thread* t, traceid tid = 0, oop vthread = NULL); size_t flush_type_set(); - static void create_thread_blob(Thread* thread); - static void write_thread_checkpoint(Thread* thread); friend class Jfr; friend class JfrRecorder; diff --git a/src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointWriter.cpp b/src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointWriter.cpp index 12497802cf7..4085abbcd28 100644 --- a/src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointWriter.cpp +++ b/src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointWriter.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2016, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/jfr/recorder/checkpoint/types/jfrThreadGroup.cpp b/src/hotspot/share/jfr/recorder/checkpoint/types/jfrThreadGroup.cpp index 6d362305400..6dd8becd77e 100644 --- a/src/hotspot/share/jfr/recorder/checkpoint/types/jfrThreadGroup.cpp +++ b/src/hotspot/share/jfr/recorder/checkpoint/types/jfrThreadGroup.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2016, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -182,7 +182,7 @@ int JfrThreadGroupsHelper::populate_thread_group_hierarchy(const JavaThread* jt, } static traceid next_id() { - static traceid _current_threadgroup_id = 0; + static traceid _current_threadgroup_id = 1; // 1 is reserved for thread group "VirtualThreads" return ++_current_threadgroup_id; } @@ -284,6 +284,7 @@ void JfrThreadGroup::set_instance(JfrThreadGroup* new_instance) { } traceid JfrThreadGroup::thread_group_id(const JavaThread* jt, Thread* current) { + HandleMark hm(current); JfrThreadGroupsHelper helper(jt, current); return helper.is_valid() ? thread_group_id_internal(helper) : 0; } @@ -353,7 +354,14 @@ int JfrThreadGroup::add_entry(JfrThreadGroupEntry* tge) { void JfrThreadGroup::write_thread_group_entries(JfrCheckpointWriter& writer) const { assert(_list != NULL && !_list->is_empty(), "should not need be here!"); const int number_of_tg_entries = _list->length(); - writer.write_count(number_of_tg_entries); + writer.write_count(number_of_tg_entries + 1); // + VirtualThread group + writer.write_key(1); // 1 is reserved for VirtualThread group + writer.write(0); // parent + const oop vgroup = java_lang_Thread_Constants::get_VTHREAD_GROUP(); + assert(vgroup != (oop)NULL, "invariant"); + const char* const vgroup_name = java_lang_ThreadGroup::name(vgroup); + assert(vgroup_name != NULL, "invariant"); + writer.write(vgroup_name); for (int index = 0; index < number_of_tg_entries; ++index) { const JfrThreadGroupEntry* const curtge = _list->at(index); writer.write_key(curtge->thread_group_id()); @@ -365,6 +373,7 @@ void JfrThreadGroup::write_thread_group_entries(JfrCheckpointWriter& writer) con void JfrThreadGroup::write_selective_thread_group(JfrCheckpointWriter* writer, traceid thread_group_id) const { assert(writer != NULL, "invariant"); assert(_list != NULL && !_list->is_empty(), "should not need be here!"); + assert(thread_group_id != 1, "should not need be here!"); const int number_of_tg_entries = _list->length(); // save context diff --git a/src/hotspot/share/jfr/recorder/checkpoint/types/jfrThreadState.cpp b/src/hotspot/share/jfr/recorder/checkpoint/types/jfrThreadState.cpp index 4526ff108a9..5c2576f3b81 100644 --- a/src/hotspot/share/jfr/recorder/checkpoint/types/jfrThreadState.cpp +++ b/src/hotspot/share/jfr/recorder/checkpoint/types/jfrThreadState.cpp @@ -1,5 +1,5 @@ /* -* Copyright (c) 2016, 2021, Oracle and/or its affiliates. All rights reserved. +* Copyright (c) 2016, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -84,13 +84,16 @@ void JfrThreadState::serialize(JfrCheckpointWriter& writer) { } } -traceid JfrThreadId::id(const Thread* t) { - assert(t != NULL, "invariant"); +traceid JfrThreadId::id(const Thread* t, oop vthread) { + assert(t != nullptr, "invariant"); if (!t->is_Java_thread()) { return os_id(t); } + if (vthread != nullptr) { + return java_lang_Thread::thread_id(vthread); + } const oop thread_obj = JavaThread::cast(t)->threadObj(); - return thread_obj != NULL ? java_lang_Thread::thread_id(thread_obj) : 0; + return thread_obj != nullptr ? java_lang_Thread::thread_id(thread_obj) : 0; } traceid JfrThreadId::os_id(const Thread* t) { @@ -99,29 +102,31 @@ traceid JfrThreadId::os_id(const Thread* t) { return os_thread != NULL ? os_thread->thread_id() : 0; } -traceid JfrThreadId::jfr_id(const Thread* t) { +traceid JfrThreadId::jfr_id(const Thread* t, traceid tid) { assert(t != NULL, "invariant"); - return t->jfr_thread_local()->thread_id(); + return tid != 0 ? tid : JfrThreadLocal::jvm_thread_id(t); } // caller needs ResourceMark -const char* get_java_thread_name(const JavaThread* jt) { - assert(jt != NULL, "invariant"); +const char* get_java_thread_name(const JavaThread* jt, int& length, oop vthread) { + assert(jt != nullptr, "invariant"); const char* name_str = ""; - const oop thread_obj = jt->threadObj(); - if (thread_obj != NULL) { - const oop name = java_lang_Thread::name(thread_obj); - if (name != NULL) { - name_str = java_lang_String::as_utf8_string(name); + oop thread_obj = vthread != NULL ? vthread : jt->threadObj(); + if (thread_obj == NULL) { + if (jt->is_attaching_via_jni()) { + name_str = ""; + } + } else { + const oop name = java_lang_Thread::name(thread_obj); + if (name != nullptr) { + name_str = java_lang_String::as_utf8_string(name, length); } - } else if (jt->is_attaching_via_jni()) { - name_str = ""; } - assert(name_str != NULL, "unexpected NULL thread name"); + assert(name_str != nullptr, "unexpected nullptr thread name"); return name_str; } -const char* JfrThreadName::name(const Thread* t) { - assert(t != NULL, "invariant"); - return t->is_Java_thread() ? get_java_thread_name(JavaThread::cast(t)) : t->name(); +const char* JfrThreadName::name(const Thread* t, int& length, oop vthread) { + assert(t != nullptr, "invariant"); + return t->is_Java_thread() ? get_java_thread_name(JavaThread::cast(t), length, vthread) : t->name(); } diff --git a/src/hotspot/share/jfr/recorder/checkpoint/types/jfrThreadState.hpp b/src/hotspot/share/jfr/recorder/checkpoint/types/jfrThreadState.hpp index 4fd519b5064..633e4061253 100644 --- a/src/hotspot/share/jfr/recorder/checkpoint/types/jfrThreadState.hpp +++ b/src/hotspot/share/jfr/recorder/checkpoint/types/jfrThreadState.hpp @@ -37,15 +37,15 @@ class JfrThreadState : public AllStatic { class JfrThreadId : public AllStatic { public: - static traceid id(const Thread* t); + static traceid id(const Thread* t, oop vthread = NULL); static traceid os_id(const Thread* t); - static traceid jfr_id(const Thread* t); + static traceid jfr_id(const Thread* t, traceid tid = 0); }; class JfrThreadName : public AllStatic { public: // Requires a ResourceMark for get_thread_name/as_utf8 - static const char* name(const Thread* t); + static const char* name(const Thread* t, int& length, oop vthread = NULL); }; #endif // SHARE_JFR_RECORDER_CHECKPOINT_TYPES_JFRTHREADSTATE_HPP diff --git a/src/hotspot/share/jfr/recorder/checkpoint/types/jfrType.cpp b/src/hotspot/share/jfr/recorder/checkpoint/types/jfrType.cpp index 7c52e5741e3..5803d84605e 100644 --- a/src/hotspot/share/jfr/recorder/checkpoint/types/jfrType.cpp +++ b/src/hotspot/share/jfr/recorder/checkpoint/types/jfrType.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2016, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -88,23 +88,28 @@ class JfrCheckpointThreadClosure : public ThreadClosure { void do_thread(Thread* t); }; +// Only static thread ids, virtual threads are handled dynamically. void JfrCheckpointThreadClosure::do_thread(Thread* t) { assert(t != NULL, "invariant"); ++_count; - _writer.write_key(JfrThreadId::jfr_id(t)); - const char* const name = JfrThreadName::name(t); + const traceid tid = JfrThreadId::jfr_id(t); + assert(tid != 0, "invariant"); + _writer.write_key(tid); + int length = -1; + const char* const name = JfrThreadName::name(t, length); assert(name != NULL, "invariant"); _writer.write(name); _writer.write(JfrThreadId::os_id(t)); - if (t->is_Java_thread()) { + if (!t->is_Java_thread()) { + _writer.write((const char*)nullptr); // java name + _writer.write((traceid)0); // java thread id + _writer.write((traceid)0); // java thread group + } else { _writer.write(name); - _writer.write(JfrThreadId::id(t)); + _writer.write(tid); _writer.write(JfrThreadGroup::thread_group_id(JavaThread::cast(t), _curthread)); - return; } - _writer.write((const char*)NULL); // java name - _writer.write((traceid)0); // java thread id - _writer.write((traceid)0); // java thread group + _writer.write(false); // isVirtual } void JfrThreadConstantSet::serialize(JfrCheckpointWriter& writer) { @@ -265,26 +270,40 @@ void ThreadStateConstant::serialize(JfrCheckpointWriter& writer) { JfrThreadState::serialize(writer); } -void JfrThreadConstant::serialize(JfrCheckpointWriter& writer) { - assert(_thread != NULL, "invariant"); - assert(_thread == Thread::current(), "invariant"); - writer.write_count(1); - writer.write_key(JfrThreadId::jfr_id(_thread)); - const char* const name = JfrThreadName::name(_thread); - writer.write(name); - writer.write(JfrThreadId::os_id(_thread)); - if (_thread->is_Java_thread()) { - writer.write(name); - writer.write(JfrThreadId::id(_thread)); - JavaThread* const jt = JavaThread::cast(_thread); - const traceid thread_group_id = JfrThreadGroup::thread_group_id(jt, jt); - writer.write(thread_group_id); - JfrThreadGroup::serialize(&writer, thread_group_id); +void JfrThreadConstant::write_name(JfrCheckpointWriter& writer, const char* name, int length) { + if (length == 0) { + writer.write_empty_string(); return; } - writer.write((const char*)NULL); // java name - writer.write((traceid)0); // java thread id - writer.write((traceid)0); // java thread group + writer.write(name); +} + +void JfrThreadConstant::serialize(JfrCheckpointWriter& writer) { + assert(_thread != NULL, "invariant"); + const bool vthread = _vthread != nullptr; + writer.write_key(JfrThreadId::jfr_id(_thread, _tid)); + int length = -1; + const char* const name = JfrThreadName::name(_thread, length, _vthread); + write_name(writer, name, length); + writer.write(_vthread != nullptr ? static_cast(0) : JfrThreadId::os_id(_thread)); + if (!_thread->is_Java_thread()) { + write_name(writer, nullptr, 0); // java name + writer.write(0); // java thread id + writer.write(0); // java thread group + writer.write(false); // isVirtual + } else { + write_name(writer, name, length); + writer.write(JfrThreadId::jfr_id(_thread, _tid)); + // java thread group - VirtualThread threadgroup reserved id 1 + const traceid thread_group_id = vthread ? 1 : + JfrThreadGroup::thread_group_id(JavaThread::cast(_thread), Thread::current()); + writer.write(thread_group_id); + writer.write(vthread); // isVirtual + if (!vthread) { + JfrThreadGroup::serialize(&writer, thread_group_id); + } + // VirtualThread threadgroup already serialized invariant. + } } void BytecodeConstant::serialize(JfrCheckpointWriter& writer) { diff --git a/src/hotspot/share/jfr/recorder/checkpoint/types/jfrType.hpp b/src/hotspot/share/jfr/recorder/checkpoint/types/jfrType.hpp index 5973b444aa4..731dee626c4 100644 --- a/src/hotspot/share/jfr/recorder/checkpoint/types/jfrType.hpp +++ b/src/hotspot/share/jfr/recorder/checkpoint/types/jfrType.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2016, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -105,8 +105,11 @@ class ThreadStateConstant : public JfrSerializer { class JfrThreadConstant : public JfrSerializer { private: Thread* _thread; + traceid _tid; + oop _vthread; + void write_name(JfrCheckpointWriter& writer, const char* name, int length); public: - JfrThreadConstant(Thread* t) : _thread(t) {} + JfrThreadConstant(Thread* t, traceid tid, oop vthread = NULL) : _thread(t), _tid(tid), _vthread(vthread) {} void serialize(JfrCheckpointWriter& writer); }; diff --git a/src/hotspot/share/jfr/recorder/checkpoint/types/jfrTypeManager.cpp b/src/hotspot/share/jfr/recorder/checkpoint/types/jfrTypeManager.cpp index 40525969efe..472712a875e 100644 --- a/src/hotspot/share/jfr/recorder/checkpoint/types/jfrTypeManager.cpp +++ b/src/hotspot/share/jfr/recorder/checkpoint/types/jfrTypeManager.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2016, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -23,6 +23,8 @@ */ #include "precompiled.hpp" +#include "classfile/systemDictionary.hpp" +#include "classfile/vmSymbols.hpp" #include "jfr/metadata/jfrSerializer.hpp" #include "jfr/recorder/checkpoint/jfrCheckpointWriter.hpp" #include "jfr/recorder/checkpoint/types/jfrType.hpp" @@ -32,11 +34,9 @@ #include "jfr/utilities/jfrIterator.hpp" #include "jfr/utilities/jfrLinkedList.inline.hpp" #include "memory/resourceArea.hpp" -#include "runtime/handles.inline.hpp" -#include "runtime/safepoint.hpp" #include "runtime/semaphore.hpp" #include "runtime/thread.inline.hpp" -#include "utilities/exceptions.hpp" +#include "utilities/macros.hpp" class JfrSerializerRegistration : public JfrCHeapObj { public: @@ -101,26 +101,34 @@ void JfrTypeManager::write_threads(JfrCheckpointWriter& writer) { serialize_thread_groups(writer); } -void JfrTypeManager::create_thread_blob(Thread* t) { - assert(t != NULL, "invariant"); - ResourceMark rm(t); - HandleMark hm(t); - JfrThreadConstant type_thread(t); - JfrCheckpointWriter writer(t, true, THREADS, false); +JfrBlobHandle JfrTypeManager::create_thread_blob(JavaThread* jt, traceid tid /* 0 */, oop vthread /* nullptr */) { + assert(jt != NULL, "invariant"); + ResourceMark rm(jt); + JfrCheckpointWriter writer(jt, true, THREADS, false); // Thread local lease for blob creation. + // TYPE_THREAD and count is written unconditionally for blobs, also for vthreads. writer.write_type(TYPE_THREAD); + writer.write_count(1); + JfrThreadConstant type_thread(jt, tid, vthread); type_thread.serialize(writer); - // create and install a checkpoint blob - t->jfr_thread_local()->set_thread_blob(writer.move()); - assert(t->jfr_thread_local()->has_thread_blob(), "invariant"); + return writer.move(); } -void JfrTypeManager::write_thread_checkpoint(Thread* t) { +void JfrTypeManager::write_checkpoint(Thread* t, traceid tid /* 0 */, oop vthread /* nullptr */) { assert(t != NULL, "invariant"); - if (!t->jfr_thread_local()->has_thread_blob()) { - create_thread_blob(t); + Thread* const current = Thread::current(); // not necessarily the same as t + assert(current != NULL, "invariant"); + const bool is_vthread = vthread != nullptr; + ResourceMark rm(current); + JfrCheckpointWriter writer(current, true, THREADS, !is_vthread); // Virtual Threads use thread local lease. + if (is_vthread) { + // TYPE_THREAD and count is written later as part of vthread bulk serialization. + writer.set_count(1); // Only a logical marker for the checkpoint header. + } else { + writer.write_type(TYPE_THREAD); + writer.write_count(1); } - JfrCheckpointWriter writer(t, false, THREADS, false); - t->jfr_thread_local()->thread_blob()->write(writer); + JfrThreadConstant type_thread(t, tid, vthread); + type_thread.serialize(writer); } class SerializerRegistrationGuard : public StackObj { @@ -201,6 +209,16 @@ static bool register_static_type(JfrTypeId id, bool permit_cache, JfrSerializer* return true; } +// This klass is explicitly loaded to ensure the thread group for virtual threads is available. +static bool load_thread_constants(TRAPS) { + Symbol* const thread_constants_sym = vmSymbols::java_lang_Thread_Constants(); + assert(thread_constants_sym != nullptr, "invariant"); + Klass* const k_thread_constants = SystemDictionary::resolve_or_fail(thread_constants_sym, false, CHECK_false); + assert(k_thread_constants != nullptr, "invariant"); + k_thread_constants->initialize(THREAD); + return true; +} + bool JfrTypeManager::initialize() { SerializerRegistrationGuard guard; register_static_type(TYPE_FLAGVALUEORIGIN, true, new FlagValueOriginConstant()); @@ -218,7 +236,7 @@ bool JfrTypeManager::initialize() { register_static_type(TYPE_THREADSTATE, true, new ThreadStateConstant()); register_static_type(TYPE_BYTECODE, true, new BytecodeConstant()); register_static_type(TYPE_COMPILERTYPE, true, new CompilerTypeConstant()); - return true; + return load_thread_constants(JavaThread::current()); } // implementation for the static registration function exposed in the JfrSerializer api diff --git a/src/hotspot/share/jfr/recorder/checkpoint/types/jfrTypeManager.hpp b/src/hotspot/share/jfr/recorder/checkpoint/types/jfrTypeManager.hpp index 407008e6f5c..90c314b0549 100644 --- a/src/hotspot/share/jfr/recorder/checkpoint/types/jfrTypeManager.hpp +++ b/src/hotspot/share/jfr/recorder/checkpoint/types/jfrTypeManager.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2016, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -25,9 +25,13 @@ #define SHARE_JFR_RECORDER_CHECKPOINT_TYPES_JFRTYPEMANAGER_HPP #include "jfr/utilities/jfrAllocation.hpp" +#include "jfr/utilities/jfrBlob.hpp" +#include "jfr/utilities/jfrTypes.hpp" +#include "oops/oopsHierarchy.hpp" class JavaThread; class JfrCheckpointWriter; +class Thread; class JfrTypeManager : public AllStatic { public: @@ -35,8 +39,8 @@ class JfrTypeManager : public AllStatic { static void destroy(); static void on_rotation(); static void write_threads(JfrCheckpointWriter& writer); - static void create_thread_blob(Thread* t); - static void write_thread_checkpoint(Thread* t); + static JfrBlobHandle create_thread_blob(JavaThread* jt, traceid tid = 0, oop vthread = nullptr); + static void write_checkpoint(Thread* t, traceid tid = 0, oop vthread = nullptr); static void write_static_types(JfrCheckpointWriter& writer); }; diff --git a/src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrOopTraceId.hpp b/src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrOopTraceId.hpp new file mode 100644 index 00000000000..c277bacf4ff --- /dev/null +++ b/src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrOopTraceId.hpp @@ -0,0 +1,45 @@ +/* +* Copyright (c) 2020, 2022, Oracle and/or its affiliates. All rights reserved. +* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +* +* This code is free software; you can redistribute it and/or modify it +* under the terms of the GNU General Public License version 2 only, as +* published by the Free Software Foundation. +* +* This code is distributed in the hope that it will be useful, but WITHOUT +* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +* version 2 for more details (a copy is included in the LICENSE file that +* accompanied this code). +* +* You should have received a copy of the GNU General Public License version +* 2 along with this work; if not, write to the Free Software Foundation, +* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. +* +* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA +* or visit www.oracle.com if you need additional information or have any +* questions. +* +*/ + +#ifndef SHARE_JFR_RECORDER_CHECKPOINT_TYPES_TRACEID_JFROOPTRACEID_HPP +#define SHARE_JFR_RECORDER_CHECKPOINT_TYPES_TRACEID_JFROOPTRACEID_HPP + +#include "jfr/utilities/jfrTypes.hpp" +#include "memory/allocation.hpp" +#include "oops/oopsHierarchy.hpp" + +template +class JfrOopTraceId : AllStatic { + public: + static traceid id(oop ref); + static u2 epoch(oop ref); + static u2 current_epoch(); + static void set_epoch(oop ref); + static void set_epoch(oop ref, u2 epoch); + static bool is_excluded(oop ref); + static void exclude(oop ref); + static void include(oop ref); +}; + +#endif // SHARE_JFR_RECORDER_CHECKPOINT_TYPES_TRACEID_JFROOPTRACEID_HPP diff --git a/src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrOopTraceId.inline.hpp b/src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrOopTraceId.inline.hpp new file mode 100644 index 00000000000..6c023a51b91 --- /dev/null +++ b/src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrOopTraceId.inline.hpp @@ -0,0 +1,75 @@ +/* +* Copyright (c) 2020, 2022, Oracle and/or its affiliates. All rights reserved. +* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +* +* This code is free software; you can redistribute it and/or modify it +* under the terms of the GNU General Public License version 2 only, as +* published by the Free Software Foundation. +* +* This code is distributed in the hope that it will be useful, but WITHOUT +* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +* version 2 for more details (a copy is included in the LICENSE file that +* accompanied this code). +* +* You should have received a copy of the GNU General Public License version +* 2 along with this work; if not, write to the Free Software Foundation, +* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. +* +* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA +* or visit www.oracle.com if you need additional information or have any +* questions. +* +*/ + +#ifndef SHARE_JFR_RECORDER_CHECKPOINT_TYPES_TRACEID_JFROOPTRACEID_INLINE_HPP +#define SHARE_JFR_RECORDER_CHECKPOINT_TYPES_TRACEID_JFROOPTRACEID_INLINE_HPP + +#include "jfr/recorder/checkpoint/types/traceid/jfrOopTraceId.hpp" + +#include "jfr/recorder/checkpoint/types/traceid/jfrTraceIdEpoch.hpp" + +template +inline traceid JfrOopTraceId::id(oop ref) { + assert(ref != nullptr, "invariant"); + return T::id(ref); +} + +template +inline u2 JfrOopTraceId::epoch(oop ref) { + assert(ref != nullptr, "invariant"); + return T::epoch(ref); +} + +template +inline u2 JfrOopTraceId::current_epoch() { + return JfrTraceIdEpoch::epoch_generation(); +} + +template +inline void JfrOopTraceId::set_epoch(oop ref, u2 epoch) { + assert(ref != nullptr, "invariant"); + T::set_epoch(ref, epoch); +} + +template +inline void JfrOopTraceId::set_epoch(oop ref) { + set_epoch(ref, JfrTraceIdEpoch::epoch_generation()); +} + +template +inline bool JfrOopTraceId::is_excluded(oop ref) { + return T::is_excluded(ref); +} + +template +inline void JfrOopTraceId::exclude(oop ref) { + T::exclude(ref); +} + +template +inline void JfrOopTraceId::include(oop ref) { + T::include(ref); +} + +#endif // SHARE_JFR_RECORDER_CHECKPOINT_TYPES_TRACEID_JFROOPTRACEID_INLINE_HPP diff --git a/src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrTraceId.cpp b/src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrTraceId.cpp index 1af21844179..660b0dc6968 100644 --- a/src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrTraceId.cpp +++ b/src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrTraceId.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2011, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -39,14 +39,14 @@ #include "runtime/thread.inline.hpp" #include "utilities/debug.hpp" - // returns updated value -static traceid atomic_inc(traceid volatile* const dest) { +// returns updated value +static traceid atomic_inc(traceid volatile* const dest, traceid stride = 1) { assert(VM_Version::supports_cx8(), "invariant"); traceid compare_value; traceid exchange_value; do { compare_value = *dest; - exchange_value = compare_value + 1; + exchange_value = compare_value + stride; } while (Atomic::cmpxchg(dest, compare_value, exchange_value) != compare_value); return exchange_value; } @@ -56,11 +56,6 @@ static traceid next_class_id() { return atomic_inc(&class_id_counter) << TRACE_ID_SHIFT; } -static traceid next_thread_id() { - static volatile traceid thread_id_counter = 0; - return atomic_inc(&thread_id_counter); -} - static traceid next_module_id() { static volatile traceid module_id_counter = 0; return atomic_inc(&module_id_counter) << TRACE_ID_SHIFT; @@ -152,10 +147,6 @@ traceid JfrTraceId::assign_primitive_klass_id() { return next_class_id(); } -traceid JfrTraceId::assign_thread_id() { - return next_thread_id(); -} - // A mirror representing a primitive class (e.g. int.class) has no reified Klass*, // instead it has an associated TypeArrayKlass* (e.g. int[].class). // We can use the TypeArrayKlass* as a proxy for deriving the id of the primitive class. diff --git a/src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrTraceId.hpp b/src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrTraceId.hpp index 456599b45f3..f662fbb5ca1 100644 --- a/src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrTraceId.hpp +++ b/src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrTraceId.hpp @@ -34,7 +34,6 @@ class Klass; class Method; class ModuleEntry; class PackageEntry; -class Thread; /* * JfrTraceId is a means of tagging, e.g. marking, specific instances as being actively in-use. @@ -84,7 +83,6 @@ class JfrTraceId : public AllStatic { static void assign(const PackageEntry* package); static void assign(const ClassLoaderData* cld); static traceid assign_primitive_klass_id(); - static traceid assign_thread_id(); // through load barrier static traceid load(const Klass* klass); @@ -99,7 +97,6 @@ class JfrTraceId : public AllStatic { // load barrier elision static traceid load_raw(const Klass* klass); static traceid load_raw(jclass jc); - static traceid load_raw(const Thread* thread); static traceid load_raw(const Method* method); static traceid load_raw(const ModuleEntry* module); static traceid load_raw(const PackageEntry* package); diff --git a/src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrTraceId.inline.hpp b/src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrTraceId.inline.hpp index 4cd3feba7ae..b9a13e76140 100644 --- a/src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrTraceId.inline.hpp +++ b/src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrTraceId.inline.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2016, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -74,11 +74,6 @@ inline traceid JfrTraceId::load_raw(const Klass* klass) { return raw_load(klass); } -inline traceid JfrTraceId::load_raw(const Thread* t) { - assert(t != NULL, "invariant"); - return TRACE_ID_RAW(t->jfr_thread_local()); -} - inline traceid JfrTraceId::load_raw(const Method* method) { return (METHOD_ID(method->method_holder(), method)); } diff --git a/src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrTraceIdEpoch.cpp b/src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrTraceIdEpoch.cpp index 03c3ad48ea2..328c3a7c9e1 100644 --- a/src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrTraceIdEpoch.cpp +++ b/src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrTraceIdEpoch.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2016, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -24,12 +24,23 @@ #include "precompiled.hpp" #include "jfr/recorder/checkpoint/types/traceid/jfrTraceIdEpoch.hpp" +#include "jfr/support/jfrThreadId.inline.hpp" #include "runtime/safepoint.hpp" +/* + * The epoch generation is the range [1-32767]. + * + * When the epoch value is stored in a thread object, + * the most significant bit of the u2 is used to denote + * thread exclusion, i.e 1 << 15 == 32768 denotes exclusion. +*/ +u2 JfrTraceIdEpoch::_generation = 0; JfrSignal JfrTraceIdEpoch::_tag_state; bool JfrTraceIdEpoch::_epoch_state = false; bool JfrTraceIdEpoch::_synchronizing = false; +static constexpr const u2 epoch_generation_overflow = excluded_bit; + void JfrTraceIdEpoch::begin_epoch_shift() { assert(SafepointSynchronize::is_at_safepoint(), "invariant"); _synchronizing = true; @@ -40,7 +51,12 @@ void JfrTraceIdEpoch::end_epoch_shift() { assert(SafepointSynchronize::is_at_safepoint(), "invariant"); assert(_synchronizing, "invariant"); _epoch_state = !_epoch_state; + ++_generation; + if (epoch_generation_overflow == _generation) { + _generation = 1; + } + assert(_generation != 0, "invariant"); + assert(_generation < epoch_generation_overflow, "invariant"); OrderAccess::storestore(); _synchronizing = false; } - diff --git a/src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrTraceIdEpoch.hpp b/src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrTraceIdEpoch.hpp index 1216f6956ef..f9dcf407d13 100644 --- a/src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrTraceIdEpoch.hpp +++ b/src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrTraceIdEpoch.hpp @@ -55,6 +55,7 @@ class JfrTraceIdEpoch : AllStatic { friend class JfrCheckpointManager; private: + static u2 _generation; static JfrSignal _tag_state; static bool _epoch_state; static bool _synchronizing; @@ -71,10 +72,22 @@ class JfrTraceIdEpoch : AllStatic { return (address)&_epoch_state; } + static address epoch_generation_address() { + return (address)&_generation; + } + static u1 current() { return _epoch_state ? (u1)1 : (u1)0; } + static u2 epoch_generation() { + return _generation; + } + + static bool is_current_epoch_generation(u2 generation) { + return _generation == generation; + } + static u1 previous() { return _epoch_state ? (u1)0 : (u1)1; } diff --git a/src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrTraceIdLoadBarrier.hpp b/src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrTraceIdLoadBarrier.hpp index e7b7e1a8780..13b8a579e1b 100644 --- a/src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrTraceIdLoadBarrier.hpp +++ b/src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrTraceIdLoadBarrier.hpp @@ -68,8 +68,8 @@ class PackageEntry; * */ class JfrTraceIdLoadBarrier : AllStatic { - friend class Jfr; friend class JfrCheckpointManager; + friend class JfrIntrinsicSupport; friend class JfrStackTrace; friend class JfrThreadSampler; private: diff --git a/src/hotspot/share/jfr/recorder/jfrRecorder.cpp b/src/hotspot/share/jfr/recorder/jfrRecorder.cpp index 6e50420c63e..3c5ba07fbfb 100644 --- a/src/hotspot/share/jfr/recorder/jfrRecorder.cpp +++ b/src/hotspot/share/jfr/recorder/jfrRecorder.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2012, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -41,6 +41,7 @@ #include "jfr/recorder/storage/jfrStorage.hpp" #include "jfr/recorder/stacktrace/jfrStackTraceRepository.hpp" #include "jfr/recorder/stringpool/jfrStringPool.hpp" +#include "jfr/support/jfrThreadLocal.hpp" #include "jfr/utilities/jfrTime.hpp" #include "jfr/writers/jfrJavaEventWriter.hpp" #include "logging/log.hpp" @@ -197,6 +198,8 @@ bool JfrRecorder::on_create_vm_2() { return true; } JavaThread* const thread = JavaThread::current(); + JfrThreadLocal::assign_thread_id(thread, thread->jfr_thread_local()); + if (!JfrOptionSet::initialize(thread)) { return false; } diff --git a/src/hotspot/share/jfr/recorder/repository/jfrEmergencyDump.cpp b/src/hotspot/share/jfr/recorder/repository/jfrEmergencyDump.cpp index 8aeea6bdb44..7f4314867a7 100644 --- a/src/hotspot/share/jfr/recorder/repository/jfrEmergencyDump.cpp +++ b/src/hotspot/share/jfr/recorder/repository/jfrEmergencyDump.cpp @@ -41,8 +41,6 @@ #include "utilities/growableArray.hpp" #include "utilities/ostream.hpp" -char JfrEmergencyDump::_dump_path[JVM_MAXPATHLEN] = { 0 }; - static const char vm_error_filename_fmt[] = "hs_err_pid%p.jfr"; static const char vm_oom_filename_fmt[] = "hs_oom_pid%p.jfr"; static const char vm_soe_filename_fmt[] = "hs_soe_pid%p.jfr"; @@ -51,6 +49,7 @@ static const size_t iso8601_len = 19; // "YYYY-MM-DDTHH:MM:SS" (note: we just us static fio_fd emergency_fd = invalid_fd; static const int64_t chunk_file_header_size = 68; static const size_t chunk_file_extension_length = sizeof chunk_file_jfr_ext - 1; +static char _dump_path[JVM_MAXPATHLEN] = { 0 }; /* * The emergency dump logic is restrictive when it comes to @@ -132,7 +131,7 @@ static const char* create_emergency_dump_path() { return result ? _path_buffer : NULL; } -bool JfrEmergencyDump::open_emergency_dump_file() { +static bool open_emergency_dump_file() { if (is_emergency_dump_file_open()) { // opened already return true; @@ -166,12 +165,12 @@ static void report(outputStream* st, bool emergency_file_opened, const char* rep } } -void JfrEmergencyDump::set_dump_path(const char* dump_path) { - if (dump_path == NULL || *dump_path == '\0') { +void JfrEmergencyDump::set_dump_path(const char* path) { + if (path == NULL || *path == '\0') { os::get_current_directory(_dump_path, sizeof(_dump_path)); } else { - if (strlen(dump_path) < JVM_MAXPATHLEN) { - strncpy(_dump_path, dump_path, JVM_MAXPATHLEN); + if (strlen(path) < JVM_MAXPATHLEN) { + strncpy(_dump_path, path, JVM_MAXPATHLEN); _dump_path[JVM_MAXPATHLEN - 1] = '\0'; } } diff --git a/src/hotspot/share/jfr/recorder/repository/jfrEmergencyDump.hpp b/src/hotspot/share/jfr/recorder/repository/jfrEmergencyDump.hpp index 7912daf35c9..7db4b511746 100644 --- a/src/hotspot/share/jfr/recorder/repository/jfrEmergencyDump.hpp +++ b/src/hotspot/share/jfr/recorder/repository/jfrEmergencyDump.hpp @@ -26,20 +26,16 @@ #define SHARE_JFR_RECORDER_REPOSITORY_JFREMERGENCYDUMP_HPP #include "memory/allStatic.hpp" -#include "utilities/ostream.hpp" + +class outputStream; // // Responsible for creating an hs_err.jfr file in exceptional shutdown situations (crash, OOM) // class JfrEmergencyDump : AllStatic { - private: - static char _dump_path[JVM_MAXPATHLEN]; - - static bool open_emergency_dump_file(); - public: - static void set_dump_path(const char* dump_path); static const char* get_dump_path(); + static void set_dump_path(const char* dump_path); static const char* chunk_path(const char* repository_path); static void on_vm_error(const char* repository_path); static void on_vm_error_report(outputStream* st, const char* repository_path); diff --git a/src/hotspot/share/jfr/recorder/service/jfrEvent.hpp b/src/hotspot/share/jfr/recorder/service/jfrEvent.hpp index e87c34c2530..3a8562d25de 100644 --- a/src/hotspot/share/jfr/recorder/service/jfrEvent.hpp +++ b/src/hotspot/share/jfr/recorder/service/jfrEvent.hpp @@ -43,7 +43,7 @@ class JfrEventVerifier { template friend class JfrEvent; private: - // verification of event fields + // Verification of fields. BitMap::bm_word_t _verification_storage[1]; BitMapView _verification_bit_map; bool _committed; @@ -76,7 +76,7 @@ class JfrEvent { , _verifier() #endif { - if (T::is_enabled()) { + if (T::is_enabled() && JfrThreadLocal::is_included(Thread::current())) { _started = true; if (TIMED == timing && !T::isInstant) { set_starttime(JfrTicks::now()); @@ -186,31 +186,51 @@ class JfrEvent { return T::hasThrottle ? JfrEventThrottler::accept(T::eventId, _untimed ? 0 : _end_time) : true; } + traceid thread_id(Thread* thread) { + return T::hasThread ? JfrThreadLocal::thread_id(thread) : 0; + } + + traceid stack_trace_id(Thread* thread, const JfrThreadLocal* tl) { + return T::hasStackTrace && is_stacktrace_enabled() ? + tl->has_cached_stack_trace() ? tl->cached_stack_trace_id() : + JfrStackTraceRepository::record(thread) : 0; + } + + /* + * Support for virtual threads involves oops, access of which may trigger + * events, i.e. load barriers. Hence, write_event() must be re-entrant + * for recursion. Getting the thread id and capturing a stacktrace may + * involve oop access, and are therefore hoisted before claiming a buffer + * and binding it to a writer. + */ void write_event() { DEBUG_ONLY(assert_precondition();) - Thread* const event_thread = Thread::current(); - JfrThreadLocal* const tl = event_thread->jfr_thread_local(); + Thread* const thread = Thread::current(); + JfrThreadLocal* const tl = thread->jfr_thread_local(); + const traceid tid = thread_id(thread); + const traceid sid = stack_trace_id(thread, tl); + // Keep tid and sid above this line. JfrBuffer* const buffer = tl->native_buffer(); - if (buffer == NULL) { - // most likely a pending OOM + if (buffer == nullptr) { + // Most likely a pending OOM. return; } bool large = is_large(); - if (write_sized_event(buffer, event_thread, tl, large)) { + if (write_sized_event(buffer, thread, tid, sid, large)) { // Event written successfully return; } if (!large) { - // Try large size - if (write_sized_event(buffer, event_thread, tl, true)) { - // Event written successfully, use large size from now on + // Try large size. + if (write_sized_event(buffer, thread, tid, sid, true)) { + // Event written successfully, use large size from now on. set_large(); } } } - bool write_sized_event(JfrBuffer* const buffer, Thread* const event_thread, JfrThreadLocal* const tl, bool large_size) { - JfrNativeEventWriter writer(buffer, event_thread); + bool write_sized_event(JfrBuffer* buffer, Thread* thread, traceid tid, traceid sid, bool large_size) { + JfrNativeEventWriter writer(buffer, thread); writer.begin_event_write(large_size); writer.write(T::eventId); assert(_start_time != 0, "invariant"); @@ -220,27 +240,19 @@ class JfrEvent { writer.write(_end_time - _start_time); } if (T::hasThread) { - writer.write(tl->thread_id()); + writer.write(tid); } if (T::hasStackTrace) { - if (is_stacktrace_enabled()) { - if (tl->has_cached_stack_trace()) { - writer.write(tl->cached_stack_trace_id()); - } else { - writer.write(JfrStackTraceRepository::record(event_thread)); - } - } else { - writer.write(0); - } + writer.write(sid); } - // payload + // Payload. static_cast(this)->writeData(writer); return writer.end_event_write(large_size) > 0; } #ifdef ASSERT private: - // verification of event fields + // Verification of fields. JfrEventVerifier _verifier; void assert_precondition() { @@ -252,8 +264,8 @@ class JfrEvent { protected: void set_field_bit(size_t field_idx) { _verifier.set_field_bit(field_idx); - // it is ok to reuse an already committed event - // granted you provide new informational content + // It is ok to reuse an already committed event + // granted you provide new informational content. _verifier.clear_committed(); } diff --git a/src/hotspot/share/jfr/recorder/service/jfrRecorderThread.cpp b/src/hotspot/share/jfr/recorder/service/jfrRecorderThread.cpp index 00f504a81e5..e4b615be81a 100644 --- a/src/hotspot/share/jfr/recorder/service/jfrRecorderThread.cpp +++ b/src/hotspot/share/jfr/recorder/service/jfrRecorderThread.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2012, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -23,21 +23,16 @@ */ #include "precompiled.hpp" -#include "jni.h" -#include "classfile/javaClasses.hpp" -#include "classfile/symbolTable.hpp" #include "classfile/systemDictionary.hpp" #include "classfile/vmClasses.hpp" #include "classfile/vmSymbols.hpp" -#include "jfr/jfr.hpp" #include "jfr/jni/jfrJavaSupport.hpp" -#include "jfr/recorder/jfrRecorder.hpp" #include "jfr/recorder/checkpoint/jfrCheckpointManager.hpp" +#include "jfr/recorder/service/jfrPostBox.hpp" #include "jfr/recorder/service/jfrRecorderThread.hpp" #include "memory/resourceArea.hpp" #include "memory/universe.hpp" #include "runtime/handles.inline.hpp" -#include "runtime/mutexLocker.hpp" #include "runtime/thread.inline.hpp" #include "utilities/preserveException.hpp" #include "utilities/macros.hpp" @@ -91,7 +86,6 @@ bool JfrRecorderThread::start(JfrCheckpointManager* cp_manager, JfrPostBox* post // attempt thread start Thread* const t = start_thread(h_thread_oop, recorderthread_entry,THREAD); if (!HAS_PENDING_EXCEPTION) { - Jfr::exclude_thread(t); return true; } assert(HAS_PENDING_EXCEPTION, "invariant"); diff --git a/src/hotspot/share/jfr/recorder/stacktrace/jfrStackTrace.cpp b/src/hotspot/share/jfr/recorder/stacktrace/jfrStackTrace.cpp index 925607cd6d5..86e90bb496d 100644 --- a/src/hotspot/share/jfr/recorder/stacktrace/jfrStackTrace.cpp +++ b/src/hotspot/share/jfr/recorder/stacktrace/jfrStackTrace.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2011, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -29,8 +29,12 @@ #include "jfr/recorder/stacktrace/jfrStackTrace.hpp" #include "jfr/recorder/storage/jfrBuffer.hpp" #include "jfr/support/jfrMethodLookup.hpp" +#include "jfr/support/jfrThreadLocal.hpp" #include "memory/allocation.inline.hpp" #include "oops/instanceKlass.inline.hpp" +#include "runtime/continuation.hpp" +#include "runtime/continuationEntry.inline.hpp" +#include "runtime/handles.inline.hpp" #include "runtime/vframe.inline.hpp" static void copy_frames(JfrStackFrame** lhs_frames, u4 length, const JfrStackFrame* rhs_frames) { @@ -132,21 +136,61 @@ void JfrStackFrame::write(JfrCheckpointWriter& cpw) const { write_frame(cpw, _methodid, _line, _bci, _type); } -class vframeStreamSamples : public vframeStreamCommon { +class JfrVframeStream : public vframeStreamCommon { + private: + const ContinuationEntry* _cont_entry; + bool _async_mode; + bool _vthread; + bool step_to_sender(); + void next_frame(); public: - // constructor that starts with sender of frame fr (top_frame) - vframeStreamSamples(JavaThread *jt, frame fr, bool stop_at_java_call_stub) : vframeStreamCommon(jt, false /* process_frames */) { - _stop_at_java_call_stub = stop_at_java_call_stub; - _frame = fr; - - // We must always have a valid frame to start filling - bool filled_in = fill_from_frame(); - assert(filled_in, "invariant"); - } - void samples_next(); - void stop() {} + JfrVframeStream(JavaThread* jt, const frame& fr, bool stop_at_java_call_stub, bool async_mode); + void next_vframe(); }; +JfrVframeStream::JfrVframeStream(JavaThread* jt, const frame& fr, bool stop_at_java_call_stub, bool async_mode) : + vframeStreamCommon(RegisterMap(jt, false, false, true)), _cont_entry(JfrThreadLocal::is_vthread(jt) ? jt->last_continuation() : nullptr), + _async_mode(async_mode), _vthread(JfrThreadLocal::is_vthread(jt)) { + assert(!_vthread || _cont_entry != nullptr, "invariant"); + _reg_map.set_async(async_mode); + _frame = fr; + _stop_at_java_call_stub = stop_at_java_call_stub; + while (!fill_from_frame()) { + step_to_sender(); + } +} + +inline bool JfrVframeStream::step_to_sender() { + if (_async_mode && !_frame.safe_for_sender(_thread)) { + _mode = at_end_mode; + return false; + } + _frame = _frame.sender(&_reg_map); + return true; +} + +inline void JfrVframeStream::next_frame() { + static constexpr const u4 loop_max = MAX_STACK_DEPTH * 2; + u4 loop_count = 0; + do { + if (_vthread && Continuation::is_continuation_enterSpecial(_frame)) { + if (_cont_entry->is_virtual_thread()) { + // An entry of a vthread continuation is a termination point. + _mode = at_end_mode; + break; + } + _cont_entry = _cont_entry->parent(); + } + if (_async_mode) { + ++loop_count; + if (loop_count > loop_max) { + _mode = at_end_mode; + break; + } + } + } while (step_to_sender() && !fill_from_frame()); +} + // Solaris SPARC Compiler1 needs an additional check on the grandparent // of the top_frame when the parent of the top_frame is interpreted and // the grandparent is compiled. However, in this method we do not know @@ -155,25 +199,12 @@ class vframeStreamSamples : public vframeStreamCommon { // interpreted and the current sender is compiled, we verify that the // current sender is also walkable. If it is not walkable, then we mark // the current vframeStream as at the end. -void vframeStreamSamples::samples_next() { +void JfrVframeStream::next_vframe() { // handle frames with inlining - if (_mode == compiled_mode && - vframeStreamCommon::fill_in_compiled_inlined_sender()) { + if (_mode == compiled_mode && fill_in_compiled_inlined_sender()) { return; } - - // handle general case - u4 loop_count = 0; - u4 loop_max = MAX_STACK_DEPTH * 2; - do { - loop_count++; - // By the time we get here we should never see unsafe but better safe then segv'd - if (loop_count > loop_max || !_frame.safe_for_sender(_thread)) { - _mode = at_end_mode; - return; - } - _frame = _frame.sender(&_reg_map); - } while (!fill_from_frame()); + next_frame(); } static const size_t min_valid_free_size_bytes = 16; @@ -182,86 +213,73 @@ static inline bool is_full(const JfrBuffer* enqueue_buffer) { return enqueue_buffer->free_size() < min_valid_free_size_bytes; } -bool JfrStackTrace::record_thread(JavaThread& thread, frame& frame) { +bool JfrStackTrace::record_async(JavaThread* jt, const frame& frame) { + assert(jt != NULL, "invariant"); + assert(!_lineno, "invariant"); + Thread* current_thread = Thread::current(); + assert(jt != current_thread, "invariant"); // Explicitly monitor the available space of the thread-local buffer used for enqueuing klasses as part of tagging methods. // We do this because if space becomes sparse, we cannot rely on the implicit allocation of a new buffer as part of the // regular tag mechanism. If the free list is empty, a malloc could result, and the problem with that is that the thread // we have suspended could be the holder of the malloc lock. If there is no more available space, the attempt is aborted. - const JfrBuffer* const enqueue_buffer = JfrTraceIdLoadBarrier::get_enqueue_buffer(Thread::current()); - assert(enqueue_buffer != nullptr, "invariant"); - vframeStreamSamples st(&thread, frame, false); + const JfrBuffer* const enqueue_buffer = JfrTraceIdLoadBarrier::get_enqueue_buffer(current_thread); + HandleMark hm(current_thread); // RegisterMap uses Handles to support continuations. + JfrVframeStream vfs(jt, frame, false, true); u4 count = 0; _reached_root = true; _hash = 1; - while (!st.at_end()) { + while (!vfs.at_end()) { if (count >= _max_frames) { _reached_root = false; break; } - const Method* method = st.method(); + const Method* method = vfs.method(); if (!Method::is_valid_method(method) || is_full(enqueue_buffer)) { // we throw away everything we've gathered in this sample since // none of it is safe return false; } const traceid mid = JfrTraceId::load(method); - int type = st.is_interpreted_frame() ? JfrStackFrame::FRAME_INTERPRETER : JfrStackFrame::FRAME_JIT; + int type = vfs.is_interpreted_frame() ? JfrStackFrame::FRAME_INTERPRETER : JfrStackFrame::FRAME_JIT; int bci = 0; if (method->is_native()) { type = JfrStackFrame::FRAME_NATIVE; } else { - bci = st.bci(); + bci = vfs.bci(); } - intptr_t* frame_id = st.frame_id(); - st.samples_next(); - if (type == JfrStackFrame::FRAME_JIT && !st.at_end() && frame_id == st.frame_id()) { + intptr_t* frame_id = vfs.frame_id(); + vfs.next_vframe(); + if (type == JfrStackFrame::FRAME_JIT && !vfs.at_end() && frame_id == vfs.frame_id()) { // This frame and the caller frame are both the same physical // frame, so this frame is inlined into the caller. type = JfrStackFrame::FRAME_INLINE; } - - const int lineno = method->line_number_from_bci(bci); _hash = (_hash * 31) + mid; _hash = (_hash * 31) + bci; _hash = (_hash * 31) + type; - _frames[count] = JfrStackFrame(mid, bci, type, lineno, method->method_holder()); + _frames[count] = JfrStackFrame(mid, bci, type, method->line_number_from_bci(bci), method->method_holder()); count++; } - _lineno = true; _nr_of_frames = count; - return true; + return count > 0; } -void JfrStackFrame::resolve_lineno() const { - assert(_klass, "no klass pointer"); - assert(_line == 0, "already have linenumber"); - const Method* const method = JfrMethodLookup::lookup(_klass, _methodid); - assert(method != NULL, "invariant"); - assert(method->method_holder() == _klass, "invariant"); - _line = method->line_number_from_bci(_bci); -} - -void JfrStackTrace::resolve_linenos() const { - for (unsigned int i = 0; i < _nr_of_frames; i++) { - _frames[i].resolve_lineno(); - } - _lineno = true; -} - -bool JfrStackTrace::record_safe(JavaThread* thread, int skip) { - assert(thread == Thread::current(), "Thread stack needs to be walkable"); - vframeStream vfs(thread, false /* stop_at_java_call_stub */, false /* process_frames */); +bool JfrStackTrace::record(JavaThread* jt, const frame& frame, int skip) { + assert(jt != NULL, "invariant"); + assert(jt == Thread::current(), "invariant"); + assert(!_lineno, "invariant"); + HandleMark hm(jt); // RegisterMap uses Handles to support continuations. + JfrVframeStream vfs(jt, frame, false, false); u4 count = 0; _reached_root = true; - for (int i = 0; i < skip; i++) { + for (int i = 0; i < skip; ++i) { if (vfs.at_end()) { break; } - vfs.next(); + vfs.next_vframe(); } - _hash = 1; while (!vfs.at_end()) { if (count >= _max_frames) { @@ -274,26 +292,49 @@ bool JfrStackTrace::record_safe(JavaThread* thread, int skip) { int bci = 0; if (method->is_native()) { type = JfrStackFrame::FRAME_NATIVE; - } - else { + } else { bci = vfs.bci(); } + intptr_t* frame_id = vfs.frame_id(); - vfs.next(); + vfs.next_vframe(); if (type == JfrStackFrame::FRAME_JIT && !vfs.at_end() && frame_id == vfs.frame_id()) { // This frame and the caller frame are both the same physical // frame, so this frame is inlined into the caller. type = JfrStackFrame::FRAME_INLINE; } - _hash = (_hash * 31) + mid; _hash = (_hash * 31) + bci; _hash = (_hash * 31) + type; _frames[count] = JfrStackFrame(mid, bci, type, method->method_holder()); count++; } - _nr_of_frames = count; - return true; + return count > 0; } +bool JfrStackTrace::record(JavaThread* current_thread, int skip) { + assert(current_thread != NULL, "invariant"); + assert(current_thread == Thread::current(), "invariant"); + if (!current_thread->has_last_Java_frame()) { + return false; + } + return record(current_thread, current_thread->last_frame(), skip); +} + +void JfrStackFrame::resolve_lineno() const { + assert(_klass, "no klass pointer"); + assert(_line == 0, "already have linenumber"); + const Method* const method = JfrMethodLookup::lookup(_klass, _methodid); + assert(method != NULL, "invariant"); + assert(method->method_holder() == _klass, "invariant"); + _line = method->line_number_from_bci(_bci); +} + +void JfrStackTrace::resolve_linenos() const { + assert(!_lineno, "invariant"); + for (unsigned int i = 0; i < _nr_of_frames; i++) { + _frames[i].resolve_lineno(); + } + _lineno = true; +} diff --git a/src/hotspot/share/jfr/recorder/stacktrace/jfrStackTrace.hpp b/src/hotspot/share/jfr/recorder/stacktrace/jfrStackTrace.hpp index c3aabca4038..49e7d7a1920 100644 --- a/src/hotspot/share/jfr/recorder/stacktrace/jfrStackTrace.hpp +++ b/src/hotspot/share/jfr/recorder/stacktrace/jfrStackTrace.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2011, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -93,8 +93,9 @@ class JfrStackTrace : public JfrCHeapObj { void set_reached_root(bool reached_root) { _reached_root = reached_root; } void resolve_linenos() const; - bool record_thread(JavaThread& thread, frame& frame); - bool record_safe(JavaThread* thread, int skip); + bool record(JavaThread* current_thread, int skip); + bool record(JavaThread* current_thread, const frame& frame, int skip); + bool record_async(JavaThread* other_thread, const frame& frame); bool have_lineno() const { return _lineno; } bool full_stacktrace() const { return _reached_root; } diff --git a/src/hotspot/share/jfr/recorder/stacktrace/jfrStackTraceRepository.cpp b/src/hotspot/share/jfr/recorder/stacktrace/jfrStackTraceRepository.cpp index 4685cca8312..162a65376e3 100644 --- a/src/hotspot/share/jfr/recorder/stacktrace/jfrStackTraceRepository.cpp +++ b/src/hotspot/share/jfr/recorder/stacktrace/jfrStackTraceRepository.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2011, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -146,14 +146,14 @@ size_t JfrStackTraceRepository::clear(JfrStackTraceRepository& repo) { return processed; } -traceid JfrStackTraceRepository::record(Thread* thread, int skip /* 0 */) { - assert(thread == Thread::current(), "invariant"); - JfrThreadLocal* const tl = thread->jfr_thread_local(); +traceid JfrStackTraceRepository::record(Thread* current_thread, int skip /* 0 */) { + assert(current_thread == Thread::current(), "invariant"); + JfrThreadLocal* const tl = current_thread->jfr_thread_local(); assert(tl != NULL, "invariant"); if (tl->has_cached_stack_trace()) { return tl->cached_stack_trace_id(); } - if (!thread->is_Java_thread() || thread->is_hidden_from_external_view() || tl->is_excluded()) { + if (!current_thread->is_Java_thread() || current_thread->is_hidden_from_external_view()) { return 0; } JfrStackFrame* frames = tl->stackframes(); @@ -163,12 +163,12 @@ traceid JfrStackTraceRepository::record(Thread* thread, int skip /* 0 */) { } assert(frames != NULL, "invariant"); assert(tl->stackframes() == frames, "invariant"); - return instance().record_for(JavaThread::cast(thread), skip, frames, tl->stackdepth()); + return instance().record(JavaThread::cast(current_thread), skip, frames, tl->stackdepth()); } -traceid JfrStackTraceRepository::record_for(JavaThread* thread, int skip, JfrStackFrame *frames, u4 max_frames) { +traceid JfrStackTraceRepository::record(JavaThread* current_thread, int skip, JfrStackFrame *frames, u4 max_frames) { JfrStackTrace stacktrace(frames, max_frames); - return stacktrace.record_safe(thread, skip) ? add(instance(), stacktrace) : 0; + return stacktrace.record(current_thread, skip) ? add(instance(), stacktrace) : 0; } traceid JfrStackTraceRepository::add(JfrStackTraceRepository& repo, const JfrStackTrace& stacktrace) { traceid tid = repo.add_trace(stacktrace); @@ -184,13 +184,14 @@ traceid JfrStackTraceRepository::add(const JfrStackTrace& stacktrace) { return add(instance(), stacktrace); } -void JfrStackTraceRepository::record_for_leak_profiler(JavaThread* thread, int skip /* 0 */) { - assert(thread != NULL, "invariant"); - JfrThreadLocal* const tl = thread->jfr_thread_local(); +void JfrStackTraceRepository::record_for_leak_profiler(JavaThread* current_thread, int skip /* 0 */) { + assert(current_thread != NULL, "invariant"); + assert(current_thread == Thread::current(), "invariant"); + JfrThreadLocal* const tl = current_thread->jfr_thread_local(); assert(tl != NULL, "invariant"); assert(!tl->has_cached_stack_trace(), "invariant"); JfrStackTrace stacktrace(tl->stackframes(), tl->stackdepth()); - stacktrace.record_safe(thread, skip); + stacktrace.record(current_thread, skip); const unsigned int hash = stacktrace.hash(); if (hash != 0) { tl->set_cached_stack_trace_id(add(leak_profiler_instance(), stacktrace), hash); @@ -199,6 +200,7 @@ void JfrStackTraceRepository::record_for_leak_profiler(JavaThread* thread, int s traceid JfrStackTraceRepository::add_trace(const JfrStackTrace& stacktrace) { MutexLocker lock(JfrStacktrace_lock, Mutex::_no_safepoint_check_flag); + assert(stacktrace._nr_of_frames > 0, "invariant"); const size_t index = stacktrace._hash % TABLE_SIZE; const JfrStackTrace* table_entry = _table[index]; diff --git a/src/hotspot/share/jfr/recorder/stacktrace/jfrStackTraceRepository.hpp b/src/hotspot/share/jfr/recorder/stacktrace/jfrStackTraceRepository.hpp index 92ac39c158e..ba0f966ed44 100644 --- a/src/hotspot/share/jfr/recorder/stacktrace/jfrStackTraceRepository.hpp +++ b/src/hotspot/share/jfr/recorder/stacktrace/jfrStackTraceRepository.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2011, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -67,10 +67,10 @@ class JfrStackTraceRepository : public JfrCHeapObj { traceid add_trace(const JfrStackTrace& stacktrace); static traceid add(JfrStackTraceRepository& repo, const JfrStackTrace& stacktrace); static traceid add(const JfrStackTrace& stacktrace); - traceid record_for(JavaThread* thread, int skip, JfrStackFrame* frames, u4 max_frames); + traceid record(JavaThread* current_thread, int skip, JfrStackFrame* frames, u4 max_frames); public: - static traceid record(Thread* thread, int skip = 0); + static traceid record(Thread* current_thread, int skip = 0); }; #endif // SHARE_JFR_RECORDER_STACKTRACE_JFRSTACKTRACEREPOSITORY_HPP diff --git a/src/hotspot/share/jfr/recorder/storage/jfrBuffer.cpp b/src/hotspot/share/jfr/recorder/storage/jfrBuffer.cpp index 838d126c00a..2e1c8c600a2 100644 --- a/src/hotspot/share/jfr/recorder/storage/jfrBuffer.cpp +++ b/src/hotspot/share/jfr/recorder/storage/jfrBuffer.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2011, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -53,14 +53,6 @@ bool JfrBuffer::initialize(size_t header_size, size_t size) { void JfrBuffer::reinitialize(bool exclusion /* false */) { acquire_critical_section_top(); - if (exclusion != excluded()) { - // update - if (exclusion) { - set_excluded(); - } else { - clear_excluded(); - } - } set_pos(start()); release_critical_section_top(start()); clear_retired(); @@ -242,24 +234,6 @@ void JfrBuffer::clear_lease() { assert(!lease(), "invariant"); } -bool JfrBuffer::excluded() const { - return test(&_flags, EXCLUDED); -} - -void JfrBuffer::set_excluded() { - assert(acquired_by_self(), "invariant"); - set(&_flags, EXCLUDED); - assert(excluded(), "invariant"); -} - -void JfrBuffer::clear_excluded() { - if (excluded()) { - assert(identity() != NULL, "invariant"); - clear(&_flags, EXCLUDED); - } - assert(!excluded(), "invariant"); -} - bool JfrBuffer::retired() const { return test(&_flags, RETIRED); } diff --git a/src/hotspot/share/jfr/recorder/storage/jfrBuffer.hpp b/src/hotspot/share/jfr/recorder/storage/jfrBuffer.hpp index 120c7311321..154920c30e2 100644 --- a/src/hotspot/share/jfr/recorder/storage/jfrBuffer.hpp +++ b/src/hotspot/share/jfr/recorder/storage/jfrBuffer.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2012, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -170,10 +170,6 @@ class JfrBuffer { void set_retired(); void clear_retired(); - bool excluded() const; - void set_excluded(); - void clear_excluded(); - u1 context() const; void set_context(u1 context); void clear_context(); diff --git a/src/hotspot/share/jfr/recorder/storage/jfrEpochStorage.inline.hpp b/src/hotspot/share/jfr/recorder/storage/jfrEpochStorage.inline.hpp index f4b50f8e989..ff5c6c3ac2b 100644 --- a/src/hotspot/share/jfr/recorder/storage/jfrEpochStorage.inline.hpp +++ b/src/hotspot/share/jfr/recorder/storage/jfrEpochStorage.inline.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -75,8 +75,7 @@ template void JfrEpochStorageHost::iterate(Functor& functor, bool previous_epoch) { typedef ReinitializeAllReleaseRetiredOp PreviousEpochReleaseOperation; typedef CompositeOperation PreviousEpochOperation; - typedef ReleaseRetiredOp CurrentEpochReleaseOperation; - typedef CompositeOperation CurrentEpochOperation; + typedef ReleaseRetiredOp CurrentEpochOperation; if (previous_epoch) { PreviousEpochReleaseOperation pero(_mspace, _mspace->live_list(true)); PreviousEpochOperation peo(&functor, &pero); @@ -84,8 +83,7 @@ void JfrEpochStorageHost::iterate(Funct return; } if (EagerReclaim) { - CurrentEpochReleaseOperation cero(_mspace, _mspace->live_list()); - CurrentEpochOperation ceo(&functor, &cero); + CurrentEpochOperation ceo(functor, _mspace, _mspace->live_list()); process_live_list(ceo, _mspace, false); // current epoch list return; } diff --git a/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp b/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp index dde81d9c188..95e6c114efa 100644 --- a/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp +++ b/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2016, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -560,7 +560,6 @@ inline bool ScavengingReleaseOp::excise_with_release(typename List assert(node->identity() != NULL, "invariant"); assert(node->empty(), "invariant"); assert(!node->lease(), "invariant"); - assert(!node->excluded(), "invariant"); ++_count; _amount += node->total_size(); node->clear_retired(); @@ -569,23 +568,27 @@ inline bool ScavengingReleaseOp::excise_with_release(typename List return true; } -template +template class ReleaseRetiredOp : public StackObj { private: + Functor& _functor; Mspace* _mspace; FromList& _list; typename Mspace::NodePtr _prev; public: typedef typename Mspace::Node Node; - ReleaseRetiredOp(Mspace* mspace, FromList& list) : - _mspace(mspace), _list(list), _prev(NULL) {} + ReleaseRetiredOp(Functor& functor, Mspace* mspace, FromList& list) : + _functor(functor), _mspace(mspace), _list(list), _prev(NULL) {} bool process(Node* node); }; -template -inline bool ReleaseRetiredOp::process(typename Mspace::Node* node) { +template +inline bool ReleaseRetiredOp::process(typename Mspace::Node* node) { assert(node != NULL, "invariant"); - if (node->retired()) { + const bool is_retired = node->retired(); + const bool result = _functor.process(node); + if (is_retired) { + assert(node->unflushed_size() == 0, "invariant"); _prev = _list.excise(_prev, node); node->reinitialize(); assert(node->empty(), "invariant"); @@ -595,7 +598,7 @@ inline bool ReleaseRetiredOp::process(typename Mspace::Node* n } else { _prev = node; } - return true; + return result; } template diff --git a/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp b/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp index f83132807dd..512d2ad8ab0 100644 --- a/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp +++ b/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2012, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -241,17 +241,6 @@ bool JfrStorage::flush_regular_buffer(BufferPtr buffer, Thread* thread) { return true; } - if (buffer->excluded()) { - const bool thread_is_excluded = thread->jfr_thread_local()->is_excluded(); - buffer->reinitialize(thread_is_excluded); - assert(buffer->empty(), "invariant"); - if (!thread_is_excluded) { - // state change from exclusion to inclusion requires a thread checkpoint - JfrCheckpointManager::write_thread_checkpoint(thread); - } - return true; - } - BufferPtr const promotion_buffer = acquire_promotion_buffer(unflushed_size, _global_mspace, *this, promotion_retry, thread); if (promotion_buffer == NULL) { write_data_loss(buffer, thread); @@ -307,7 +296,6 @@ void JfrStorage::release(BufferPtr buffer, Thread* thread) { } assert(buffer->empty(), "invariant"); assert(buffer->identity() != NULL, "invariant"); - buffer->clear_excluded(); buffer->set_retired(); } @@ -384,7 +372,6 @@ static void assert_flush_large_precondition(ConstBufferPtr cur, const u1* const assert(t != NULL, "invariant"); assert(cur != NULL, "invariant"); assert(cur->lease(), "invariant"); - assert(!cur->excluded(), "invariant"); assert(cur_pos != NULL, "invariant"); assert(native ? t->jfr_thread_local()->native_buffer() == cur : t->jfr_thread_local()->java_buffer() == cur, "invariant"); assert(t->jfr_thread_local()->shelved_buffer() != NULL, "invariant"); @@ -410,9 +397,6 @@ BufferPtr JfrStorage::flush_regular(BufferPtr cur, const u1* const cur_pos, size // possible and valid to migrate data after the flush. This is however only // the case for stable thread local buffers; it is not the case for large buffers. flush_regular_buffer(cur, t); - if (cur->excluded()) { - return cur; - } if (cur->free_size() >= req) { // simplest case, no switching of buffers if (used > 0) { @@ -502,23 +486,19 @@ typedef UnBufferedWriteToChunk WriteOperation; typedef MutexedWriteOp MutexedWriteOperation; typedef ConcurrentWriteOp ConcurrentWriteOperation; -typedef Excluded NonExcluded; -typedef PredicatedConcurrentWriteOp ConcurrentNonExcludedWriteOperation; - typedef ScavengingReleaseOp ReleaseThreadLocalOperation; -typedef CompositeOperation ConcurrentWriteReleaseThreadLocalOperation; +typedef CompositeOperation ConcurrentWriteReleaseThreadLocalOperation; size_t JfrStorage::write() { const size_t full_elements = write_full(); WriteOperation wo(_chunkwriter); - NonExcluded ne; - ConcurrentNonExcludedWriteOperation cnewo(wo, ne); + ConcurrentWriteOperation cwo(wo); ReleaseThreadLocalOperation rtlo(_thread_local_mspace, _thread_local_mspace->live_list()); - ConcurrentWriteReleaseThreadLocalOperation tlop(&cnewo, &rtlo); + ConcurrentWriteReleaseThreadLocalOperation tlop(&cwo, &rtlo); process_live_list(tlop, _thread_local_mspace); assert(_global_mspace->free_list_is_empty(), "invariant"); assert(_global_mspace->live_list_is_nonempty(), "invariant"); - process_live_list(cnewo, _global_mspace); + process_live_list(cwo, _global_mspace); return full_elements + wo.elements(); } @@ -526,12 +506,11 @@ size_t JfrStorage::write_at_safepoint() { assert(SafepointSynchronize::is_at_safepoint(), "invariant"); const size_t full_elements = write_full(); WriteOperation wo(_chunkwriter); - NonExcluded ne; - ConcurrentNonExcludedWriteOperation cnewo(wo, ne); // concurrent because of gc's - process_live_list(cnewo, _thread_local_mspace); + ConcurrentWriteOperation cwo(wo); // concurrent because of gc's + process_live_list(cwo, _thread_local_mspace); assert(_global_mspace->free_list_is_empty(), "invariant"); assert(_global_mspace->live_list_is_nonempty(), "invariant"); - process_live_list(cnewo, _global_mspace); + process_live_list(cwo, _global_mspace); return full_elements + wo.elements(); } diff --git a/src/hotspot/share/jfr/recorder/storage/jfrStorageUtils.hpp b/src/hotspot/share/jfr/recorder/storage/jfrStorageUtils.hpp index 36ae38b6c2b..0ac44944bdd 100644 --- a/src/hotspot/share/jfr/recorder/storage/jfrStorageUtils.hpp +++ b/src/hotspot/share/jfr/recorder/storage/jfrStorageUtils.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2016, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -104,16 +104,6 @@ class Retired { } }; -template -class Excluded { - public: - typedef T Type; - bool process(Type* t) { - assert(t != NULL, "invariant"); - return negation ? !t->excluded() : t->excluded(); - } -}; - template class MutexedWriteOp { private: diff --git a/src/hotspot/share/jfr/support/jfrEpochSynchronization.hpp b/src/hotspot/share/jfr/support/jfrEpochSynchronization.hpp index 7dd35912119..1e4ede051d2 100644 --- a/src/hotspot/share/jfr/support/jfrEpochSynchronization.hpp +++ b/src/hotspot/share/jfr/support/jfrEpochSynchronization.hpp @@ -1,5 +1,5 @@ /* -* Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. +* Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -25,6 +25,8 @@ #ifndef SHARE_JFR_SUPPORT_JFREPOCHSYNCHRONIZATION_HPP #define SHARE_JFR_SUPPORT_JFREPOCHSYNCHRONIZATION_HPP +#include "jfr/support/jfrThreadId.hpp" + /* * JavaThreads running _thread_in_native (Compiler threads) must synchronize * with the upcoming epoch in case there is an epoch shift in-progress. diff --git a/src/hotspot/share/jfr/support/jfrIntrinsics.cpp b/src/hotspot/share/jfr/support/jfrIntrinsics.cpp new file mode 100644 index 00000000000..776df81f136 --- /dev/null +++ b/src/hotspot/share/jfr/support/jfrIntrinsics.cpp @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2020, 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "jfr/jni/jfrJavaSupport.hpp" +#include "jfr/recorder/checkpoint/types/traceid/jfrTraceIdEpoch.hpp" +#include "jfr/recorder/checkpoint/types/traceid/jfrTraceIdLoadBarrier.inline.hpp" +#include "jfr/support/jfrIntrinsics.hpp" +#include "jfr/support/jfrThreadId.inline.hpp" +#include "jfr/support/jfrThreadLocal.hpp" +#include "jfr/writers/jfrJavaEventWriter.hpp" +#include "runtime/interfaceSupport.inline.hpp" + +#ifdef ASSERT +static void assert_precondition(JavaThread* jt) { + assert(jt != NULL, "invariant"); + DEBUG_ONLY(JfrJavaSupport::check_java_thread_in_java(jt);) + assert(jt->has_last_Java_frame(), "invariant"); +} + +static void assert_epoch_identity(JavaThread* jt, u2 current_epoch) { + assert_precondition(jt); + // Verify the epoch updates got written through also to the vthread object. + const u2 epoch_raw = ThreadIdAccess::epoch(jt->vthread()); + const bool excluded = epoch_raw & excluded_bit; + assert(!excluded, "invariant"); + assert(!JfrThreadLocal::is_excluded(jt), "invariant"); + const u2 vthread_epoch = epoch_raw & epoch_mask; + assert(vthread_epoch == current_epoch, "invariant"); +} +#endif + +void* JfrIntrinsicSupport::event_writer(JavaThread* jt) { + DEBUG_ONLY(assert_precondition(jt);) + // Can safepoint here. + ThreadInVMfromJava transition(jt); + return JfrJavaEventWriter::event_writer(jt); +} + +void* JfrIntrinsicSupport::write_checkpoint(JavaThread* jt) { + DEBUG_ONLY(assert_precondition(jt);) + assert(JfrThreadLocal::is_vthread(jt), "invariant"); + const u2 vthread_thread_local_epoch = JfrThreadLocal::vthread_epoch(jt); + const u2 current_epoch = ThreadIdAccess::current_epoch(); + if (vthread_thread_local_epoch == current_epoch) { + // After the epoch test in the intrinsic, the thread sampler interleaved + // and suspended the thread. As part of taking a sample, it updated + // the vthread object and the thread local "for us". We are good. + DEBUG_ONLY(assert_epoch_identity(jt, current_epoch);) + ThreadInVMfromJava transition(jt); + return JfrJavaEventWriter::event_writer(jt); + } + const traceid vthread_tid = JfrThreadLocal::vthread_id(jt); + // Transition before reading the epoch generation anew, now as _thread_in_vm. Can safepoint here. + ThreadInVMfromJava transition(jt); + JfrThreadLocal::set_vthread_epoch(jt, vthread_tid, ThreadIdAccess::current_epoch()); + return JfrJavaEventWriter::event_writer(jt); +} + +void JfrIntrinsicSupport::load_barrier(const Klass* klass) { + assert(klass != NULL, "sanity"); + JfrTraceIdLoadBarrier::load_barrier(klass); +} + +address JfrIntrinsicSupport::epoch_address() { + return JfrTraceIdEpoch::epoch_address(); +} + +address JfrIntrinsicSupport::epoch_generation_address() { + return JfrTraceIdEpoch::epoch_generation_address(); +} + +address JfrIntrinsicSupport::signal_address() { + return JfrTraceIdEpoch::signal_address(); +} diff --git a/src/hotspot/share/jfr/support/jfrIntrinsics.hpp b/src/hotspot/share/jfr/support/jfrIntrinsics.hpp index 6919c180290..a925d18da44 100644 --- a/src/hotspot/share/jfr/support/jfrIntrinsics.hpp +++ b/src/hotspot/share/jfr/support/jfrIntrinsics.hpp @@ -1,5 +1,5 @@ /* -* Copyright (c) 2012, 2019, Oracle and/or its affiliates. All rights reserved. +* Copyright (c) 2012, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -28,26 +28,37 @@ #include "utilities/macros.hpp" #if INCLUDE_JFR +#include "jfr/recorder/checkpoint/types/traceid/jfrTraceIdMacros.hpp" #include "jfr/support/jfrKlassExtension.hpp" #include "jfr/support/jfrThreadExtension.hpp" #include "jfr/utilities/jfrTime.hpp" -#include "jfr/recorder/checkpoint/types/traceid/jfrTraceIdMacros.hpp" +#include "memory/allocation.hpp" -#define JFR_TEMPLATES(template) \ +class JfrIntrinsicSupport : AllStatic { + public: + static void* event_writer(JavaThread* jt); + static void* write_checkpoint(JavaThread* jt); + static void load_barrier(const Klass* klass); + static address epoch_address(); + static address signal_address(); + static address epoch_generation_address(); +}; + +#define JFR_HAVE_INTRINSICS + +#define JFR_TEMPLATES(template) \ template(jdk_jfr_internal_JVM, "jdk/jfr/internal/JVM") \ template(jdk_jfr_internal_handlers_EventHandler_signature, "Ljdk/jfr/internal/handlers/EventHandler;") \ template(eventHandler_name, "eventHandler") \ + template(void_eventWriter_signature, "()Ljdk/jfr/internal/EventWriter;") \ -#define JFR_INTRINSICS(do_intrinsic, do_class, do_name, do_signature, do_alias) \ - do_intrinsic(_counterTime, jdk_jfr_internal_JVM, counterTime_name, void_long_signature, F_SN) \ - do_name( counterTime_name, "counterTime") \ - do_intrinsic(_getClassId, jdk_jfr_internal_JVM, getClassId_name, class_long_signature, F_SN) \ - do_name( getClassId_name, "getClassId") \ - do_intrinsic(_getEventWriter, jdk_jfr_internal_JVM, getEventWriter_name, void_object_signature, F_SN) \ - do_name( getEventWriter_name, "getEventWriter") \ - -#define JFR_HAVE_INTRINSICS -#define JFR_TIME_FUNCTION JfrTime::time_function() +#define JFR_INTRINSICS(do_intrinsic, do_class, do_name, do_signature, do_alias) \ + do_intrinsic(_counterTime, jdk_jfr_internal_JVM, counterTime_name, void_long_signature, F_SN) \ + do_name( counterTime_name, "counterTime") \ + do_intrinsic(_getClassId, jdk_jfr_internal_JVM, getClassId_name, class_long_signature, F_SN) \ + do_name( getClassId_name, "getClassId") \ + do_intrinsic(_getEventWriter, jdk_jfr_internal_JVM, getEventWriter_name, void_eventWriter_signature, F_SN) \ + do_name( getEventWriter_name, "getEventWriter") \ #else // !INCLUDE_JFR diff --git a/src/hotspot/share/jfr/support/jfrThreadExtension.hpp b/src/hotspot/share/jfr/support/jfrThreadExtension.hpp index 2ac303ac105..57c19a2b036 100644 --- a/src/hotspot/share/jfr/support/jfrThreadExtension.hpp +++ b/src/hotspot/share/jfr/support/jfrThreadExtension.hpp @@ -1,5 +1,5 @@ /* -* Copyright (c) 2012, 2019, Oracle and/or its affiliates. All rights reserved. +* Copyright (c) 2012, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -26,7 +26,7 @@ #define SHARE_JFR_SUPPORT_JFRTHREADEXTENSION_HPP #include "jfr/periodic/sampling/jfrThreadSampler.hpp" -#include "jfr/support/jfrThreadLocal.hpp" +#include "jfr/support/jfrThreadId.hpp" #define DEFINE_THREAD_LOCAL_FIELD_JFR mutable JfrThreadLocal _jfr_thread_local @@ -41,7 +41,13 @@ #define DEFINE_THREAD_LOCAL_ACCESSOR_JFR \ JfrThreadLocal* jfr_thread_local() const { return &_jfr_thread_local; } -#define THREAD_ID_OFFSET_JFR JfrThreadLocal::trace_id_offset() +#define VTHREAD_ID_OFFSET_JFR JfrThreadLocal::vthread_id_offset() + +#define VTHREAD_OFFSET_JFR JfrThreadLocal::vthread_offset() + +#define VTHREAD_EPOCH_OFFSET_JFR JfrThreadLocal::vthread_epoch_offset() + +#define VTHREAD_EXCLUDED_OFFSET_JFR JfrThreadLocal::vthread_excluded_offset() #define THREAD_LOCAL_WRITER_OFFSET_JFR \ JfrThreadLocal::java_event_writer_offset() + THREAD_LOCAL_OFFSET_JFR diff --git a/src/hotspot/share/jfr/support/jfrThreadId.hpp b/src/hotspot/share/jfr/support/jfrThreadId.hpp index 5104fe1c739..23e0e9b581f 100644 --- a/src/hotspot/share/jfr/support/jfrThreadId.hpp +++ b/src/hotspot/share/jfr/support/jfrThreadId.hpp @@ -1,5 +1,5 @@ /* -* Copyright (c) 2012, 2021, Oracle and/or its affiliates. All rights reserved. +* Copyright (c) 2012, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -25,17 +25,19 @@ #ifndef SHARE_JFR_SUPPORT_JFRTHREADID_HPP #define SHARE_JFR_SUPPORT_JFRTHREADID_HPP -#include "utilities/macros.hpp" #include "utilities/globalDefinitions.hpp" +#include "utilities/macros.hpp" #if INCLUDE_JFR #include "jfr/support/jfrThreadLocal.hpp" #include "jfr/utilities/jfrTypes.hpp" -#define JFR_THREAD_ID(thread) ((thread)->jfr_thread_local()->thread_id()) +#define JFR_THREAD_ID(thread) (JfrThreadLocal::external_thread_id(thread)) +#define JFR_JVM_THREAD_ID(thread) (JfrThreadLocal::jvm_thread_id(thread)) #else #include "runtime/osThread.hpp" typedef u8 traceid; #define JFR_THREAD_ID(thread) ((traceid)(thread)->osthread()->thread_id()) +#define JFR_JVM_THREAD_ID(thread) ((traceid)(thread)->osthread()->thread_id()) #endif #endif // SHARE_JFR_SUPPORT_JFRTHREADID_HPP diff --git a/src/hotspot/share/jfr/support/jfrThreadId.inline.hpp b/src/hotspot/share/jfr/support/jfrThreadId.inline.hpp new file mode 100644 index 00000000000..94278eda0a5 --- /dev/null +++ b/src/hotspot/share/jfr/support/jfrThreadId.inline.hpp @@ -0,0 +1,64 @@ +/* +* Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved. +* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +* +* This code is free software; you can redistribute it and/or modify it +* under the terms of the GNU General Public License version 2 only, as +* published by the Free Software Foundation. +* +* This code is distributed in the hope that it will be useful, but WITHOUT +* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +* version 2 for more details (a copy is included in the LICENSE file that +* accompanied this code). +* +* You should have received a copy of the GNU General Public License version +* 2 along with this work; if not, write to the Free Software Foundation, +* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. +* +* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA +* or visit www.oracle.com if you need additional information or have any +* questions. +* +*/ + +#ifndef SHARE_JFR_SUPPORT_JFRTHREADID_INLINE_HPP +#define SHARE_JFR_SUPPORT_JFRTHREADID_INLINE_HPP + +#include "jfr/support/jfrThreadId.hpp" + +#include "classfile/javaClasses.inline.hpp" +#include "jfr/recorder/checkpoint/types/traceid/jfrTraceIdEpoch.hpp" +#include "jfr/utilities/jfrTypes.hpp" +#include "memory/allocation.inline.hpp" + +static constexpr const u2 excluded_bit = 32768; +static constexpr const u2 epoch_mask = excluded_bit - 1; + +class ThreadIdAccess : AllStatic { + public: + static traceid id(oop ref) { + return static_cast(java_lang_Thread::thread_id(ref)); + } + static bool is_excluded(oop ref) { + return epoch(ref) & excluded_bit; + } + static void include(oop ref) { + assert(is_excluded(ref), "invariant"); + set_epoch(ref, epoch(ref) ^ excluded_bit); + } + static void exclude(oop ref) { + set_epoch(ref, excluded_bit | epoch(ref)); + } + static u2 epoch(oop ref) { + return java_lang_Thread::jfr_epoch(ref); + } + static void set_epoch(oop ref, u2 epoch) { + java_lang_Thread::set_jfr_epoch(ref, epoch); + } + static u2 current_epoch() { + return JfrTraceIdEpoch::epoch_generation(); + } +}; + +#endif // SHARE_JFR_SUPPORT_JFRTHREADID_INLINE_HPP diff --git a/src/hotspot/share/jfr/support/jfrThreadLocal.cpp b/src/hotspot/share/jfr/support/jfrThreadLocal.cpp index 772438198a0..1115a274fb4 100644 --- a/src/hotspot/share/jfr/support/jfrThreadLocal.cpp +++ b/src/hotspot/share/jfr/support/jfrThreadLocal.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2012, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -28,13 +28,20 @@ #include "jfr/leakprofiler/checkpoint/objectSampleCheckpoint.hpp" #include "jfr/periodic/jfrThreadCPULoadEvent.hpp" #include "jfr/recorder/checkpoint/jfrCheckpointManager.hpp" -#include "jfr/recorder/checkpoint/types/traceid/jfrTraceIdEpoch.hpp" +#include "jfr/recorder/checkpoint/types/traceid/jfrOopTraceId.inline.hpp" #include "jfr/recorder/jfrRecorder.hpp" #include "jfr/recorder/service/jfrOptionSet.hpp" +#include "jfr/recorder/stacktrace/jfrStackTraceRepository.hpp" #include "jfr/recorder/storage/jfrStorage.hpp" +#include "jfr/support/jfrThreadId.inline.hpp" #include "jfr/support/jfrThreadLocal.hpp" +#include "jfr/utilities/jfrSpinlockHelper.hpp" +#include "jfr/writers/jfrJavaEventWriter.hpp" +#include "logging/log.hpp" #include "memory/allocation.inline.hpp" +#include "runtime/atomic.hpp" #include "runtime/os.hpp" +#include "runtime/threadIdentifier.hpp" #include "runtime/thread.inline.hpp" #include "utilities/sizes.hpp" @@ -45,21 +52,30 @@ JfrThreadLocal::JfrThreadLocal() : _shelved_buffer(NULL), _load_barrier_buffer_epoch_0(NULL), _load_barrier_buffer_epoch_1(NULL), + _checkpoint_buffer_epoch_0(NULL), + _checkpoint_buffer_epoch_1(NULL), _stackframes(NULL), - _trace_id(JfrTraceId::assign_thread_id()), _thread(), + _vthread_id(0), + _jvm_thread_id(0), + _thread_id_alias(max_julong), _data_lost(0), _stack_trace_id(max_julong), + _parent_trace_id(0), _user_time(0), _cpu_time(0), _wallclock_time(os::javaTimeNanos()), _stack_trace_hash(0), _stackdepth(0), _entering_suspend_flag(0), - _excluded(false), + _critical_section(0), + _vthread_epoch(0), + _vthread_excluded(false), + _jvm_thread_excluded(false), + _vthread(false), _dead(false) { Thread* thread = Thread::current_or_null(); - _parent_trace_id = thread != NULL ? thread->jfr_thread_local()->trace_id() : (traceid)0; + _parent_trace_id = thread != NULL ? jvm_thread_id(thread) : (traceid)0; } u8 JfrThreadLocal::add_data_lost(u8 value) { @@ -81,22 +97,26 @@ const JfrBlobHandle& JfrThreadLocal::thread_blob() const { } static void send_java_thread_start_event(JavaThread* jt) { + assert(jt != NULL, "invariant"); + assert(Thread::current() == jt, "invariant"); + if (!JfrJavaSupport::on_thread_start(jt)) { + // thread is excluded + return; + } EventThreadStart event; - event.set_thread(jt->jfr_thread_local()->thread_id()); + traceid thread_id = JfrThreadLocal::jvm_thread_id(jt); + assert(thread_id != 0, "invariant"); + event.set_thread(thread_id); event.set_parentThread(jt->jfr_thread_local()->parent_thread_id()); event.commit(); } void JfrThreadLocal::on_start(Thread* t) { - assert(t != NULL, "invariant"); - assert(Thread::current() == t, "invariant"); - JfrJavaSupport::on_thread_start(t); + assign_thread_id(t, t->jfr_thread_local()); if (JfrRecorder::is_recording()) { - JfrCheckpointManager::write_thread_checkpoint(t); - if (!t->jfr_thread_local()->is_excluded()) { - if (t->is_Java_thread()) { - send_java_thread_start_event(JavaThread::cast(t)); - } + JfrCheckpointManager::write_checkpoint(t); + if (t->is_Java_thread()) { + send_java_thread_start_event(JavaThread::cast(t)); } } if (t->jfr_thread_local()->has_cached_stack_trace()) { @@ -104,15 +124,19 @@ void JfrThreadLocal::on_start(Thread* t) { } } -static void send_java_thread_end_events(traceid id, JavaThread* jt) { - assert(jt != NULL, "invariant"); - assert(Thread::current() == jt, "invariant"); - assert(jt->jfr_thread_local()->trace_id() == id, "invariant"); - if (JfrRecorder::is_recording()) { - EventThreadEnd event; - event.set_thread(id); - event.commit(); - JfrThreadCPULoadEvent::send_event_for_thread(jt); +// The starter thread ensures that the startee has a valid _vm_thread_id and _contextual_id. +// This is to avoid recursion in thread assignment since accessing the java threadObj can lead +// to events being fired, a situation the starter thread can handle but not the startee. +void JfrThreadLocal::on_java_thread_start(JavaThread* starter, JavaThread* startee) { + assert(starter != nullptr, "invariant"); + assert(startee != nullptr, "invariant"); + JfrThreadLocal* const tl = startee->jfr_thread_local(); + assign_thread_id(startee, tl); + assert(vthread_id(startee) != 0, "invariant"); + assert(jvm_thread_id(startee) == vthread_id(startee), "invariant"); + if (JfrRecorder::is_recording() && EventThreadStart::is_enabled() && EventThreadStart::is_stacktrace_enabled()) { + // skip level 2 to skip frames Thread.start() and Thread.start0() + startee->jfr_thread_local()->set_cached_stack_trace_id(JfrStackTraceRepository::record(starter, 2)); } } @@ -142,6 +166,14 @@ void JfrThreadLocal::release(Thread* t) { _load_barrier_buffer_epoch_1->set_retired(); _load_barrier_buffer_epoch_1 = NULL; } + if (_checkpoint_buffer_epoch_0 != NULL) { + _checkpoint_buffer_epoch_0->set_retired(); + _checkpoint_buffer_epoch_0 = NULL; + } + if (_checkpoint_buffer_epoch_1 != NULL) { + _checkpoint_buffer_epoch_1->set_retired(); + _checkpoint_buffer_epoch_1 = NULL; + } } void JfrThreadLocal::release(JfrThreadLocal* tl, Thread* t) { @@ -154,38 +186,44 @@ void JfrThreadLocal::release(JfrThreadLocal* tl, Thread* t) { tl->release(t); } +static void send_java_thread_end_event(JavaThread* jt, traceid tid) { + assert(jt != NULL, "invariant"); + assert(Thread::current() == jt, "invariant"); + assert(tid != 0, "invariant"); + if (JfrRecorder::is_recording()) { + EventThreadEnd event; + event.set_thread(tid); + event.commit(); + ObjectSampleCheckpoint::on_thread_exit(tid); + } +} + void JfrThreadLocal::on_exit(Thread* t) { assert(t != NULL, "invariant"); JfrThreadLocal * const tl = t->jfr_thread_local(); assert(!tl->is_dead(), "invariant"); - if (JfrRecorder::is_recording()) { - if (t->is_Java_thread()) { - JavaThread* const jt = JavaThread::cast(t); - ObjectSampleCheckpoint::on_thread_exit(jt); - send_java_thread_end_events(tl->thread_id(), jt); - } + if (t->is_Java_thread()) { + JavaThread* const jt = JavaThread::cast(t); + send_java_thread_end_event(jt, JfrThreadLocal::jvm_thread_id(jt)); + JfrThreadCPULoadEvent::send_event_for_thread(jt); } release(tl, Thread::current()); // because it could be that Thread::current() != t } -static JfrBuffer* acquire_buffer(bool excluded) { - JfrBuffer* const buffer = JfrStorage::acquire_thread_local(Thread::current()); - if (buffer != NULL && excluded) { - buffer->set_excluded(); - } - return buffer; +static JfrBuffer* acquire_buffer() { + return JfrStorage::acquire_thread_local(Thread::current()); } JfrBuffer* JfrThreadLocal::install_native_buffer() const { assert(!has_native_buffer(), "invariant"); - _native_buffer = acquire_buffer(_excluded); + _native_buffer = acquire_buffer(); return _native_buffer; } JfrBuffer* JfrThreadLocal::install_java_buffer() const { assert(!has_java_buffer(), "invariant"); assert(!has_java_event_writer(), "invariant"); - _java_buffer = acquire_buffer(_excluded); + _java_buffer = acquire_buffer(); return _java_buffer; } @@ -195,26 +233,232 @@ JfrStackFrame* JfrThreadLocal::install_stackframes() const { return _stackframes; } -ByteSize JfrThreadLocal::trace_id_offset() { - return in_ByteSize(offset_of(JfrThreadLocal, _trace_id)); -} - ByteSize JfrThreadLocal::java_event_writer_offset() { return in_ByteSize(offset_of(JfrThreadLocal, _java_event_writer)); } -void JfrThreadLocal::exclude(Thread* t) { - assert(t != NULL, "invariant"); - t->jfr_thread_local()->_excluded = true; - t->jfr_thread_local()->release(t); +ByteSize JfrThreadLocal::vthread_id_offset() { + return in_ByteSize(offset_of(JfrThreadLocal, _vthread_id)); } -void JfrThreadLocal::include(Thread* t) { - assert(t != NULL, "invariant"); - t->jfr_thread_local()->_excluded = false; - t->jfr_thread_local()->release(t); +ByteSize JfrThreadLocal::vthread_offset() { + return in_ByteSize(offset_of(JfrThreadLocal, _vthread)); +} + +ByteSize JfrThreadLocal::vthread_epoch_offset() { + return in_ByteSize(offset_of(JfrThreadLocal, _vthread_epoch)); +} + +ByteSize JfrThreadLocal::vthread_excluded_offset() { + return in_ByteSize(offset_of(JfrThreadLocal, _vthread_excluded)); +} + +void JfrThreadLocal::set(bool* exclusion_field, bool state) { + assert(exclusion_field != nullptr, "invariant"); + *exclusion_field = state; +} + +bool JfrThreadLocal::is_vthread_excluded() const { + return Atomic::load(&_vthread_excluded); +} + +bool JfrThreadLocal::is_jvm_thread_excluded(const Thread* t) { + assert(t != nullptr, "invariant"); + return t->jfr_thread_local()->_jvm_thread_excluded; +} + +void JfrThreadLocal::exclude_vthread(const JavaThread* jt) { + set(&jt->jfr_thread_local()->_vthread_excluded, true); + JfrJavaEventWriter::exclude(vthread_id(jt), jt); +} + +void JfrThreadLocal::include_vthread(const JavaThread* jt) { + set(&jt->jfr_thread_local()->_vthread_excluded, false); + JfrJavaEventWriter::include(vthread_id(jt), jt); +} + +void JfrThreadLocal::exclude_jvm_thread(const Thread* t) { + set(&t->jfr_thread_local()->_jvm_thread_excluded, true); + if (t->is_Java_thread()) { + JfrJavaEventWriter::exclude(t->jfr_thread_local()->_jvm_thread_id, JavaThread::cast(t)); + } +} + +void JfrThreadLocal::include_jvm_thread(const Thread* t) { + set(&t->jfr_thread_local()->_jvm_thread_excluded, false); + if (t->is_Java_thread()) { + JfrJavaEventWriter::include(t->jfr_thread_local()->_jvm_thread_id, JavaThread::cast(t)); + } +} + +bool JfrThreadLocal::is_excluded() const { + return Atomic::load_acquire(&_vthread) ? is_vthread_excluded(): _jvm_thread_excluded; +} + +bool JfrThreadLocal::is_included() const { + return !is_excluded(); +} + +bool JfrThreadLocal::is_excluded(const Thread* t) { + assert(t != nullptr, "invariant"); + return t->jfr_thread_local()->is_excluded(); +} + +bool JfrThreadLocal::is_included(const Thread* t) { + assert(t != nullptr, "invariant"); + return t->jfr_thread_local()->is_included(); } u4 JfrThreadLocal::stackdepth() const { return _stackdepth != 0 ? _stackdepth : (u4)JfrOptionSet::stackdepth(); } + +bool JfrThreadLocal::is_impersonating(const Thread* t) { + return t->jfr_thread_local()->_thread_id_alias != max_julong; +} + +void JfrThreadLocal::impersonate(const Thread* t, traceid other_thread_id) { + assert(t != NULL, "invariant"); + assert(other_thread_id != 0, "invariant"); + JfrThreadLocal* const tl = t->jfr_thread_local(); + tl->_thread_id_alias = other_thread_id; +} + +void JfrThreadLocal::stop_impersonating(const Thread* t) { + assert(t != NULL, "invariant"); + JfrThreadLocal* const tl = t->jfr_thread_local(); + if (is_impersonating(t)) { + tl->_thread_id_alias = max_julong; + } + assert(!is_impersonating(t), "invariant"); +} + +typedef JfrOopTraceId AccessThreadTraceId; + +void JfrThreadLocal::set_vthread_epoch(const JavaThread* jt, traceid tid, u2 epoch) { + assert(jt != nullptr, "invariant"); + assert(is_vthread(jt), "invariant"); + // To support event recursion, we update the native side first, + // this provides the terminating case. + Atomic::store(&jt->jfr_thread_local()->_vthread_epoch, epoch); + /* + * The java side, i.e. the vthread object, can now be updated. + * Accessing the vthread object itself is a recursive case, + * because it can trigger additional events, e.g. + * loading the oop through load barriers. + * Note there is a potential problem with this solution: + * The recursive write hitting the terminating case will + * use the thread id _before_ the checkpoint is committed. + * Hence, the periodic thread can possibly flush that event + * to a segment that does not include an associated checkpoint. + * Considered rare and quite benign for now. The worst case is + * that thread information for that event is not resolvable, i.e. null. + */ + oop vthread = jt->vthread(); + assert(vthread != nullptr, "invariant"); + AccessThreadTraceId::set_epoch(vthread, epoch); + JfrCheckpointManager::write_checkpoint(const_cast(jt), tid, vthread); +} + +traceid JfrThreadLocal::vthread_id(const Thread* t) { + assert(t != nullptr, "invariant"); + return Atomic::load(&t->jfr_thread_local()->_vthread_id); +} + +u2 JfrThreadLocal::vthread_epoch(const JavaThread* jt) { + assert(jt != nullptr, "invariant"); + return Atomic::load(&jt->jfr_thread_local()->_vthread_epoch); +} + +traceid JfrThreadLocal::thread_id(const Thread* t) { + assert(t != NULL, "invariant"); + if (is_impersonating(t)) { + return t->jfr_thread_local()->_thread_id_alias; + } + JfrThreadLocal* const tl = t->jfr_thread_local(); + if (!t->is_Java_thread() || !Atomic::load_acquire(&tl->_vthread)) { + return jvm_thread_id(t, tl); + } + // virtual thread + const JavaThread* jt = JavaThread::cast(t); + const traceid tid = vthread_id(jt); + assert(tid != 0, "invariant"); + if (!tl->is_vthread_excluded()) { + const u2 current_epoch = AccessThreadTraceId::current_epoch(); + if (vthread_epoch(jt) != current_epoch) { + set_vthread_epoch(jt, tid, current_epoch); + } + } + return tid; +} + +// When not recording, there is no checkpoint system +// in place for writing vthread information. +traceid JfrThreadLocal::external_thread_id(const Thread* t) { + assert(t != NULL, "invariant"); + return JfrRecorder::is_recording() ? thread_id(t) : jvm_thread_id(t); +} + +inline traceid load_java_thread_id(const Thread* t) { + assert(t != nullptr, "invariant"); + assert(t->is_Java_thread(), "invariant"); + oop threadObj = JavaThread::cast(t)->threadObj(); + return threadObj != nullptr ? AccessThreadTraceId::id(threadObj) : 0; +} + +traceid JfrThreadLocal::assign_thread_id(const Thread* t, JfrThreadLocal* tl) { + assert(t != nullptr, "invariant"); + assert(tl != nullptr, "invariant"); + JfrSpinlockHelper spinlock(&tl->_critical_section); + traceid tid = tl->_jvm_thread_id; + if (tid == 0) { + if (t->is_Java_thread()) { + tid = load_java_thread_id(t); + tl->_jvm_thread_id = tid; + Atomic::store(&tl->_vthread_id, tid); + return tid; + } + tid = static_cast(ThreadIdentifier::next()); + tl->_jvm_thread_id = tid; + } + return tid; +} + +traceid JfrThreadLocal::jvm_thread_id(const Thread* t, JfrThreadLocal* tl) { + assert(t != nullptr, "invariant"); + assert(tl != nullptr, "invariant"); + return tl->_jvm_thread_id != 0 ? tl->_jvm_thread_id : JfrThreadLocal::assign_thread_id(t, tl); +} + +traceid JfrThreadLocal::jvm_thread_id(const Thread* t) { + assert(t != nullptr, "invariant"); + return jvm_thread_id(t, t->jfr_thread_local()); +} + +bool JfrThreadLocal::is_vthread(const JavaThread* jt) { + assert(jt != nullptr, "invariant"); + return Atomic::load_acquire(&jt->jfr_thread_local()->_vthread); +} + +inline bool is_virtual(const JavaThread* jt, oop thread) { + assert(jt != nullptr, "invariant"); + return thread != jt->threadObj(); +} + +void JfrThreadLocal::on_set_current_thread(JavaThread* jt, oop thread) { + assert(jt != nullptr, "invariant"); + assert(thread != nullptr, "invariant"); + JfrThreadLocal* const tl = jt->jfr_thread_local(); + if (!is_virtual(jt, thread)) { + Atomic::release_store(&tl->_vthread, false); + return; + } + Atomic::store(&tl->_vthread_id, AccessThreadTraceId::id(thread)); + const u2 epoch_raw = AccessThreadTraceId::epoch(thread); + const bool excluded = epoch_raw & excluded_bit; + Atomic::store(&tl->_vthread_excluded, excluded); + if (!excluded) { + Atomic::store(&tl->_vthread_epoch, static_cast(epoch_raw & epoch_mask)); + } + Atomic::release_store(&tl->_vthread, true); +} diff --git a/src/hotspot/share/jfr/support/jfrThreadLocal.hpp b/src/hotspot/share/jfr/support/jfrThreadLocal.hpp index 7e4a0819cf2..b8460fbe4c9 100644 --- a/src/hotspot/share/jfr/support/jfrThreadLocal.hpp +++ b/src/hotspot/share/jfr/support/jfrThreadLocal.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2012, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -34,6 +34,10 @@ class JfrStackFrame; class Thread; class JfrThreadLocal { + friend class Jfr; + friend class JfrIntrinsicSupport; + friend class JfrJavaSupport; + friend class JfrRecorder; private: jobject _java_event_writer; mutable JfrBuffer* _java_buffer; @@ -41,20 +45,28 @@ class JfrThreadLocal { JfrBuffer* _shelved_buffer; JfrBuffer* _load_barrier_buffer_epoch_0; JfrBuffer* _load_barrier_buffer_epoch_1; + JfrBuffer* _checkpoint_buffer_epoch_0; + JfrBuffer* _checkpoint_buffer_epoch_1; mutable JfrStackFrame* _stackframes; - mutable traceid _trace_id; JfrBlobHandle _thread; + mutable traceid _vthread_id; + mutable traceid _jvm_thread_id; + mutable traceid _thread_id_alias; u8 _data_lost; traceid _stack_trace_id; + traceid _parent_trace_id; jlong _user_time; jlong _cpu_time; jlong _wallclock_time; unsigned int _stack_trace_hash; mutable u4 _stackdepth; volatile jint _entering_suspend_flag; - bool _excluded; + mutable volatile int _critical_section; + u2 _vthread_epoch; + bool _vthread_excluded; + bool _jvm_thread_excluded; + bool _vthread; bool _dead; - traceid _parent_trace_id; JfrBuffer* install_native_buffer() const; JfrBuffer* install_java_buffer() const; @@ -62,6 +74,17 @@ class JfrThreadLocal { void release(Thread* t); static void release(JfrThreadLocal* tl, Thread* t); + static void set(bool* excluded_field, bool state); + static traceid assign_thread_id(const Thread* t, JfrThreadLocal* tl); + static traceid vthread_id(const Thread* t); + static void set_vthread_epoch(const JavaThread* jt, traceid id, u2 epoch); + bool is_vthread_excluded() const; + static void exclude_vthread(const JavaThread* jt); + static void include_vthread(const JavaThread* jt); + static bool is_jvm_thread_excluded(const Thread* t); + static void exclude_jvm_thread(const Thread* t); + static void include_jvm_thread(const Thread* t); + public: JfrThreadLocal(); @@ -123,13 +146,25 @@ class JfrThreadLocal { _stackdepth = depth; } - traceid thread_id() const { - return _trace_id; - } + // Contextually defined thread id that is volatile, + // a function of Java carrier thread mounts / unmounts. + static traceid thread_id(const Thread* t); + static bool is_vthread(const JavaThread* jt); + static u2 vthread_epoch(const JavaThread* jt); - void set_thread_id(traceid thread_id) { - _trace_id = thread_id; - } + // Exposed to external code that use a thread id unconditionally. + // Jfr might not even be running. + static traceid external_thread_id(const Thread* t); + + // Non-volatile thread id, for Java carrier threads and non-java threads. + static traceid jvm_thread_id(const Thread* t); + static traceid jvm_thread_id(const Thread* t, JfrThreadLocal* tl); + + // To impersonate is to temporarily masquerade as another thread. + // For example, when writing an event that should be attributed to some other thread. + static void impersonate(const Thread* t, traceid other_thread_id); + static void stop_impersonating(const Thread* t); + static bool is_impersonating(const Thread* t); traceid parent_thread_id() const { return _parent_trace_id; @@ -199,40 +234,34 @@ class JfrThreadLocal { _wallclock_time = wallclock_time; } - traceid trace_id() const { - return _trace_id; - } - - traceid* const trace_id_addr() const { - return &_trace_id; - } - - void set_trace_id(traceid id) const { - _trace_id = id; - } - - bool is_excluded() const { - return _excluded; - } - bool is_dead() const { return _dead; } + bool is_excluded() const; + bool is_included() const; + static bool is_excluded(const Thread* thread); + static bool is_included(const Thread* thread); + bool has_thread_blob() const; void set_thread_blob(const JfrBlobHandle& handle); const JfrBlobHandle& thread_blob() const; - static void exclude(Thread* t); - static void include(Thread* t); - + // Hooks static void on_start(Thread* t); static void on_exit(Thread* t); + static void on_set_current_thread(JavaThread* jt, oop thread); + static void on_java_thread_start(JavaThread* starter, JavaThread* startee); // Code generation - static ByteSize trace_id_offset(); static ByteSize java_event_writer_offset(); + static ByteSize vthread_id_offset(); + static ByteSize vthread_offset(); + static ByteSize vthread_epoch_offset(); + static ByteSize vthread_excluded_offset(); + friend class JfrJavaThread; + friend class JfrCheckpointManager; template friend class JfrEpochQueueKlassPolicy; }; diff --git a/src/hotspot/share/jfr/writers/jfrJavaEventWriter.cpp b/src/hotspot/share/jfr/writers/jfrJavaEventWriter.cpp index e315a656eca..ad809620fed 100644 --- a/src/hotspot/share/jfr/writers/jfrJavaEventWriter.cpp +++ b/src/hotspot/share/jfr/writers/jfrJavaEventWriter.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2016, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -29,7 +29,7 @@ #include "classfile/vmSymbols.hpp" #include "jfr/jni/jfrJavaSupport.hpp" #include "jfr/recorder/storage/jfrStorage.hpp" -#include "jfr/support/jfrThreadId.hpp" +#include "jfr/support/jfrThreadLocal.hpp" #include "jfr/utilities/jfrTypes.hpp" #include "jfr/writers/jfrJavaEventWriter.hpp" #include "memory/iterator.hpp" @@ -45,34 +45,10 @@ static int start_pos_address_offset = invalid_offset; static int current_pos_offset = invalid_offset; static int max_pos_offset = invalid_offset; static int notified_offset = invalid_offset; +static int excluded_offset = invalid_offset; +static int thread_id_offset = invalid_offset; static int valid_offset = invalid_offset; -static bool find_field(InstanceKlass* ik, - Symbol* name_symbol, - Symbol* signature_symbol, - fieldDescriptor* fd, - bool is_static = false, - bool allow_super = false) { - if (allow_super || is_static) { - return ik->find_field(name_symbol, signature_symbol, is_static, fd) != NULL; - } else { - return ik->find_local_field(name_symbol, signature_symbol, fd); - } -} - -static void compute_offset(int &dest_offset, - Klass* klass, - Symbol* name_symbol, - Symbol* signature_symbol, - bool is_static = false, bool allow_super = false) { - fieldDescriptor fd; - InstanceKlass* ik = InstanceKlass::cast(klass); - if (!find_field(ik, name_symbol, signature_symbol, &fd, is_static, allow_super)) { - assert(false, "invariant"); - } - dest_offset = fd.offset(); -} - static bool setup_event_writer_offsets(TRAPS) { const char class_name[] = "jdk/jfr/internal/EventWriter"; Symbol* const k_sym = SymbolTable::new_symbol(class_name); @@ -84,42 +60,56 @@ static bool setup_event_writer_offsets(TRAPS) { Symbol* const start_pos_sym = SymbolTable::new_symbol(start_pos_name); assert(start_pos_sym != NULL, "invariant"); assert(invalid_offset == start_pos_offset, "invariant"); - compute_offset(start_pos_offset, klass, start_pos_sym, vmSymbols::long_signature()); + JfrJavaSupport::compute_field_offset(start_pos_offset, klass, start_pos_sym, vmSymbols::long_signature()); assert(start_pos_offset != invalid_offset, "invariant"); const char start_pos_address_name[] = "startPositionAddress"; Symbol* const start_pos_address_sym = SymbolTable::new_symbol(start_pos_address_name); assert(start_pos_address_sym != NULL, "invariant"); assert(invalid_offset == start_pos_address_offset, "invariant"); - compute_offset(start_pos_address_offset, klass, start_pos_address_sym, vmSymbols::long_signature()); + JfrJavaSupport::compute_field_offset(start_pos_address_offset, klass, start_pos_address_sym, vmSymbols::long_signature()); assert(start_pos_address_offset != invalid_offset, "invariant"); const char event_pos_name[] = "currentPosition"; Symbol* const event_pos_sym = SymbolTable::new_symbol(event_pos_name); assert(event_pos_sym != NULL, "invariant"); assert(invalid_offset == current_pos_offset, "invariant"); - compute_offset(current_pos_offset, klass, event_pos_sym,vmSymbols::long_signature()); + JfrJavaSupport::compute_field_offset(current_pos_offset, klass, event_pos_sym,vmSymbols::long_signature()); assert(current_pos_offset != invalid_offset, "invariant"); const char max_pos_name[] = "maxPosition"; Symbol* const max_pos_sym = SymbolTable::new_symbol(max_pos_name); assert(max_pos_sym != NULL, "invariant"); assert(invalid_offset == max_pos_offset, "invariant"); - compute_offset(max_pos_offset, klass, max_pos_sym, vmSymbols::long_signature()); + JfrJavaSupport::compute_field_offset(max_pos_offset, klass, max_pos_sym, vmSymbols::long_signature()); assert(max_pos_offset != invalid_offset, "invariant"); const char notified_name[] = "notified"; Symbol* const notified_sym = SymbolTable::new_symbol(notified_name); assert (notified_sym != NULL, "invariant"); assert(invalid_offset == notified_offset, "invariant"); - compute_offset(notified_offset, klass, notified_sym, vmSymbols::bool_signature()); + JfrJavaSupport::compute_field_offset(notified_offset, klass, notified_sym, vmSymbols::bool_signature()); assert(notified_offset != invalid_offset, "invariant"); + const char excluded_name[] = "excluded"; + Symbol* const excluded_sym = SymbolTable::new_symbol(excluded_name); + assert(excluded_sym != NULL, "invariant"); + assert(invalid_offset == excluded_offset, "invariant"); + JfrJavaSupport::compute_field_offset(excluded_offset, klass, excluded_sym, vmSymbols::bool_signature()); + assert(excluded_offset != invalid_offset, "invariant"); + + const char threadID_name[] = "threadID"; + Symbol * const threadID_sym = SymbolTable::new_symbol(threadID_name); + assert(threadID_sym != NULL, "invariant"); + assert(invalid_offset == thread_id_offset, "invariant"); + JfrJavaSupport::compute_field_offset(thread_id_offset, klass, threadID_sym, vmSymbols::long_signature()); + assert(thread_id_offset != invalid_offset, "invariant"); + const char valid_name[] = "valid"; Symbol* const valid_sym = SymbolTable::new_symbol(valid_name); assert (valid_sym != NULL, "invariant"); assert(invalid_offset == valid_offset, "invariant"); - compute_offset(valid_offset, klass, valid_sym, vmSymbols::bool_signature()); + JfrJavaSupport::compute_field_offset(valid_offset, klass, valid_sym, vmSymbols::bool_signature()); assert(valid_offset != invalid_offset, "invariant"); return true; } @@ -181,6 +171,28 @@ void JfrJavaEventWriter::notify() { Threads::threads_do(&closure); } +static void set_excluded_field(traceid tid, const JavaThread* jt, bool state) { + assert(jt != nullptr, "invariant"); + jobject event_writer_handle = jt->jfr_thread_local()->java_event_writer(); + if (event_writer_handle == nullptr) { + return; + } + oop event_writer = JNIHandles::resolve_non_null(event_writer_handle); + assert(event_writer != nullptr, "invariant"); + const jlong event_writer_tid = event_writer->long_field(thread_id_offset); + if (event_writer_tid == static_cast(tid)) { + event_writer->bool_field_put(excluded_offset, state); + } +} + +void JfrJavaEventWriter::exclude(traceid tid, const JavaThread* jt) { + set_excluded_field(tid, jt, true); +} + +void JfrJavaEventWriter::include(traceid tid, const JavaThread* jt) { + set_excluded_field(tid, jt, false); +} + void JfrJavaEventWriter::notify(JavaThread* jt) { assert(jt != NULL, "invariant"); assert(SafepointSynchronize::is_at_safepoint(), "invariant"); @@ -191,30 +203,44 @@ void JfrJavaEventWriter::notify(JavaThread* jt) { } } -static jobject create_new_event_writer(JfrBuffer* buffer, TRAPS) { +static jobject create_new_event_writer(JfrBuffer* buffer, JfrThreadLocal* tl, TRAPS) { assert(buffer != NULL, "invariant"); DEBUG_ONLY(JfrJavaSupport::check_java_thread_in_vm(THREAD)); HandleMark hm(THREAD); static const char klass[] = "jdk/jfr/internal/EventWriter"; static const char method[] = ""; - static const char signature[] = "(JJJJZ)V"; + static const char signature[] = "(JJJJZZ)V"; JavaValue result(T_OBJECT); JfrJavaArguments args(&result, klass, method, signature, CHECK_NULL); + // parameters args.push_long((jlong)buffer->pos()); args.push_long((jlong)buffer->end()); args.push_long((jlong)buffer->pos_address()); - args.push_long((jlong)JFR_THREAD_ID(THREAD)); - args.push_int((int)JNI_TRUE); + args.push_long((jlong)JfrThreadLocal::thread_id(THREAD)); + args.push_int((jint)JNI_TRUE); // valid + args.push_int(tl->is_excluded() ? (jint)JNI_TRUE : (jint)JNI_FALSE); // excluded JfrJavaSupport::new_object_global_ref(&args, CHECK_NULL); return result.get_jobject(); } -jobject JfrJavaEventWriter::event_writer(JavaThread* t) { - DEBUG_ONLY(JfrJavaSupport::check_java_thread_in_vm(t)); - JfrThreadLocal* const tl = t->jfr_thread_local(); +jobject JfrJavaEventWriter::event_writer(JavaThread* jt) { + DEBUG_ONLY(JfrJavaSupport::check_java_thread_in_vm(jt)); + JfrThreadLocal* const tl = jt->jfr_thread_local(); assert(tl->shelved_buffer() == NULL, "invariant"); - return tl->java_event_writer(); + jobject h_writer = tl->java_event_writer(); + if (h_writer != NULL) { + oop writer = JNIHandles::resolve_non_null(h_writer); + assert(writer != NULL, "invariant"); + const jlong event_writer_tid = writer->long_field(thread_id_offset); + const jlong current_tid = static_cast(JfrThreadLocal::thread_id(jt)); + if (event_writer_tid != current_tid) { + const bool excluded = tl->is_excluded(); + writer->bool_field_put(excluded_offset, excluded); + writer->long_field_put(thread_id_offset, current_tid); + } + } + return h_writer; } jobject JfrJavaEventWriter::new_event_writer(TRAPS) { @@ -227,8 +253,8 @@ jobject JfrJavaEventWriter::new_event_writer(TRAPS) { JfrJavaSupport::throw_out_of_memory_error("OOME for thread local buffer", THREAD); return NULL; } - jobject java_event_writer = create_new_event_writer(buffer, CHECK_NULL); - tl->set_java_event_writer(java_event_writer); + jobject h_writer = create_new_event_writer(buffer, tl, CHECK_NULL); + tl->set_java_event_writer(h_writer); assert(tl->has_java_event_writer(), "invariant"); - return java_event_writer; + return h_writer; } diff --git a/src/hotspot/share/jfr/writers/jfrJavaEventWriter.hpp b/src/hotspot/share/jfr/writers/jfrJavaEventWriter.hpp index 1159766428e..af99690ba47 100644 --- a/src/hotspot/share/jfr/writers/jfrJavaEventWriter.hpp +++ b/src/hotspot/share/jfr/writers/jfrJavaEventWriter.hpp @@ -26,7 +26,9 @@ #define SHARE_JFR_WRITERS_JFRJAVAEVENTWRITER_HPP #include "jni.h" +#include "jfr/utilities/jfrTypes.hpp" #include "memory/allStatic.hpp" +#include "utilities/exceptions.hpp" class JavaThread; class Thread; @@ -42,6 +44,8 @@ class JfrJavaEventWriter : AllStatic { public: static void notify(); + static void exclude(traceid tid, const JavaThread* jt); + static void include(traceid tid, const JavaThread* jt); static jobject event_writer(JavaThread* t); static jobject new_event_writer(TRAPS); static jboolean flush(jobject writer, jint used, jint requested, JavaThread* jt); diff --git a/src/hotspot/share/jfr/writers/jfrStorageAdapter.hpp b/src/hotspot/share/jfr/writers/jfrStorageAdapter.hpp index c8459bae7af..f6453ff956f 100644 --- a/src/hotspot/share/jfr/writers/jfrStorageAdapter.hpp +++ b/src/hotspot/share/jfr/writers/jfrStorageAdapter.hpp @@ -79,10 +79,10 @@ class Adapter { } bool flush(size_t used, size_t requested) { - assert(_thread != NULL, "invariant"); + assert(_thread != nullptr, "invariant"); Flush f(_storage, used, requested, _thread); _storage = f.result(); - return _storage != NULL && !_storage->excluded(); + return _storage != nullptr; } void release() { diff --git a/src/hotspot/share/jfr/writers/jfrStreamWriterHost.hpp b/src/hotspot/share/jfr/writers/jfrStreamWriterHost.hpp index 5487692fbeb..0defe7390ea 100644 --- a/src/hotspot/share/jfr/writers/jfrStreamWriterHost.hpp +++ b/src/hotspot/share/jfr/writers/jfrStreamWriterHost.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2016, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -52,6 +52,7 @@ class StreamWriterHost : public MemoryWriterHost { int64_t current_offset() const; void seek(int64_t offset); void flush(); + void write_buffered(const void* src, intptr_t len); void write_unbuffered(const void* src, intptr_t len); bool is_valid() const; void close_fd(); diff --git a/src/hotspot/share/jfr/writers/jfrStreamWriterHost.inline.hpp b/src/hotspot/share/jfr/writers/jfrStreamWriterHost.inline.hpp index 73404a1aede..652561981f5 100644 --- a/src/hotspot/share/jfr/writers/jfrStreamWriterHost.inline.hpp +++ b/src/hotspot/share/jfr/writers/jfrStreamWriterHost.inline.hpp @@ -123,6 +123,11 @@ void StreamWriterHost::flush() { } } +template +void StreamWriterHost::write_buffered(const void* buf, intptr_t len) { + this->write_bytes(this->current_pos(), (const u1*)buf, len); +} + template void StreamWriterHost::write_unbuffered(const void* buf, intptr_t len) { this->flush(); diff --git a/src/hotspot/share/jfr/writers/jfrWriterHost.hpp b/src/hotspot/share/jfr/writers/jfrWriterHost.hpp index 14c66e2f971..753d32b40a3 100644 --- a/src/hotspot/share/jfr/writers/jfrWriterHost.hpp +++ b/src/hotspot/share/jfr/writers/jfrWriterHost.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2016, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -90,6 +90,7 @@ class WriterHost : public WriterPolicyImpl { void write(const JfrTickspan& time); void write_bytes(const void* buf, intptr_t len); void write_utf8_u2_len(const char* value); + void write_empty_string(); template void write_padded_at_offset(T value, int64_t offset); template diff --git a/src/hotspot/share/jfr/writers/jfrWriterHost.inline.hpp b/src/hotspot/share/jfr/writers/jfrWriterHost.inline.hpp index e09a574e63a..6c677b1a808 100644 --- a/src/hotspot/share/jfr/writers/jfrWriterHost.inline.hpp +++ b/src/hotspot/share/jfr/writers/jfrWriterHost.inline.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2016, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -207,6 +207,11 @@ inline void WriterHost::write(char* value) { write(const_cast(value)); } +template +inline void WriterHost::write_empty_string() { + write(EMPTY_STRING); +} + template inline void WriterHost::write(jstring string) { if (string == NULL) { @@ -217,7 +222,7 @@ inline void WriterHost::write(jstring string) { assert(string_oop != NULL, "invariant"); const size_t length = (size_t)java_lang_String::length(string_oop); if (0 == length) { - write(EMPTY_STRING); + write_empty_string(); return; } const bool is_latin1_encoded = java_lang_String::is_latin1(string_oop); diff --git a/src/hotspot/share/jvmci/jvmciCodeInstaller.cpp b/src/hotspot/share/jvmci/jvmciCodeInstaller.cpp index cd46aa5c523..5a5f16f9304 100644 --- a/src/hotspot/share/jvmci/jvmciCodeInstaller.cpp +++ b/src/hotspot/share/jvmci/jvmciCodeInstaller.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2011, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -541,7 +541,7 @@ JVMCI::CodeInstallResult CodeInstaller::install(JVMCICompiler* compiler, result = runtime()->register_method(jvmci_env(), method, nmethod_handle, entry_bci, &_offsets, _orig_pc_offset, &buffer, stack_slots, _debug_recorder->_oopmaps, &_exception_handler_table, &_implicit_exception_table, compiler, _debug_recorder, _dependencies, id, - has_unsafe_access, _has_wide_vector, compiled_code, mirror, + _has_monitors, has_unsafe_access, _has_wide_vector, compiled_code, mirror, failed_speculations, speculations, speculations_len); if (result == JVMCI::ok) { nmethod* nm = nmethod_handle.code()->as_nmethod_or_null(); @@ -1003,6 +1003,9 @@ void CodeInstaller::record_scope(jint pc_offset, JVMCIObject position, ScopeMode jint local_count = jvmci_env()->get_BytecodeFrame_numLocals(frame); jint expression_count = jvmci_env()->get_BytecodeFrame_numStack(frame); jint monitor_count = jvmci_env()->get_BytecodeFrame_numLocks(frame); + if (monitor_count > 0 && !_has_monitors) { + _has_monitors = true; + } JVMCIObjectArray values = jvmci_env()->get_BytecodeFrame_values(frame); JVMCIObjectArray slotKinds = jvmci_env()->get_BytecodeFrame_slotKinds(frame); diff --git a/src/hotspot/share/jvmci/jvmciCodeInstaller.hpp b/src/hotspot/share/jvmci/jvmciCodeInstaller.hpp index 2c8e231c7c5..f83797c9346 100644 --- a/src/hotspot/share/jvmci/jvmciCodeInstaller.hpp +++ b/src/hotspot/share/jvmci/jvmciCodeInstaller.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2011, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -136,6 +136,7 @@ private: jint _parameter_count; jint _constants_size; + bool _has_monitors; bool _has_wide_vector; MarkId _next_call_type; diff --git a/src/hotspot/share/jvmci/jvmciCompilerToVM.cpp b/src/hotspot/share/jvmci/jvmciCompilerToVM.cpp index 73ed9b59941..4fa2987854c 100644 --- a/src/hotspot/share/jvmci/jvmciCompilerToVM.cpp +++ b/src/hotspot/share/jvmci/jvmciCompilerToVM.cpp @@ -1584,12 +1584,12 @@ C2V_VMENTRY(void, materializeVirtualObjects, (JNIEnv* env, jobject, jobject _hs_ for (int frame_index = 0; frame_index < virtualFrames->length(); frame_index++) { compiledVFrame* cvf = virtualFrames->at(frame_index); - GrowableArray* scopeLocals = cvf->scope()->locals(); + GrowableArray* extentLocals = cvf->scope()->locals(); StackValueCollection* locals = cvf->locals(); if (locals != NULL) { for (int i2 = 0; i2 < locals->size(); i2++) { StackValue* var = locals->at(i2); - if (var->type() == T_OBJECT && scopeLocals->at(i2)->is_object()) { + if (var->type() == T_OBJECT && extentLocals->at(i2)->is_object()) { jvalue val; val.l = cast_from_oop(locals->at(i2)->get_obj()()); cvf->update_local(T_OBJECT, i2, val); diff --git a/src/hotspot/share/jvmci/jvmciRuntime.cpp b/src/hotspot/share/jvmci/jvmciRuntime.cpp index 8d58d390514..9363053d666 100644 --- a/src/hotspot/share/jvmci/jvmciRuntime.cpp +++ b/src/hotspot/share/jvmci/jvmciRuntime.cpp @@ -2038,6 +2038,7 @@ JVMCI::CodeInstallResult JVMCIRuntime::register_method(JVMCIEnv* JVMCIENV, DebugInformationRecorder* debug_info, Dependencies* dependencies, int compile_id, + bool has_monitors, bool has_unsafe_access, bool has_wide_vector, JVMCIObject compiled_code, @@ -2135,6 +2136,7 @@ JVMCI::CodeInstallResult JVMCIRuntime::register_method(JVMCIEnv* JVMCIENV, } else { nm->set_has_unsafe_access(has_unsafe_access); nm->set_has_wide_vectors(has_wide_vector); + nm->set_has_monitors(has_monitors); // Record successful registration. // (Put nm into the task handle *before* publishing to the Java heap.) diff --git a/src/hotspot/share/jvmci/jvmciRuntime.hpp b/src/hotspot/share/jvmci/jvmciRuntime.hpp index 3b30977282c..d1be337df95 100644 --- a/src/hotspot/share/jvmci/jvmciRuntime.hpp +++ b/src/hotspot/share/jvmci/jvmciRuntime.hpp @@ -413,6 +413,7 @@ class JVMCIRuntime: public CHeapObj { DebugInformationRecorder* debug_info, Dependencies* dependencies, int compile_id, + bool has_monitors, bool has_unsafe_access, bool has_wide_vector, JVMCIObject compiled_code, diff --git a/src/hotspot/share/jvmci/vmStructs_jvmci.cpp b/src/hotspot/share/jvmci/vmStructs_jvmci.cpp index 9f091e04489..d32d39f9095 100644 --- a/src/hotspot/share/jvmci/vmStructs_jvmci.cpp +++ b/src/hotspot/share/jvmci/vmStructs_jvmci.cpp @@ -171,6 +171,7 @@ nonstatic_field(JVMCICompileState, _compilation_ticks, jint) \ \ nonstatic_field(JavaThread, _threadObj, OopHandle) \ + nonstatic_field(JavaThread, _vthread, OopHandle) \ nonstatic_field(JavaThread, _anchor, JavaFrameAnchor) \ nonstatic_field(JavaThread, _vm_result, oop) \ nonstatic_field(JavaThread, _stack_overflow_state._stack_overflow_limit, address) \ @@ -190,6 +191,7 @@ nonstatic_field(JavaThread, _jni_environment, JNIEnv) \ nonstatic_field(JavaThread, _poll_data, SafepointMechanism::ThreadData) \ nonstatic_field(JavaThread, _stack_overflow_state._reserved_stack_activation, address) \ + nonstatic_field(JavaThread, _held_monitor_count, int) \ \ static_field(java_lang_Class, _klass_offset, int) \ static_field(java_lang_Class, _array_klass_offset, int) \ @@ -335,6 +337,8 @@ static_field(StubRoutines, _vectorizedMismatch, address) \ static_field(StubRoutines, _bigIntegerRightShiftWorker, address) \ static_field(StubRoutines, _bigIntegerLeftShiftWorker, address) \ + static_field(StubRoutines, _cont_doYield, address) \ + static_field(StubRoutines, _cont_thaw, address) \ \ nonstatic_field(Thread, _tlab, ThreadLocalAllocBuffer) \ nonstatic_field(Thread, _allocated_bytes, jlong) \ @@ -619,6 +623,7 @@ declare_constant(Method::_hidden) \ declare_constant(Method::_intrinsic_candidate) \ declare_constant(Method::_reserved_stack_access) \ + declare_constant(Method::_changes_current_thread) \ \ declare_constant(Method::nonvirtual_vtable_index) \ declare_constant(Method::invalid_vtable_index) \ diff --git a/src/hotspot/share/logging/logTag.hpp b/src/hotspot/share/logging/logTag.hpp index 9b216a9d35f..d5271c9e615 100644 --- a/src/hotspot/share/logging/logTag.hpp +++ b/src/hotspot/share/logging/logTag.hpp @@ -58,6 +58,7 @@ class outputStream; LOG_TAG(constantpool) \ LOG_TAG(constraints) \ LOG_TAG(container) \ + LOG_TAG(continuations) \ LOG_TAG(coops) \ LOG_TAG(cpu) \ LOG_TAG(cset) \ @@ -142,6 +143,7 @@ class outputStream; LOG_TAG(phases) \ LOG_TAG(plab) \ LOG_TAG(placeholders) \ + LOG_TAG(preempt) \ LOG_TAG(preorder) /* Trace all classes loaded in order referenced (not loaded) */ \ LOG_TAG(preview) /* Trace loading of preview feature types */ \ LOG_TAG(promotion) \ diff --git a/src/hotspot/share/memory/iterator.cpp b/src/hotspot/share/memory/iterator.cpp index 6790289ae54..08470b4ed97 100644 --- a/src/hotspot/share/memory/iterator.cpp +++ b/src/hotspot/share/memory/iterator.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -25,6 +25,7 @@ #include "precompiled.hpp" #include "classfile/classLoaderDataGraph.hpp" #include "code/nmethod.hpp" +#include "gc/shared/barrierSetNMethod.hpp" #include "memory/iterator.inline.hpp" #include "oops/oop.inline.hpp" #include "utilities/debug.hpp" @@ -57,7 +58,22 @@ void CodeBlobToOopClosure::do_code_blob(CodeBlob* cb) { void MarkingCodeBlobClosure::do_code_blob(CodeBlob* cb) { nmethod* nm = cb->as_nmethod_or_null(); if (nm != NULL && nm->oops_do_try_claim()) { - do_nmethod(nm); + // Process the oops in the nmethod + nm->oops_do(_cl); + + if (_keepalive_nmethods) { + // CodeCache sweeper support + nm->mark_as_maybe_on_continuation(); + + BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod(); + if (bs_nm != NULL) { + bs_nm->disarm(nm); + } + } + + if (_fix_relocations) { + nm->fix_oop_relocations(); + } } } diff --git a/src/hotspot/share/memory/iterator.hpp b/src/hotspot/share/memory/iterator.hpp index 9f707a266f5..dcbfe44b0a5 100644 --- a/src/hotspot/share/memory/iterator.hpp +++ b/src/hotspot/share/memory/iterator.hpp @@ -28,6 +28,7 @@ #include "memory/allocation.hpp" #include "memory/memRegion.hpp" #include "oops/oopsHierarchy.hpp" +#include "utilities/bitMap.hpp" class CodeBlob; class nmethod; @@ -98,10 +99,17 @@ class OopIterateClosure : public OopClosure { // 1) do_klass on the header klass pointer. // 2) do_klass on the klass pointer in the mirrors. // 3) do_cld on the class loader data in class loaders. + // + // Used to determine metadata liveness for class unloading GCs. virtual bool do_metadata() = 0; virtual void do_klass(Klass* k) = 0; virtual void do_cld(ClassLoaderData* cld) = 0; + + // Class redefinition needs to get notified about methods from stackChunkOops + virtual void do_method(Method* m) = 0; + // The code cache sweeper needs to get notified about methods from stackChunkOops + virtual void do_nmethod(nmethod* nm) = 0; }; // An OopIterateClosure that can be used when there's no need to visit the Metadata. @@ -112,6 +120,15 @@ public: virtual bool do_metadata() { return false; } virtual void do_klass(Klass* k) { ShouldNotReachHere(); } virtual void do_cld(ClassLoaderData* cld) { ShouldNotReachHere(); } + virtual void do_method(Method* m) { ShouldNotReachHere(); } + virtual void do_nmethod(nmethod* nm) { ShouldNotReachHere(); } +}; + +enum class derived_pointer : intptr_t; +class DerivedOopClosure : public Closure { + public: + enum { SkipNull = true }; + virtual void do_derived_oop(oop* base, derived_pointer* derived) = 0; }; class KlassClosure : public Closure { @@ -161,6 +178,8 @@ class ClaimMetadataVisitingOopIterateClosure : public OopIterateClosure { virtual bool do_metadata() { return true; } virtual void do_klass(Klass* k); virtual void do_cld(ClassLoaderData* cld); + virtual void do_method(Method* m); + virtual void do_nmethod(nmethod* nm); }; // The base class for all concurrent marking closures, @@ -234,9 +253,9 @@ class CodeBlobClosure : public Closure { // Applies an oop closure to all ref fields in code blobs // iterated over in an object iteration. class CodeBlobToOopClosure : public CodeBlobClosure { + protected: OopClosure* _cl; bool _fix_relocations; - protected: void do_nmethod(nmethod* nm); public: // If fix_relocations(), then cl must copy objects to their new location immediately to avoid @@ -249,10 +268,14 @@ class CodeBlobToOopClosure : public CodeBlobClosure { }; class MarkingCodeBlobClosure : public CodeBlobToOopClosure { - public: - MarkingCodeBlobClosure(OopClosure* cl, bool fix_relocations) : CodeBlobToOopClosure(cl, fix_relocations) {} - // Called for each code blob, but at most once per unique blob. + bool _keepalive_nmethods; + public: + MarkingCodeBlobClosure(OopClosure* cl, bool fix_relocations, bool keepalive_nmethods) : + CodeBlobToOopClosure(cl, fix_relocations), + _keepalive_nmethods(keepalive_nmethods) {} + + // Called for each code blob, but at most once per unique blob. virtual void do_code_blob(CodeBlob* cb); }; @@ -355,6 +378,8 @@ class Devirtualizer { template static void do_klass(OopClosureType* closure, Klass* k); template static void do_cld(OopClosureType* closure, ClassLoaderData* cld); template static bool do_metadata(OopClosureType* closure); + template static void do_derived_oop(DerivedOopClosureType* closure, oop* base, derived_pointer* derived); + template static bool do_bit(BitMapClosureType* closure, BitMap::idx_t index); }; class OopIteratorClosureDispatch { diff --git a/src/hotspot/share/memory/iterator.inline.hpp b/src/hotspot/share/memory/iterator.inline.hpp index 90d705ccab6..1a2bf486417 100644 --- a/src/hotspot/share/memory/iterator.inline.hpp +++ b/src/hotspot/share/memory/iterator.inline.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2014, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -28,6 +28,7 @@ #include "memory/iterator.hpp" #include "classfile/classLoaderData.hpp" +#include "code/nmethod.hpp" #include "oops/access.inline.hpp" #include "oops/compressedOops.inline.hpp" #include "oops/klass.hpp" @@ -35,6 +36,7 @@ #include "oops/instanceMirrorKlass.inline.hpp" #include "oops/instanceClassLoaderKlass.inline.hpp" #include "oops/instanceRefKlass.inline.hpp" +#include "oops/instanceStackChunkKlass.inline.hpp" #include "oops/objArrayKlass.inline.hpp" #include "oops/typeArrayKlass.inline.hpp" #include "utilities/debug.hpp" @@ -52,6 +54,15 @@ inline void ClaimMetadataVisitingOopIterateClosure::do_klass(Klass* k) { ClaimMetadataVisitingOopIterateClosure::do_cld(cld); } +inline void ClaimMetadataVisitingOopIterateClosure::do_nmethod(nmethod* nm) { + nm->follow_nmethod(this); +} + +inline void ClaimMetadataVisitingOopIterateClosure::do_method(Method* m) { + // Mark interpreted frames for class redefinition + m->record_gc_epoch(); +} + // Implementation of the non-virtual do_oop dispatch. // // The same implementation is used for do_metadata, do_klass, and do_cld. @@ -169,6 +180,25 @@ void Devirtualizer::do_cld(OopClosureType* closure, ClassLoaderData* cld) { call_do_cld(&OopClosureType::do_cld, &OopIterateClosure::do_cld, closure, cld); } +// Implementation of the non-virtual do_derived_oop dispatch. + +template +static typename EnableIf::value, void>::type +call_do_derived_oop(void (Receiver::*)(oop*, derived_pointer*), void (Base::*)(oop*, derived_pointer*), DerivedOopClosureType* closure, oop* base, derived_pointer* derived) { + closure->do_derived_oop(base, derived); +} + +template +static typename EnableIf::value, void>::type +call_do_derived_oop(void (Receiver::*)(oop*, derived_pointer*), void (Base::*)(oop*, derived_pointer*), DerivedOopClosureType* closure, oop* base, derived_pointer* derived) { + closure->DerivedOopClosureType::do_derived_oop(base, derived); +} + +template +inline void Devirtualizer::do_derived_oop(DerivedOopClosureType* closure, oop* base, derived_pointer* derived) { + call_do_derived_oop(&DerivedOopClosureType::do_derived_oop, &DerivedOopClosure::do_derived_oop, closure, base, derived); +} + // Dispatch table implementation for *Klass::oop_oop_iterate // // It allows for a single call to do a multi-dispatch to an optimized version @@ -252,6 +282,7 @@ private: set_init_function(); set_init_function(); set_init_function(); + set_init_function(); set_init_function(); set_init_function(); } @@ -314,6 +345,7 @@ private: set_init_function(); set_init_function(); set_init_function(); + set_init_function(); set_init_function(); set_init_function(); } @@ -376,6 +408,7 @@ private: set_init_function(); set_init_function(); set_init_function(); + set_init_function(); set_init_function(); set_init_function(); } diff --git a/src/hotspot/share/oops/accessBackend.cpp b/src/hotspot/share/oops/accessBackend.cpp index 3317cc5153e..7352eab3ca7 100644 --- a/src/hotspot/share/oops/accessBackend.cpp +++ b/src/hotspot/share/oops/accessBackend.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -27,8 +27,11 @@ #include "gc/shared/collectedHeap.hpp" #include "oops/oop.inline.hpp" #include "runtime/mutexLocker.hpp" +#include "runtime/thread.inline.hpp" #include "runtime/vm_version.hpp" #include "utilities/copy.hpp" +#include "utilities/debug.hpp" +#include "utilities/vmError.hpp" namespace AccessInternal { // VM_Version::supports_cx8() is a surrogate for 'supports atomic long memory ops'. @@ -203,4 +206,22 @@ namespace AccessInternal { void arraycopy_conjoint_atomic(void* src, void* dst, size_t length) { Copy::conjoint_memory_atomic(src, dst, length); } + +#ifdef ASSERT + void check_access_thread_state() { + if (VMError::is_error_reported() || Debugging) { + return; + } + + Thread* thread = Thread::current(); + if (!thread->is_Java_thread()) { + return; + } + + JavaThread* java_thread = JavaThread::cast(thread); + JavaThreadState state = java_thread->thread_state(); + assert(state == _thread_in_vm || state == _thread_in_Java || state == _thread_new, + "Wrong thread state for accesses: %d", (int)state); + } +#endif } diff --git a/src/hotspot/share/oops/accessBackend.hpp b/src/hotspot/share/oops/accessBackend.hpp index 4fa92c53c16..3bcfe797e0c 100644 --- a/src/hotspot/share/oops/accessBackend.hpp +++ b/src/hotspot/share/oops/accessBackend.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -390,6 +390,11 @@ public: static void clone(oop src, oop dst, size_t size); }; +namespace AccessInternal { + DEBUG_ONLY(void check_access_thread_state()); +#define assert_access_thread_state() DEBUG_ONLY(check_access_thread_state()) +} + // Below is the implementation of the first 4 steps of the template pipeline: // * Step 1: Set default decorators and decay types. This step gets rid of CV qualifiers // and sets default decorators to sensible values. @@ -451,6 +456,7 @@ namespace AccessInternal { static void store_init(void* addr, T value); static inline void store(void* addr, T value) { + assert_access_thread_state(); _store_func(addr, value); } }; @@ -463,6 +469,7 @@ namespace AccessInternal { static void store_at_init(oop base, ptrdiff_t offset, T value); static inline void store_at(oop base, ptrdiff_t offset, T value) { + assert_access_thread_state(); _store_at_func(base, offset, value); } }; @@ -475,6 +482,7 @@ namespace AccessInternal { static T load_init(void* addr); static inline T load(void* addr) { + assert_access_thread_state(); return _load_func(addr); } }; @@ -487,6 +495,7 @@ namespace AccessInternal { static T load_at_init(oop base, ptrdiff_t offset); static inline T load_at(oop base, ptrdiff_t offset) { + assert_access_thread_state(); return _load_at_func(base, offset); } }; @@ -499,6 +508,7 @@ namespace AccessInternal { static T atomic_cmpxchg_init(void* addr, T compare_value, T new_value); static inline T atomic_cmpxchg(void* addr, T compare_value, T new_value) { + assert_access_thread_state(); return _atomic_cmpxchg_func(addr, compare_value, new_value); } }; @@ -511,6 +521,7 @@ namespace AccessInternal { static T atomic_cmpxchg_at_init(oop base, ptrdiff_t offset, T compare_value, T new_value); static inline T atomic_cmpxchg_at(oop base, ptrdiff_t offset, T compare_value, T new_value) { + assert_access_thread_state(); return _atomic_cmpxchg_at_func(base, offset, compare_value, new_value); } }; @@ -523,6 +534,7 @@ namespace AccessInternal { static T atomic_xchg_init(void* addr, T new_value); static inline T atomic_xchg(void* addr, T new_value) { + assert_access_thread_state(); return _atomic_xchg_func(addr, new_value); } }; @@ -535,6 +547,7 @@ namespace AccessInternal { static T atomic_xchg_at_init(oop base, ptrdiff_t offset, T new_value); static inline T atomic_xchg_at(oop base, ptrdiff_t offset, T new_value) { + assert_access_thread_state(); return _atomic_xchg_at_func(base, offset, new_value); } }; @@ -551,6 +564,7 @@ namespace AccessInternal { static inline bool arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw, arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw, size_t length) { + assert_access_thread_state(); return _arraycopy_func(src_obj, src_offset_in_bytes, src_raw, dst_obj, dst_offset_in_bytes, dst_raw, length); @@ -565,6 +579,7 @@ namespace AccessInternal { static void clone_init(oop src, oop dst, size_t size); static inline void clone(oop src, oop dst, size_t size) { + assert_access_thread_state(); _clone_func(src, dst, size); } }; diff --git a/src/hotspot/share/oops/constMethod.hpp b/src/hotspot/share/oops/constMethod.hpp index dfa4a352c2d..f7c22f42fe0 100644 --- a/src/hotspot/share/oops/constMethod.hpp +++ b/src/hotspot/share/oops/constMethod.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2003, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2003, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -217,6 +217,7 @@ private: u2 _max_stack; // Maximum number of entries on the expression stack u2 _max_locals; // Number of local variables used by this method u2 _size_of_parameters; // size of the parameter block (receiver + arguments) in words + u2 _num_stack_arg_slots; // Number of arguments passed on the stack even when compiled u2 _orig_method_idnum; // Original unique identification number for the method // Constructor @@ -491,6 +492,10 @@ public: int size_of_parameters() const { return _size_of_parameters; } void set_size_of_parameters(int size) { _size_of_parameters = size; } + // Number of arguments passed on the stack even when compiled + int num_stack_arg_slots() const { return _num_stack_arg_slots; } + void set_num_stack_arg_slots(int n) { _num_stack_arg_slots = n; } + // result type (basic type of return value) BasicType result_type() const { assert(_result_type >= T_BOOLEAN, "Must be set"); return (BasicType)_result_type; } diff --git a/src/hotspot/share/oops/constantPool.cpp b/src/hotspot/share/oops/constantPool.cpp index 46b4e4575ae..289647e7dad 100644 --- a/src/hotspot/share/oops/constantPool.cpp +++ b/src/hotspot/share/oops/constantPool.cpp @@ -32,6 +32,7 @@ #include "classfile/systemDictionary.hpp" #include "classfile/vmClasses.hpp" #include "classfile/vmSymbols.hpp" +#include "code/codeCache.hpp" #include "interpreter/bootstrapInfo.hpp" #include "interpreter/linkResolver.hpp" #include "logging/log.hpp" @@ -2212,6 +2213,31 @@ int ConstantPool::copy_cpool_bytes(int cpool_size, #undef DBG +bool ConstantPool::is_maybe_on_continuation_stack() const { + // This method uses the similar logic as nmethod::is_maybe_on_continuation_stack() + if (!Continuations::enabled()) { + return false; + } + + // If the condition below is true, it means that the nmethod was found to + // be alive the previous completed marking cycle. + return cache()->gc_epoch() >= Continuations::previous_completed_gc_marking_cycle(); +} + +// For redefinition, if any methods found in loom stack chunks, the gc_epoch is +// recorded in their constant pool cache. The on_stack-ness of the constant pool controls whether +// memory for the method is reclaimed. +bool ConstantPool::on_stack() const { + if ((_flags &_on_stack) != 0) { + return true; + } + + if (_cache == nullptr) { + return false; + } + + return is_maybe_on_continuation_stack(); +} void ConstantPool::set_on_stack(const bool value) { if (value) { diff --git a/src/hotspot/share/oops/constantPool.hpp b/src/hotspot/share/oops/constantPool.hpp index 4fddadae088..728f07a82e4 100644 --- a/src/hotspot/share/oops/constantPool.hpp +++ b/src/hotspot/share/oops/constantPool.hpp @@ -205,7 +205,8 @@ class ConstantPool : public Metadata { // is on the executing stack, or as a handle in vm code, this constant pool // can't be removed from the set of previous versions saved in the instance // class. - bool on_stack() const { return (_flags &_on_stack) != 0; } + bool on_stack() const; + bool is_maybe_on_continuation_stack() const; void set_on_stack(const bool value); // Faster than MetaspaceObj::is_shared() - used by set_on_stack() diff --git a/src/hotspot/share/oops/cpCache.cpp b/src/hotspot/share/oops/cpCache.cpp index 40ca7d7efc6..9ce5eac4d01 100644 --- a/src/hotspot/share/oops/cpCache.cpp +++ b/src/hotspot/share/oops/cpCache.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1998, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1998, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -46,6 +46,7 @@ #include "prims/methodHandles.hpp" #include "runtime/arguments.hpp" #include "runtime/atomic.hpp" +#include "runtime/continuation.hpp" #include "runtime/handles.inline.hpp" #include "runtime/vm_version.hpp" #include "utilities/macros.hpp" @@ -705,6 +706,11 @@ void ConstantPoolCache::initialize(const intArray& inverse_index_map, } } +// Record the GC marking cycle when redefined vs. when found in the loom stack chunks. +void ConstantPoolCache::record_gc_epoch() { + _gc_epoch = Continuations::gc_epoch(); +} + void ConstantPoolCache::verify_just_initialized() { DEBUG_ONLY(walk_entries_for_initialization(/*check_only = */ true)); } diff --git a/src/hotspot/share/oops/cpCache.hpp b/src/hotspot/share/oops/cpCache.hpp index dd27a691f6d..53df7a1b92e 100644 --- a/src/hotspot/share/oops/cpCache.hpp +++ b/src/hotspot/share/oops/cpCache.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1998, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1998, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -412,6 +412,10 @@ class ConstantPoolCache: public MetaspaceObj { // object index to original constant pool index OopHandle _resolved_references; Array* _reference_map; + + // RedefineClasses support + uint64_t _gc_epoch; + // The narrowOop pointer to the archived resolved_references. Set at CDS dump // time when caching java heap object is supported. CDS_JAVA_HEAP_ONLY(int _archived_references_index;) @@ -505,6 +509,8 @@ class ConstantPoolCache: public MetaspaceObj { DEBUG_ONLY(bool on_stack() { return false; }) void deallocate_contents(ClassLoaderData* data); bool is_klass() const { return false; } + void record_gc_epoch(); + uint64_t gc_epoch() { return _gc_epoch; } // Printing void print_on(outputStream* st) const; diff --git a/src/hotspot/share/oops/cpCache.inline.hpp b/src/hotspot/share/oops/cpCache.inline.hpp index 8db18befc74..032677c0f92 100644 --- a/src/hotspot/share/oops/cpCache.inline.hpp +++ b/src/hotspot/share/oops/cpCache.inline.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -89,7 +89,8 @@ inline ConstantPoolCache::ConstantPoolCache(int length, const intStack& invokedynamic_inverse_index_map, const intStack& invokedynamic_references_map) : _length(length), - _constant_pool(NULL) { + _constant_pool(NULL), + _gc_epoch(0) { CDS_JAVA_HEAP_ONLY(_archived_references_index = -1;) initialize(inverse_index_map, invokedynamic_inverse_index_map, invokedynamic_references_map); diff --git a/src/hotspot/share/oops/instanceClassLoaderKlass.hpp b/src/hotspot/share/oops/instanceClassLoaderKlass.hpp index 559878a1c67..f5a6135c049 100644 --- a/src/hotspot/share/oops/instanceClassLoaderKlass.hpp +++ b/src/hotspot/share/oops/instanceClassLoaderKlass.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2011, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -50,7 +50,7 @@ public: // Oop fields (and metadata) iterators // - // The InstanceClassLoaderKlass iterators also visit the CLD pointer (or mirror of hidden klasses.) + // The InstanceClassLoaderKlass iterators also visit the CLD pointer // Forward iteration // Iterate over the oop fields and metadata. diff --git a/src/hotspot/share/oops/instanceKlass.cpp b/src/hotspot/share/oops/instanceKlass.cpp index ca98bbe1520..86e6d17f6dd 100644 --- a/src/hotspot/share/oops/instanceKlass.cpp +++ b/src/hotspot/share/oops/instanceKlass.cpp @@ -65,6 +65,7 @@ #include "oops/instanceKlass.inline.hpp" #include "oops/instanceMirrorKlass.hpp" #include "oops/instanceOop.hpp" +#include "oops/instanceStackChunkKlass.hpp" #include "oops/klass.inline.hpp" #include "oops/method.hpp" #include "oops/oop.inline.hpp" @@ -163,6 +164,12 @@ static inline bool is_class_loader(const Symbol* class_name, return false; } +static inline bool is_stack_chunk_class(const Symbol* class_name, + const ClassLoaderData* loader_data) { + return (class_name == vmSymbols::jdk_internal_vm_StackChunk() && + loader_data->is_the_null_class_loader_data()); +} + // private: called to verify that k is a static member of this nest. // We know that k is an instance class in the same package and hence the // same classloader. @@ -440,17 +447,20 @@ InstanceKlass* InstanceKlass::allocate_instance_klass(const ClassFileParser& par // Allocation if (REF_NONE == parser.reference_type()) { if (class_name == vmSymbols::java_lang_Class()) { - // mirror + // mirror - java.lang.Class ik = new (loader_data, size, THREAD) InstanceMirrorKlass(parser); + } else if (is_stack_chunk_class(class_name, loader_data)) { + // stack chunk + ik = new (loader_data, size, THREAD) InstanceStackChunkKlass(parser); } else if (is_class_loader(class_name, parser)) { - // class loader + // class loader - java.lang.ClassLoader ik = new (loader_data, size, THREAD) InstanceClassLoaderKlass(parser); } else { // normal ik = new (loader_data, size, THREAD) InstanceKlass(parser); } } else { - // reference + // java.lang.ref.Reference ik = new (loader_data, size, THREAD) InstanceRefKlass(parser); } diff --git a/src/hotspot/share/oops/instanceStackChunkKlass.cpp b/src/hotspot/share/oops/instanceStackChunkKlass.cpp new file mode 100644 index 00000000000..fb5e2282d4d --- /dev/null +++ b/src/hotspot/share/oops/instanceStackChunkKlass.cpp @@ -0,0 +1,265 @@ +/* + * Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "classfile/javaClasses.inline.hpp" +#include "classfile/vmClasses.hpp" +#include "compiler/oopMap.inline.hpp" +#include "memory/iterator.inline.hpp" +#include "memory/oopFactory.hpp" +#include "memory/resourceArea.hpp" +#include "oops/instanceStackChunkKlass.inline.hpp" +#include "oops/stackChunkOop.inline.hpp" +#include "runtime/continuation.hpp" +#include "runtime/frame.hpp" +#include "runtime/handles.hpp" +#include "runtime/registerMap.hpp" +#include "runtime/smallRegisterMap.inline.hpp" +#include "runtime/stackChunkFrameStream.inline.hpp" +#include "utilities/globalDefinitions.hpp" +#include "utilities/macros.hpp" +#include "utilities/ostream.hpp" + +int InstanceStackChunkKlass::_offset_of_stack = 0; + +#if INCLUDE_CDS +void InstanceStackChunkKlass::serialize_offsets(SerializeClosure* f) { + f->do_u4((u4*)&_offset_of_stack); +} +#endif + +InstanceStackChunkKlass::InstanceStackChunkKlass(const ClassFileParser& parser) + : InstanceKlass(parser, Kind) { + // Change the layout_helper to use the slow path because StackChunkOops are + // variable sized InstanceOops. + const jint lh = Klass::instance_layout_helper(size_helper(), true); + set_layout_helper(lh); +} + +size_t InstanceStackChunkKlass::oop_size(oop obj) const { + return instance_size(jdk_internal_vm_StackChunk::size(obj)); +} + +#ifndef PRODUCT +void InstanceStackChunkKlass::oop_print_on(oop obj, outputStream* st) { + print_chunk(stackChunkOopDesc::cast(obj), false, st); +} +#endif + +template +class StackChunkOopIterateFilterClosure: public OopClosure { +private: + OopClosureType* const _closure; + MemRegion _bound; + +public: + + StackChunkOopIterateFilterClosure(OopClosureType* closure, MemRegion bound) + : _closure(closure), + _bound(bound) {} + + virtual void do_oop(oop* p) override { do_oop_work(p); } + virtual void do_oop(narrowOop* p) override { do_oop_work(p); } + + template + void do_oop_work(T* p) { + if (_bound.contains(p)) { + Devirtualizer::do_oop(_closure, p); + } + } +}; + +class DoMethodsStackChunkFrameClosure { + OopIterateClosure* _closure; + +public: + DoMethodsStackChunkFrameClosure(OopIterateClosure* cl) : _closure(cl) {} + + template + bool do_frame(const StackChunkFrameStream& f, const RegisterMapT* map) { + if (f.is_interpreted()) { + Method* m = f.to_frame().interpreter_frame_method(); + _closure->do_method(m); + } else if (f.is_compiled()) { + nmethod* nm = f.cb()->as_nmethod(); + // The do_nmethod function takes care of having the right synchronization + // when keeping the nmethod alive during concurrent execution. + _closure->do_nmethod(nm); + // There is no need to mark the Method, as class redefinition will walk the + // CodeCache, noting their Methods + } + return true; + } +}; + +void InstanceStackChunkKlass::do_methods(stackChunkOop chunk, OopIterateClosure* cl) { + DoMethodsStackChunkFrameClosure closure(cl); + chunk->iterate_stack(&closure); +} + +class OopIterateStackChunkFrameClosure { + OopIterateClosure* const _closure; + MemRegion _bound; + const bool _do_metadata; + +public: + OopIterateStackChunkFrameClosure(OopIterateClosure* closure, MemRegion mr) + : _closure(closure), + _bound(mr), + _do_metadata(_closure->do_metadata()) { + assert(_closure != nullptr, "must be set"); + } + + template + bool do_frame(const StackChunkFrameStream& f, const RegisterMapT* map) { + if (_do_metadata) { + DoMethodsStackChunkFrameClosure(_closure).do_frame(f, map); + } + + StackChunkOopIterateFilterClosure cl(_closure, _bound); + f.iterate_oops(&cl, map); + + return true; + } +}; + +void InstanceStackChunkKlass::oop_oop_iterate_stack_slow(stackChunkOop chunk, OopIterateClosure* closure, MemRegion mr) { + if (UseZGC || UseShenandoahGC) { + // An OopClosure could apply barriers to a stack chunk. The side effects + // of the load barriers could destroy derived pointers, which must be + // processed before their base oop is processed. So we force processing + // of derived pointers before applying the closures. + chunk->relativize_derived_pointers_concurrently(); + } + OopIterateStackChunkFrameClosure frame_closure(closure, mr); + chunk->iterate_stack(&frame_closure); +} + +#ifdef ASSERT + +class DescribeStackChunkClosure { + stackChunkOop _chunk; + FrameValues _values; + RegisterMap _map; + int _frame_no; + +public: + DescribeStackChunkClosure(stackChunkOop chunk) + : _chunk(chunk), _map((JavaThread*)nullptr, true, false, true), _frame_no(0) { + _map.set_include_argument_oops(false); + } + + const RegisterMap* get_map(const RegisterMap* map, intptr_t* sp) { return map; } + const RegisterMap* get_map(const SmallRegisterMap* map, intptr_t* sp) { return map->copy_to_RegisterMap(&_map, sp); } + + template + bool do_frame(const StackChunkFrameStream& f, const RegisterMapT* map) { + ResetNoHandleMark rnhm; + HandleMark hm(Thread::current()); + + frame fr = f.to_frame(); + fr.describe(_values, _frame_no++, get_map(map, f.sp())); + return true; + } + + void describe_chunk() { + // _values.describe(-1, _chunk->start_address(), "CHUNK START"); + _values.describe(-1, _chunk->sp_address(), "CHUNK SP"); + _values.describe(-1, _chunk->bottom_address() - 1, "CHUNK ARGS"); + _values.describe(-1, _chunk->end_address() - 1, "CHUNK END"); + } + + void print_on(outputStream* out) { + if (_frame_no > 0) { + describe_chunk(); + _values.print_on(_chunk, out); + } else { + out->print_cr(" EMPTY"); + } + } +}; +#endif + +class PrintStackChunkClosure { + outputStream* _st; + +public: + PrintStackChunkClosure(outputStream* st) : _st(st) {} + + template + bool do_frame(const StackChunkFrameStream& fs, const RegisterMapT* map) { + frame f = fs.to_frame(); + _st->print_cr("-- frame sp: " INTPTR_FORMAT " interpreted: %d size: %d argsize: %d", + p2i(fs.sp()), fs.is_interpreted(), f.frame_size(), + fs.is_interpreted() ? 0 : f.compiled_frame_stack_argsize()); + #ifdef ASSERT + f.print_value_on(_st, nullptr); + #else + f.print_on(_st); + #endif + const ImmutableOopMap* oopmap = fs.oopmap(); + if (oopmap != nullptr) { + oopmap->print_on(_st); + _st->cr(); + } + return true; + } +}; + +void InstanceStackChunkKlass::print_chunk(const stackChunkOop c, bool verbose, outputStream* st) { + if (c == nullptr) { + st->print_cr("CHUNK NULL"); + return; + } + + st->print_cr("CHUNK " INTPTR_FORMAT " " INTPTR_FORMAT " - " INTPTR_FORMAT " :: " INTPTR_FORMAT, + p2i((oopDesc*)c), p2i(c->start_address()), p2i(c->end_address()), c->identity_hash()); + st->print_cr(" barriers: %d gc_mode: %d bitmap: %d parent: " INTPTR_FORMAT, + c->requires_barriers(), c->is_gc_mode(), c->has_bitmap(), p2i((oopDesc*)c->parent())); + st->print_cr(" flags mixed: %d", c->has_mixed_frames()); + st->print_cr(" size: %d argsize: %d max_size: %d sp: %d pc: " INTPTR_FORMAT, + c->stack_size(), c->argsize(), c->max_thawing_size(), c->sp(), p2i(c->pc())); + + if (verbose) { + st->cr(); + st->print_cr("------ chunk frames end: " INTPTR_FORMAT, p2i(c->bottom_address())); + PrintStackChunkClosure closure(st); + c->iterate_stack(&closure); + st->print_cr("------"); + + #ifdef ASSERT + ResourceMark rm; + DescribeStackChunkClosure describe(c); + c->iterate_stack(&describe); + describe.print_on(st); + st->print_cr("======"); + #endif + } +} + +void InstanceStackChunkKlass::init_offset_of_stack() { + // Cache the offset of the static fields in the Class instance + assert(_offset_of_stack == 0, "once"); + _offset_of_stack = cast(vmClasses::StackChunk_klass())->size_helper() << LogHeapWordSize; +} diff --git a/src/hotspot/share/oops/instanceStackChunkKlass.hpp b/src/hotspot/share/oops/instanceStackChunkKlass.hpp new file mode 100644 index 00000000000..7c069f4bafb --- /dev/null +++ b/src/hotspot/share/oops/instanceStackChunkKlass.hpp @@ -0,0 +1,178 @@ +/* + * Copyright (c) 2020, 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_OOPS_INSTANCESTACKCHUNKKLASS_HPP +#define SHARE_OOPS_INSTANCESTACKCHUNKKLASS_HPP + +#include "memory/memRegion.hpp" +#include "oops/instanceKlass.hpp" +#include "oops/oopsHierarchy.hpp" +#include "utilities/macros.hpp" +#include "utilities/ostream.hpp" + +class ClassFileParser; + +// An InstanceStackChunkKlass is a specialization of the InstanceKlass. +// +// The stackChunkOops have a header containing metadata, and a blob containing a +// stack segment (some integral number of stack frames). +// +// A chunk is said to be "mixed" if it contains interpreter frames or stubs +// (which can only be a safepoint stub as the topmost frame). Otherwise, it +// must contain only compiled Java frames. +// +// Interpreter frames in chunks have their internal pointers converted to +// relative offsets from fp. Derived pointers in compiled frames might also +// be converted to relative offsets from their base. + +/************************************************ + +Chunk layout: + + +-------------------+ + | | + | oop bitmap | + | | + | ----------------- | + | | + | [empty] | + | | + -|===================| + / | | + | | caller stack args | argsize + | | | words + | | ----------------- | + | | | + ^ | | frame | + | | | | + | size | ----------------- | + | words | | + | | | frame | + | | | | + Address | | | ----------------- | + | | | | + | | | frame | + | | | | + | | | callee stack args | + | | | ----------------- |<--\ + | | | pc | | + | | | rbp | | + | | | | | + | | | [empty] | | + | \ | | | + - |===================| | + | int maxSize | | + | long pc | | + header | byte flags | | + | int argsize | | + | int sp +---/ + | int size | + +-------------------+ + +************************************************/ + + +class InstanceStackChunkKlass: public InstanceKlass { + friend class VMStructs; + friend class InstanceKlass; + friend class Continuations; + +public: + static const KlassKind Kind = InstanceStackChunkKlassKind; + +private: + static int _offset_of_stack; + + InstanceStackChunkKlass(const ClassFileParser& parser); + +public: + InstanceStackChunkKlass() { assert(DumpSharedSpaces || UseSharedSpaces, "only for CDS"); } + + // Casting from Klass* + static InstanceStackChunkKlass* cast(Klass* k) { + assert(k->is_stack_chunk_instance_klass(), "cast to InstanceStackChunkKlass"); + return static_cast(k); + } + + inline size_t instance_size(size_t stack_size_in_words) const; + + static inline size_t bitmap_size_in_bits(size_t stack_size_in_words); // In bits + static inline size_t bitmap_size(size_t stack_size_in_words); // In words + + // Returns the size of the instance including the stack data. + virtual size_t oop_size(oop obj) const override; + + static void serialize_offsets(class SerializeClosure* f) NOT_CDS_RETURN; + + static void print_chunk(const stackChunkOop chunk, bool verbose, outputStream* st = tty); + +#ifndef PRODUCT + void oop_print_on(oop obj, outputStream* st) override; +#endif + + // Stack offset is an offset into the Heap + static int offset_of_stack() { return _offset_of_stack; } + static void init_offset_of_stack(); + + // Oop fields (and metadata) iterators + // + // The InstanceClassLoaderKlass iterators also visit the CLD pointer (or mirror of anonymous klasses). + + // Forward iteration + // Iterate over the oop fields and metadata. + template + inline void oop_oop_iterate(oop obj, OopClosureType* closure); + + // Reverse iteration + // Iterate over the oop fields and metadata. + template + inline void oop_oop_iterate_reverse(oop obj, OopClosureType* closure); + + // Bounded range iteration + // Iterate over the oop fields and metadata. + template + inline void oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr); + +private: + template + inline void oop_oop_iterate_header(stackChunkOop chunk, OopClosureType* closure); + + template + inline void oop_oop_iterate_header_bounded(stackChunkOop chunk, OopClosureType* closure, MemRegion mr); + + template + inline void oop_oop_iterate_stack(stackChunkOop chunk, OopClosureType* closure); + + template + inline void oop_oop_iterate_stack_bounded(stackChunkOop chunk, OopClosureType* closure, MemRegion mr); + + template + inline void oop_oop_iterate_stack_with_bitmap(stackChunkOop chunk, OopClosureType* closure, intptr_t* start, intptr_t* end); + + void do_methods(stackChunkOop chunk, OopIterateClosure* cl); + + void oop_oop_iterate_stack_slow(stackChunkOop chunk, OopIterateClosure* closure, MemRegion mr); +}; + +#endif // SHARE_OOPS_INSTANCESTACKCHUNKKLASS_HPP diff --git a/src/hotspot/share/oops/instanceStackChunkKlass.inline.hpp b/src/hotspot/share/oops/instanceStackChunkKlass.inline.hpp new file mode 100644 index 00000000000..fd6619f3f79 --- /dev/null +++ b/src/hotspot/share/oops/instanceStackChunkKlass.inline.hpp @@ -0,0 +1,172 @@ +/* Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_OOPS_INSTANCESTACKCHUNKKLASS_INLINE_HPP +#define SHARE_OOPS_INSTANCESTACKCHUNKKLASS_INLINE_HPP + +#include "oops/instanceStackChunkKlass.hpp" + +#include "classfile/javaClasses.inline.hpp" +#include "code/codeBlob.inline.hpp" +#include "code/codeCache.inline.hpp" +#include "code/nativeInst.hpp" +#include "compiler/oopMap.hpp" +#include "gc/shared/barrierSetNMethod.hpp" +#include "gc/shared/collectedHeap.hpp" +#include "gc/shared/gc_globals.hpp" +#include "logging/log.hpp" +#include "memory/iterator.inline.hpp" +#include "oops/instanceKlass.inline.hpp" +#include "oops/klass.hpp" +#include "oops/oop.inline.hpp" +#include "oops/stackChunkOop.inline.hpp" +#include "runtime/frame.inline.hpp" +#include "runtime/globals.hpp" +#include "runtime/handles.inline.hpp" +#include "runtime/smallRegisterMap.inline.hpp" +#include "runtime/stackChunkFrameStream.inline.hpp" +#include "utilities/bitMap.inline.hpp" +#include "utilities/debug.hpp" +#include "utilities/globalDefinitions.hpp" +#include "utilities/macros.hpp" + +inline size_t InstanceStackChunkKlass::instance_size(size_t stack_size_in_words) const { + return align_object_size(size_helper() + stack_size_in_words + bitmap_size(stack_size_in_words)); +} + +inline size_t InstanceStackChunkKlass::bitmap_size_in_bits(size_t stack_size_in_words) { + // Need one bit per potential narrowOop* or oop* address. + size_t size_in_bits = stack_size_in_words << (LogBitsPerWord - LogBitsPerHeapOop); + + return align_up(size_in_bits, BitsPerWord); +} + +inline size_t InstanceStackChunkKlass::bitmap_size(size_t stack_size_in_words) { + return bitmap_size_in_bits(stack_size_in_words) >> LogBitsPerWord; +} + +template +void InstanceStackChunkKlass::oop_oop_iterate(oop obj, OopClosureType* closure) { + stackChunkOop chunk = stackChunkOopDesc::cast(obj); + if (Devirtualizer::do_metadata(closure)) { + Devirtualizer::do_klass(closure, this); + } + oop_oop_iterate_stack(chunk, closure); + oop_oop_iterate_header(chunk, closure); +} + +template +void InstanceStackChunkKlass::oop_oop_iterate_reverse(oop obj, OopClosureType* closure) { + assert(!Devirtualizer::do_metadata(closure), "Code to handle metadata is not implemented"); + stackChunkOop chunk = stackChunkOopDesc::cast(obj); + oop_oop_iterate_stack(chunk, closure); + oop_oop_iterate_header(chunk, closure); +} + +template +void InstanceStackChunkKlass::oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr) { + stackChunkOop chunk = stackChunkOopDesc::cast(obj); + if (Devirtualizer::do_metadata(closure)) { + if (mr.contains(obj)) { + Devirtualizer::do_klass(closure, this); + } + } + oop_oop_iterate_stack_bounded(chunk, closure, mr); + oop_oop_iterate_header_bounded(chunk, closure, mr); +} + +template +void InstanceStackChunkKlass::oop_oop_iterate_header(stackChunkOop chunk, OopClosureType* closure) { + T* parent_addr = chunk->field_addr(jdk_internal_vm_StackChunk::parent_offset()); + T* cont_addr = chunk->field_addr(jdk_internal_vm_StackChunk::cont_offset()); + Devirtualizer::do_oop(closure, parent_addr); + Devirtualizer::do_oop(closure, cont_addr); +} + +template +void InstanceStackChunkKlass::oop_oop_iterate_header_bounded(stackChunkOop chunk, OopClosureType* closure, MemRegion mr) { + T* parent_addr = chunk->field_addr(jdk_internal_vm_StackChunk::parent_offset()); + T* cont_addr = chunk->field_addr(jdk_internal_vm_StackChunk::cont_offset()); + if (mr.contains(parent_addr)) { + Devirtualizer::do_oop(closure, parent_addr); + } + if (mr.contains(cont_addr)) { + Devirtualizer::do_oop(closure, cont_addr); + } +} + +template +void InstanceStackChunkKlass::oop_oop_iterate_stack_bounded(stackChunkOop chunk, OopClosureType* closure, MemRegion mr) { + if (chunk->has_bitmap()) { + intptr_t* start = chunk->sp_address() - frame::metadata_words; + intptr_t* end = chunk->end_address(); + // mr.end() can actually be less than start. In that case, we only walk the metadata + if ((intptr_t*)mr.start() > start) { + start = (intptr_t*)mr.start(); + } + if ((intptr_t*)mr.end() < end) { + end = (intptr_t*)mr.end(); + } + oop_oop_iterate_stack_with_bitmap(chunk, closure, start, end); + } else { + oop_oop_iterate_stack_slow(chunk, closure, mr); + } +} + +template +void InstanceStackChunkKlass::oop_oop_iterate_stack(stackChunkOop chunk, OopClosureType* closure) { + if (chunk->has_bitmap()) { + oop_oop_iterate_stack_with_bitmap(chunk, closure, chunk->sp_address() - frame::metadata_words, chunk->end_address()); + } else { + oop_oop_iterate_stack_slow(chunk, closure, chunk->range()); + } +} + +template +class StackChunkOopIterateBitmapClosure { + stackChunkOop _chunk; + OopClosureType* const _closure; + +public: + StackChunkOopIterateBitmapClosure(stackChunkOop chunk, OopClosureType* closure) : _chunk(chunk), _closure(closure) {} + + bool do_bit(BitMap::idx_t index) { + Devirtualizer::do_oop(_closure, _chunk->address_for_bit(index)); + return true; + } +}; + +template +void InstanceStackChunkKlass::oop_oop_iterate_stack_with_bitmap(stackChunkOop chunk, OopClosureType* closure, + intptr_t* start, intptr_t* end) { + if (Devirtualizer::do_metadata(closure)) { + do_methods(chunk, closure); + } + + if (end > start) { + StackChunkOopIterateBitmapClosure bitmap_closure(chunk, closure); + chunk->bitmap().iterate(&bitmap_closure, chunk->bit_index_for((T*)start), chunk->bit_index_for((T*)end)); + } +} + +#endif // SHARE_OOPS_INSTANCESTACKCHUNKKLASS_INLINE_HPP diff --git a/src/hotspot/share/oops/klass.hpp b/src/hotspot/share/oops/klass.hpp index 3d1d98c7037..38eb99b5f63 100644 --- a/src/hotspot/share/oops/klass.hpp +++ b/src/hotspot/share/oops/klass.hpp @@ -43,6 +43,7 @@ enum KlassKind { InstanceRefKlassKind, InstanceMirrorKlassKind, InstanceClassLoaderKlassKind, + InstanceStackChunkKlassKind, TypeArrayKlassKind, ObjArrayKlassKind }; @@ -617,13 +618,14 @@ protected: public: #endif - bool is_instance_klass() const { return assert_same_query(_kind <= InstanceClassLoaderKlassKind, is_instance_klass_slow()); } + bool is_instance_klass() const { return assert_same_query(_kind <= InstanceStackChunkKlassKind, is_instance_klass_slow()); } // Other is anything that is not one of the more specialized kinds of InstanceKlass. bool is_other_instance_klass() const { return _kind == InstanceKlassKind; } bool is_reference_instance_klass() const { return _kind == InstanceRefKlassKind; } bool is_mirror_instance_klass() const { return _kind == InstanceMirrorKlassKind; } bool is_class_loader_instance_klass() const { return _kind == InstanceClassLoaderKlassKind; } bool is_array_klass() const { return assert_same_query( _kind >= TypeArrayKlassKind, is_array_klass_slow()); } + bool is_stack_chunk_instance_klass() const { return _kind == InstanceStackChunkKlassKind; } bool is_objArray_klass() const { return assert_same_query( _kind == ObjArrayKlassKind, is_objArray_klass_slow()); } bool is_typeArray_klass() const { return assert_same_query( _kind == TypeArrayKlassKind, is_typeArray_klass_slow()); } #undef assert_same_query diff --git a/src/hotspot/share/oops/method.cpp b/src/hotspot/share/oops/method.cpp index b3f45753fb1..36be680fb5b 100644 --- a/src/hotspot/share/oops/method.cpp +++ b/src/hotspot/share/oops/method.cpp @@ -104,6 +104,7 @@ Method::Method(ConstMethod* xconst, AccessFlags access_flags, Symbol* name) { set_force_inline(false); set_hidden(false); set_dont_inline(false); + set_changes_current_thread(false); set_has_injected_profile(false); set_method_data(NULL); clear_method_counters(); @@ -662,6 +663,7 @@ void Method::compute_from_signature(Symbol* sig) { // we might as well compute the whole fingerprint. Fingerprinter fp(sig, is_static()); set_size_of_parameters(fp.size_of_parameters()); + set_num_stack_arg_slots(fp.num_stack_arg_slots()); constMethod()->set_result_type(fp.return_type()); constMethod()->set_fingerprint(fp.fingerprint()); } @@ -987,7 +989,7 @@ bool Method::is_klass_loaded(int refinfo_index, bool must_be_resolved) const { void Method::set_native_function(address function, bool post_event_flag) { assert(function != NULL, "use clear_native_function to unregister natives"); - assert(!is_method_handle_intrinsic() || function == SharedRuntime::native_method_throw_unsatisfied_link_error_entry(), ""); + assert(!is_special_native_intrinsic() || function == SharedRuntime::native_method_throw_unsatisfied_link_error_entry(), ""); address* native_function = native_function_addr(); // We can see racers trying to place the same native function into place. Once @@ -1017,7 +1019,7 @@ void Method::set_native_function(address function, bool post_event_flag) { bool Method::has_native_function() const { - if (is_method_handle_intrinsic()) + if (is_special_native_intrinsic()) return false; // special-cased in SharedRuntime::generate_native_wrapper address func = native_function(); return (func != NULL && func != SharedRuntime::native_method_throw_unsatisfied_link_error_entry()); @@ -1074,7 +1076,7 @@ void Method::print_made_not_compilable(int comp_level, bool is_osr, bool report, bool Method::is_always_compilable() const { // Generated adapters must be compiled - if (is_method_handle_intrinsic() && is_synthetic()) { + if (is_special_native_intrinsic() && is_synthetic()) { assert(!is_not_c1_compilable(), "sanity check"); assert(!is_not_c2_compilable(), "sanity check"); return true; @@ -1318,6 +1320,11 @@ void Method::set_code(const methodHandle& mh, CompiledMethod *code) { // Instantly compiled code can execute. if (!mh->is_method_handle_intrinsic()) mh->_from_interpreted_entry = mh->get_i2c_entry(); + if (mh->is_continuation_enter_intrinsic()) { + // this is the entry used when we're in interpreter-only mode; see InterpreterMacroAssembler::jump_from_interpreted + mh->_i2i_entry = mh->get_i2c_entry(); + mh->_from_interpreted_entry = mh->get_i2c_entry(); + } } @@ -1725,7 +1732,7 @@ bool Method::has_unloaded_classes_in_signature(const methodHandle& m, TRAPS) { } // Exposed so field engineers can debug VM -void Method::print_short_name(outputStream* st) { +void Method::print_short_name(outputStream* st) const { ResourceMark rm; #ifdef PRODUCT st->print(" %s::", method_holder()->external_name()); @@ -1793,7 +1800,7 @@ class SignatureTypePrinter : public SignatureTypeNames { }; -void Method::print_name(outputStream* st) { +void Method::print_name(outputStream* st) const { Thread *thread = Thread::current(); ResourceMark rm(thread); st->print("%s ", is_static() ? "static" : "virtual"); @@ -2278,6 +2285,13 @@ void Method::set_on_stack(const bool value) { } } +void Method::record_gc_epoch() { + // If any method is on the stack in continuations, none of them can be reclaimed, + // so save the marking cycle to check for the whole class in the cpCache. + // The cpCache is writeable. + constants()->cache()->record_gc_epoch(); +} + // Called when the class loader is unloaded to make all methods weak. void Method::clear_jmethod_ids(ClassLoaderData* loader_data) { loader_data->jmethod_ids()->clear_all_methods(); diff --git a/src/hotspot/share/oops/method.hpp b/src/hotspot/share/oops/method.hpp index 15d53a0cf4b..758888bf2fe 100644 --- a/src/hotspot/share/oops/method.hpp +++ b/src/hotspot/share/oops/method.hpp @@ -25,6 +25,7 @@ #ifndef SHARE_OOPS_METHOD_HPP #define SHARE_OOPS_METHOD_HPP +#include "classfile/vmSymbols.hpp" #include "code/compressedStream.hpp" #include "compiler/compilerDefinitions.hpp" #include "interpreter/invocationCounter.hpp" @@ -84,14 +85,16 @@ class Method : public Metadata { // Flags enum Flags { - _caller_sensitive = 1 << 0, - _force_inline = 1 << 1, - _dont_inline = 1 << 2, - _hidden = 1 << 3, - _has_injected_profile = 1 << 4, - _intrinsic_candidate = 1 << 5, - _reserved_stack_access = 1 << 6, - _scoped = 1 << 7 + _caller_sensitive = 1 << 0, + _force_inline = 1 << 1, + _dont_inline = 1 << 2, + _hidden = 1 << 3, + _has_injected_profile = 1 << 4, + _intrinsic_candidate = 1 << 5, + _reserved_stack_access = 1 << 6, + _scoped = 1 << 7, + _changes_current_thread = 1 << 8, + _jvmti_mount_transition = 1 << 9, }; mutable u2 _flags; @@ -469,6 +472,9 @@ public: // clear entry points. Used by sharing code during dump time void unlink_method() NOT_CDS_RETURN; + // the number of argument reg slots that the compiled method uses on the stack. + int num_stack_arg_slots() const { return constMethod()->num_stack_arg_slots(); } + virtual void metaspace_pointers_do(MetaspaceClosure* iter); virtual MetaspaceObj::Type type() const { return MethodType; } @@ -727,6 +733,13 @@ public: static methodHandle make_method_handle_intrinsic(vmIntrinsicID iid, // _invokeBasic, _linkToVirtual Symbol* signature, //anything at all TRAPS); + + + // Continuation + bool is_continuation_enter_intrinsic() const { return intrinsic_id() == vmIntrinsics::_Continuation_enterSpecial; } + + bool is_special_native_intrinsic() const { return is_method_handle_intrinsic() || is_continuation_enter_intrinsic(); } + static Klass* check_non_bcp_klass(Klass* klass); enum { @@ -751,6 +764,8 @@ public: bool on_stack() const { return access_flags().on_stack(); } void set_on_stack(const bool value); + void record_gc_epoch(); + // see the definition in Method*.cpp for the gory details bool should_not_be_cached() const; @@ -835,6 +850,20 @@ public: _flags = x ? (_flags | _dont_inline) : (_flags & ~_dont_inline); } + bool changes_current_thread() { + return (_flags & _changes_current_thread) != 0; + } + void set_changes_current_thread(bool x) { + _flags = x ? (_flags | _changes_current_thread) : (_flags & ~_changes_current_thread); + } + + bool jvmti_mount_transition() { + return (_flags & _jvmti_mount_transition) != 0; + } + void set_jvmti_mount_transition(bool x) { + _flags = x ? (_flags | _jvmti_mount_transition) : (_flags & ~_jvmti_mount_transition); + } + bool is_hidden() const { return (_flags & _hidden) != 0; } @@ -949,11 +978,11 @@ public: static bool has_unloaded_classes_in_signature(const methodHandle& m, TRAPS); // Printing - void print_short_name(outputStream* st = tty); // prints as klassname::methodname; Exposed so field engineers can debug VM + void print_short_name(outputStream* st = tty) const; // prints as klassname::methodname; Exposed so field engineers can debug VM #if INCLUDE_JVMTI - void print_name(outputStream* st = tty); // prints as "virtual void foo(int)"; exposed for -Xlog:redefine+class + void print_name(outputStream* st = tty) const; // prints as "virtual void foo(int)"; exposed for -Xlog:redefine+class #else - void print_name(outputStream* st = tty) PRODUCT_RETURN; // prints as "virtual void foo(int)" + void print_name(outputStream* st = tty) const PRODUCT_RETURN; // prints as "virtual void foo(int)" #endif typedef int (*method_comparator_func)(Method* a, Method* b); @@ -997,6 +1026,8 @@ public: // Inlined elements address* native_function_addr() const { assert(is_native(), "must be native"); return (address*) (this+1); } address* signature_handler_addr() const { return native_function_addr() + 1; } + + void set_num_stack_arg_slots(int n) { constMethod()->set_num_stack_arg_slots(n); } }; diff --git a/src/hotspot/share/oops/objArrayOop.hpp b/src/hotspot/share/oops/objArrayOop.hpp index e1a6226f88a..e93f4e82ab3 100644 --- a/src/hotspot/share/oops/objArrayOop.hpp +++ b/src/hotspot/share/oops/objArrayOop.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -39,6 +39,9 @@ class objArrayOopDesc : public arrayOopDesc { friend class Runtime1; friend class psPromotionManager; friend class CSetMarkWordClosure; + friend class Continuation; + template + friend class RawOopWriter; template T* obj_at_addr(int index) const; diff --git a/src/hotspot/share/oops/oop.cpp b/src/hotspot/share/oops/oop.cpp index 17e96450cf2..50205032ddd 100644 --- a/src/hotspot/share/oops/oop.cpp +++ b/src/hotspot/share/oops/oop.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -35,11 +35,16 @@ #include "oops/verifyOopClosure.hpp" #include "runtime/handles.inline.hpp" #include "runtime/thread.inline.hpp" -#include "utilities/copy.hpp" #include "utilities/macros.hpp" void oopDesc::print_on(outputStream* st) const { - klass()->oop_print_on(const_cast(this), st); + if (*((juint*)this) == badHeapWordVal) { + st->print("BAD WORD"); + } else if (*((juint*)this) == badMetaWordVal) { + st->print("BAD META WORD"); + } else { + klass()->oop_print_on(cast_to_oop(this), st); + } } void oopDesc::print_address_on(outputStream* st) const { @@ -136,6 +141,7 @@ void VerifyOopClosure::do_oop(narrowOop* p) { VerifyOopClosure::do_oop_work(p); // type test operations that doesn't require inclusion of oop.inline.hpp. bool oopDesc::is_instance_noinline() const { return is_instance(); } bool oopDesc::is_instanceRef_noinline() const { return is_instanceRef(); } +bool oopDesc::is_stackChunk_noinline() const { return is_stackChunk(); } bool oopDesc::is_array_noinline() const { return is_array(); } bool oopDesc::is_objArray_noinline() const { return is_objArray(); } bool oopDesc::is_typeArray_noinline() const { return is_typeArray(); } @@ -224,6 +230,12 @@ void oopDesc::verify_forwardee(oop forwardee) { #endif } -bool oopDesc::get_UseParallelGC() { return UseParallelGC; } -bool oopDesc::get_UseG1GC() { return UseG1GC; } +bool oopDesc::size_might_change() { + // UseParallelGC and UseG1GC can change the length field + // of an "old copy" of an object array in the young gen so it indicates + // the grey portion of an already copied array. This will cause the first + // disjunct below to fail if the two comparands are computed across such + // a concurrent change. + return Universe::heap()->is_gc_active() && is_objArray() && is_forwarded() && (UseParallelGC || UseG1GC); +} #endif diff --git a/src/hotspot/share/oops/oop.hpp b/src/hotspot/share/oops/oop.hpp index 08f354a905d..41afbe43d43 100644 --- a/src/hotspot/share/oops/oop.hpp +++ b/src/hotspot/share/oops/oop.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -99,7 +99,7 @@ class oopDesc { // Returns whether this is an instance of k or an instance of a subclass of k inline bool is_a(Klass* k) const; - // Returns the actual oop size of the object + // Returns the actual oop size of the object in machine words inline size_t size(); // Sometimes (for complicated concurrency-related reasons), it is useful @@ -109,6 +109,7 @@ class oopDesc { // type test operations (inlined in oop.inline.hpp) inline bool is_instance() const; inline bool is_instanceRef() const; + inline bool is_stackChunk() const; inline bool is_array() const; inline bool is_objArray() const; inline bool is_typeArray() const; @@ -116,6 +117,7 @@ class oopDesc { // type test operations that don't require inclusion of oop.inline.hpp. bool is_instance_noinline() const; bool is_instanceRef_noinline() const; + bool is_stackChunk_noinline() const; bool is_array_noinline() const; bool is_objArray_noinline() const; bool is_typeArray_noinline() const; @@ -312,9 +314,7 @@ class oopDesc { static void* load_klass_raw(oop obj); static void* load_oop_raw(oop obj, int offset); - // Avoid include gc_globals.hpp in oop.inline.hpp - DEBUG_ONLY(bool get_UseParallelGC();) - DEBUG_ONLY(bool get_UseG1GC();) + DEBUG_ONLY(bool size_might_change();) }; // An oopDesc is not initialized via a constructor. Space is allocated in diff --git a/src/hotspot/share/oops/oop.inline.hpp b/src/hotspot/share/oops/oop.inline.hpp index 9491d71e2ed..b374f2830bc 100644 --- a/src/hotspot/share/oops/oop.inline.hpp +++ b/src/hotspot/share/oops/oop.inline.hpp @@ -32,6 +32,7 @@ #include "oops/arrayKlass.hpp" #include "oops/arrayOop.hpp" #include "oops/compressedOops.inline.hpp" +#include "oops/instanceKlass.hpp" #include "oops/markWord.hpp" #include "oops/oopsHierarchy.hpp" #include "runtime/atomic.hpp" @@ -176,14 +177,7 @@ size_t oopDesc::size_given_klass(Klass* klass) { // skipping the intermediate round to HeapWordSize. s = align_up(size_in_bytes, MinObjAlignmentInBytes) / HeapWordSize; - // UseParallelGC and UseG1GC can change the length field - // of an "old copy" of an object array in the young gen so it indicates - // the grey portion of an already copied array. This will cause the first - // disjunct below to fail if the two comparands are computed across such - // a concurrent change. - assert((s == klass->oop_size(this)) || - (Universe::is_gc_active() && is_objArray() && is_forwarded() && (get_UseParallelGC() || get_UseG1GC())), - "wrong array object size"); + assert(s == klass->oop_size(this) || size_might_change(), "wrong array object size"); } else { // Must be zero, so bite the bullet and take the virtual call. s = klass->oop_size(this); @@ -195,11 +189,12 @@ size_t oopDesc::size_given_klass(Klass* klass) { return s; } -bool oopDesc::is_instance() const { return klass()->is_instance_klass(); } -bool oopDesc::is_instanceRef() const { return klass()->is_reference_instance_klass(); } -bool oopDesc::is_array() const { return klass()->is_array_klass(); } -bool oopDesc::is_objArray() const { return klass()->is_objArray_klass(); } -bool oopDesc::is_typeArray() const { return klass()->is_typeArray_klass(); } +bool oopDesc::is_instance() const { return klass()->is_instance_klass(); } +bool oopDesc::is_instanceRef() const { return klass()->is_reference_instance_klass(); } +bool oopDesc::is_stackChunk() const { return klass()->is_stack_chunk_instance_klass(); } +bool oopDesc::is_array() const { return klass()->is_array_klass(); } +bool oopDesc::is_objArray() const { return klass()->is_objArray_klass(); } +bool oopDesc::is_typeArray() const { return klass()->is_typeArray_klass(); } template T* oopDesc::field_addr(int offset) const { return reinterpret_cast(cast_from_oop(as_oop()) + offset); } diff --git a/src/hotspot/share/oops/oopsHierarchy.hpp b/src/hotspot/share/oops/oopsHierarchy.hpp index a7f8d5ba653..d2a82b9ca1b 100644 --- a/src/hotspot/share/oops/oopsHierarchy.hpp +++ b/src/hotspot/share/oops/oopsHierarchy.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -45,6 +45,7 @@ typedef void* OopOrNarrowOopStar; typedef class oopDesc* oop; typedef class instanceOopDesc* instanceOop; +typedef class stackChunkOopDesc* stackChunkOop; typedef class arrayOopDesc* arrayOop; typedef class objArrayOopDesc* objArrayOop; typedef class typeArrayOopDesc* typeArrayOop; @@ -143,6 +144,7 @@ struct PrimitiveConversions::Translate : public TrueType { }; DEF_OOP(instance); +DEF_OOP(stackChunk); DEF_OOP(array); DEF_OOP(objArray); DEF_OOP(typeArray); @@ -177,6 +179,7 @@ class InstanceKlass; class InstanceMirrorKlass; class InstanceClassLoaderKlass; class InstanceRefKlass; +class InstanceStackChunkKlass; class ArrayKlass; class ObjArrayKlass; class TypeArrayKlass; diff --git a/src/hotspot/share/oops/stackChunkOop.cpp b/src/hotspot/share/oops/stackChunkOop.cpp new file mode 100644 index 00000000000..a8ba9f84df5 --- /dev/null +++ b/src/hotspot/share/oops/stackChunkOop.cpp @@ -0,0 +1,652 @@ +/* + * Copyright (c) 2021, 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "code/compiledMethod.hpp" +#include "code/scopeDesc.hpp" +#include "logging/log.hpp" +#include "logging/logStream.hpp" +#include "memory/memRegion.hpp" +#include "oops/instanceStackChunkKlass.inline.hpp" +#include "oops/oop.inline.hpp" +#include "oops/stackChunkOop.inline.hpp" +#include "runtime/frame.hpp" +#include "runtime/registerMap.hpp" +#include "runtime/smallRegisterMap.inline.hpp" +#include "runtime/stackChunkFrameStream.inline.hpp" + +frame stackChunkOopDesc::top_frame(RegisterMap* map) { + assert(!is_empty(), ""); + StackChunkFrameStream fs(this); + + map->set_stack_chunk(this); + fs.initialize_register_map(map); + + frame f = fs.to_frame(); + + assert(to_offset(f.sp()) == sp(), "f.offset_sp(): %d sp(): %d async: %d", f.offset_sp(), sp(), map->is_async()); + relativize_frame(f); + f.set_frame_index(0); + return f; +} + +frame stackChunkOopDesc::sender(const frame& f, RegisterMap* map) { + assert(map->in_cont(), ""); + assert(!map->include_argument_oops(), ""); + assert(!f.is_empty(), ""); + assert(map->stack_chunk() == this, ""); + assert(!is_empty(), ""); + + int index = f.frame_index(); // we need to capture the index before calling derelativize, which destroys it + StackChunkFrameStream fs(this, derelativize(f)); + fs.next(map); + + if (!fs.is_done()) { + frame sender = fs.to_frame(); + assert(is_usable_in_chunk(sender.unextended_sp()), ""); + relativize_frame(sender); + + sender.set_frame_index(index+1); + return sender; + } + + if (parent() != nullptr) { + assert(!parent()->is_empty(), ""); + return parent()->top_frame(map); + } + + return Continuation::continuation_parent_frame(map); +} + +static int num_java_frames(CompiledMethod* cm, address pc) { + int count = 0; + for (ScopeDesc* scope = cm->scope_desc_at(pc); scope != nullptr; scope = scope->sender()) { + count++; + } + return count; +} + +static int num_java_frames(const StackChunkFrameStream& f) { + assert(f.is_interpreted() + || (f.cb() != nullptr && f.cb()->is_compiled() && f.cb()->as_compiled_method()->is_java_method()), ""); + return f.is_interpreted() ? 1 : num_java_frames(f.cb()->as_compiled_method(), f.orig_pc()); +} + +int stackChunkOopDesc::num_java_frames() const { + int n = 0; + for (StackChunkFrameStream f(const_cast(this)); !f.is_done(); + f.next(SmallRegisterMap::instance)) { + if (!f.is_stub()) { + n += ::num_java_frames(f); + } + } + return n; +} + +template +class DoBarriersStackClosure { + const stackChunkOop _chunk; + +public: + DoBarriersStackClosure(stackChunkOop chunk) : _chunk(chunk) {} + + template + bool do_frame(const StackChunkFrameStream& f, const RegisterMapT* map) { + _chunk->do_barriers0(f, map); + return true; + } +}; + +template +void stackChunkOopDesc::do_barriers() { + DoBarriersStackClosure closure(this); + iterate_stack(&closure); +} + +template void stackChunkOopDesc::do_barriers (); +template void stackChunkOopDesc::do_barriers(); + +class DerivedPointersSupport { +public: + static void relativize(oop* base_loc, derived_pointer* derived_loc) { + oop base = *base_loc; + if (base == nullptr) { + return; + } + assert(!UseCompressedOops || !CompressedOops::is_base(base), ""); + + // This is always a full derived pointer + uintptr_t derived_int_val = *(uintptr_t*)derived_loc; + + // Make the pointer an offset (relativize) and store it at the same location + uintptr_t offset = derived_int_val - cast_from_oop(base); + *(uintptr_t*)derived_loc = offset; + } + + static void derelativize(oop* base_loc, derived_pointer* derived_loc) { + oop base = *base_loc; + if (base == nullptr) { + return; + } + assert(!UseCompressedOops || !CompressedOops::is_base(base), ""); + + // All derived pointers should have been relativized into offsets + uintptr_t offset = *(uintptr_t*)derived_loc; + + // Restore the original derived pointer + *(uintptr_t*)derived_loc = cast_from_oop(base) + offset; + } + + struct RelativizeClosure : public DerivedOopClosure { + virtual void do_derived_oop(oop* base_loc, derived_pointer* derived_loc) override { + DerivedPointersSupport::relativize(base_loc, derived_loc); + } + }; + + struct DerelativizeClosure : public DerivedOopClosure { + virtual void do_derived_oop(oop* base_loc, derived_pointer* derived_loc) override { + DerivedPointersSupport::derelativize(base_loc, derived_loc); + } + }; +}; + +template +class FrameToDerivedPointerClosure { + DerivedPointerClosureType* _cl; + +public: + FrameToDerivedPointerClosure(DerivedPointerClosureType* cl) + : _cl(cl) {} + + template + bool do_frame(const StackChunkFrameStream& f, const RegisterMapT* map) { + f.iterate_derived_pointers(_cl, map); + return true; + } +}; + +bool stackChunkOopDesc::try_acquire_relativization() { + for (;;) { + // We use an acquiring load when reading the flags to ensure that if we leave this + // function thinking that relativization is finished, we know that if another thread + // did the relativization, we will still be able to observe the relativized derived + // pointers, which is important as subsequent modifications of derived pointers must + // happen after relativization. + uint8_t flags_before = flags_acquire(); + if ((flags_before & FLAG_GC_MODE) != 0) { + // Terminal state - relativization is ensured + return false; + } + + if ((flags_before & FLAG_CLAIM_RELATIVIZE) != 0) { + // Someone else has claimed relativization - wait for completion + MonitorLocker ml(ContinuationRelativize_lock, Mutex::_no_safepoint_check_flag); + uint8_t flags_under_lock = flags_acquire(); + if ((flags_under_lock & FLAG_GC_MODE) != 0) { + // Terminal state - relativization is ensured + return false; + } + + if ((flags_under_lock & FLAG_NOTIFY_RELATIVIZE) != 0) { + // Relativization is claimed by another thread, and it knows it needs to notify + ml.wait(); + } else if (try_set_flags(flags_under_lock, flags_under_lock | FLAG_NOTIFY_RELATIVIZE)) { + // Relativization is claimed by another thread, and it knows it needs to notify + ml.wait(); + } + // Retry - rerun the loop + continue; + } + + if (try_set_flags(flags_before, flags_before | FLAG_CLAIM_RELATIVIZE)) { + // Claimed relativization - let's do it + return true; + } + } +} + +void stackChunkOopDesc::release_relativization() { + for (;;) { + uint8_t flags_before = flags(); + if ((flags_before & FLAG_NOTIFY_RELATIVIZE) != 0) { + MonitorLocker ml(ContinuationRelativize_lock, Mutex::_no_safepoint_check_flag); + // No need to CAS the terminal state; nobody else can be racingly mutating here + // as both claim and notify flags are already set (and monotonic) + // We do however need to use a releasing store on the flags, to ensure that + // the reader of that value (using load_acquire) will be able to observe + // the relativization of the derived pointers + uint8_t flags_under_lock = flags(); + release_set_flags(flags_under_lock | FLAG_GC_MODE); + ml.notify_all(); + return; + } + + if (try_set_flags(flags_before, flags_before | FLAG_GC_MODE)) { + // Successfully set the terminal state; we are done + return; + } + } +} + +void stackChunkOopDesc::relativize_derived_pointers_concurrently() { + if (!try_acquire_relativization()) { + // Already relativized + return; + } + + DerivedPointersSupport::RelativizeClosure derived_cl; + FrameToDerivedPointerClosure frame_cl(&derived_cl); + iterate_stack(&frame_cl); + + release_relativization(); +} + +enum class OopKind { Narrow, Wide }; + +template +class CompressOopsAndBuildBitmapOopClosure : public OopClosure { + stackChunkOop _chunk; + BitMapView _bm; + + void convert_oop_to_narrowOop(oop* p) { + oop obj = *p; + *p = nullptr; + *(narrowOop*)p = CompressedOops::encode(obj); + } + + template + void do_oop_work(T* p) { + BitMap::idx_t index = _chunk->bit_index_for(p); + assert(!_bm.at(index), "must not be set already"); + _bm.set_bit(index); + } + +public: + CompressOopsAndBuildBitmapOopClosure(stackChunkOop chunk) + : _chunk(chunk), _bm(chunk->bitmap()) {} + + virtual void do_oop(oop* p) override { + if (kind == OopKind::Narrow) { + // Convert all oops to narrow before marking the oop in the bitmap. + convert_oop_to_narrowOop(p); + do_oop_work((narrowOop*)p); + } else { + do_oop_work(p); + } + } + + virtual void do_oop(narrowOop* p) override { + do_oop_work(p); + } +}; + +template +class TransformStackChunkClosure { + stackChunkOop _chunk; + +public: + TransformStackChunkClosure(stackChunkOop chunk) : _chunk(chunk) { } + + template + bool do_frame(const StackChunkFrameStream& f, const RegisterMapT* map) { + DerivedPointersSupport::RelativizeClosure derived_cl; + f.iterate_derived_pointers(&derived_cl, map); + + CompressOopsAndBuildBitmapOopClosure cl(_chunk); + f.iterate_oops(&cl, map); + + return true; + } +}; + +void stackChunkOopDesc::transform() { + assert(!is_gc_mode(), "Should only be called once per chunk"); + set_gc_mode(true); + + assert(!has_bitmap(), "Should only be set once"); + set_has_bitmap(true); + bitmap().clear(); + + if (UseCompressedOops) { + TransformStackChunkClosure closure(this); + iterate_stack(&closure); + } else { + TransformStackChunkClosure closure(this); + iterate_stack(&closure); + } +} + +template +class BarrierClosure: public OopClosure { + NOT_PRODUCT(intptr_t* _sp;) + +public: + BarrierClosure(intptr_t* sp) NOT_PRODUCT(: _sp(sp)) {} + + virtual void do_oop(oop* p) override { compressedOopsWithBitmap ? do_oop_work((narrowOop*)p) : do_oop_work(p); } + virtual void do_oop(narrowOop* p) override { do_oop_work(p); } + + template inline void do_oop_work(T* p) { + oop value = (oop)HeapAccess<>::oop_load(p); + if (barrier == stackChunkOopDesc::BarrierType::Store) { + HeapAccess<>::oop_store(p, value); + } + } +}; + +template +void stackChunkOopDesc::do_barriers0(const StackChunkFrameStream& f, const RegisterMapT* map) { + // We need to invoke the write barriers so as not to miss oops in old chunks that haven't yet been concurrently scanned + assert (!f.is_done(), ""); + + if (f.is_interpreted()) { + Method* m = f.to_frame().interpreter_frame_method(); + // Class redefinition support + m->record_gc_epoch(); + } else if (f.is_compiled()) { + nmethod* nm = f.cb()->as_nmethod(); + // The entry barrier takes care of having the right synchronization + // when keeping the nmethod alive during concurrent execution. + nm->run_nmethod_entry_barrier(); + // There is no need to mark the Method, as class redefinition will walk the + // CodeCache, noting their Methods + } + + if (has_bitmap() && UseCompressedOops) { + BarrierClosure oops_closure(f.sp()); + f.iterate_oops(&oops_closure, map); + } else { + BarrierClosure oops_closure(f.sp()); + f.iterate_oops(&oops_closure, map); + } +} + +template void stackChunkOopDesc::do_barriers0 (const StackChunkFrameStream& f, const RegisterMap* map); +template void stackChunkOopDesc::do_barriers0(const StackChunkFrameStream& f, const RegisterMap* map); +template void stackChunkOopDesc::do_barriers0 (const StackChunkFrameStream& f, const RegisterMap* map); +template void stackChunkOopDesc::do_barriers0(const StackChunkFrameStream& f, const RegisterMap* map); +template void stackChunkOopDesc::do_barriers0 (const StackChunkFrameStream& f, const SmallRegisterMap* map); +template void stackChunkOopDesc::do_barriers0(const StackChunkFrameStream& f, const SmallRegisterMap* map); +template void stackChunkOopDesc::do_barriers0 (const StackChunkFrameStream& f, const SmallRegisterMap* map); +template void stackChunkOopDesc::do_barriers0(const StackChunkFrameStream& f, const SmallRegisterMap* map); + +class UncompressOopsOopClosure : public OopClosure { +public: + void do_oop(oop* p) override { + assert(UseCompressedOops, "Only needed with compressed oops"); + oop obj = CompressedOops::decode(*(narrowOop*)p); + assert(obj == nullptr || dbg_is_good_oop(obj), "p: " INTPTR_FORMAT " obj: " INTPTR_FORMAT, p2i(p), p2i((oopDesc*)obj)); + *p = obj; + } + + void do_oop(narrowOop* p) override {} +}; + +template +void stackChunkOopDesc::fix_thawed_frame(const frame& f, const RegisterMapT* map) { + if (!(is_gc_mode() || requires_barriers())) { + return; + } + + if (has_bitmap() && UseCompressedOops) { + UncompressOopsOopClosure oop_closure; + if (f.is_interpreted_frame()) { + f.oops_interpreted_do(&oop_closure, nullptr); + } else { + OopMapDo visitor(&oop_closure, nullptr); + visitor.oops_do(&f, map, f.oop_map()); + } + } + + if (f.is_compiled_frame() && f.oop_map()->has_derived_oops()) { + DerivedPointersSupport::DerelativizeClosure derived_closure; + OopMapDo visitor(nullptr, &derived_closure); + visitor.oops_do(&f, map, f.oop_map()); + } +} + +template void stackChunkOopDesc::fix_thawed_frame(const frame& f, const RegisterMap* map); +template void stackChunkOopDesc::fix_thawed_frame(const frame& f, const SmallRegisterMap* map); + +void stackChunkOopDesc::print_on(bool verbose, outputStream* st) const { + if (*((juint*)this) == badHeapWordVal) { + st->print("BAD WORD"); + } else if (*((juint*)this) == badMetaWordVal) { + st->print("BAD META WORD"); + } else { + InstanceStackChunkKlass::print_chunk(const_cast(this), verbose, st); + } +} + +#ifdef ASSERT + +template +static inline oop safe_load(P* addr) { + oop obj = RawAccess<>::oop_load(addr); + return NativeAccess<>::oop_load(&obj); +} + +class StackChunkVerifyOopsClosure : public OopClosure { + stackChunkOop _chunk; + int _count; + +public: + StackChunkVerifyOopsClosure(stackChunkOop chunk) + : _chunk(chunk), _count(0) {} + + void do_oop(oop* p) override { (_chunk->has_bitmap() && UseCompressedOops) ? do_oop_work((narrowOop*)p) : do_oop_work(p); } + void do_oop(narrowOop* p) override { do_oop_work(p); } + + template inline void do_oop_work(T* p) { + _count++; + oop obj = safe_load(p); + assert(obj == nullptr || dbg_is_good_oop(obj), "p: " INTPTR_FORMAT " obj: " INTPTR_FORMAT, p2i(p), p2i((oopDesc*)obj)); + if (_chunk->has_bitmap()) { + BitMap::idx_t index = _chunk->bit_index_for(p); + assert(_chunk->bitmap().at(index), "Bit not set at index " SIZE_FORMAT " corresponding to " INTPTR_FORMAT, index, p2i(p)); + } + } + + int count() const { return _count; } +}; + +class VerifyStackChunkFrameClosure { + stackChunkOop _chunk; + +public: + intptr_t* _sp; + CodeBlob* _cb; + bool _callee_interpreted; + int _size; + int _argsize; + int _num_oops; + int _num_frames; + int _num_interpreted_frames; + int _num_i2c; + + VerifyStackChunkFrameClosure(stackChunkOop chunk, int num_frames, int size) + : _chunk(chunk), _sp(nullptr), _cb(nullptr), _callee_interpreted(false), + _size(size), _argsize(0), _num_oops(0), _num_frames(num_frames), _num_interpreted_frames(0), _num_i2c(0) {} + + template + bool do_frame(const StackChunkFrameStream& f, const RegisterMapT* map) { + _sp = f.sp(); + _cb = f.cb(); + + int fsize = f.frame_size() - ((f.is_interpreted() == _callee_interpreted) ? _argsize : 0); + int num_oops = f.num_oops(); + assert(num_oops >= 0, ""); + + _argsize = f.stack_argsize(); + _size += fsize; + _num_oops += num_oops; + if (f.is_interpreted()) { + _num_interpreted_frames++; + } + + log_develop_trace(continuations)("debug_verify_stack_chunk frame: %d sp: " INTPTR_FORMAT " pc: " INTPTR_FORMAT " interpreted: %d size: %d argsize: %d oops: %d", _num_frames, f.sp() - _chunk->start_address(), p2i(f.pc()), f.is_interpreted(), fsize, _argsize, num_oops); + LogTarget(Trace, continuations) lt; + if (lt.develop_is_enabled()) { + LogStream ls(lt); + f.print_value_on(&ls); + } + assert(f.pc() != nullptr, + "young: %d num_frames: %d sp: " INTPTR_FORMAT " start: " INTPTR_FORMAT " end: " INTPTR_FORMAT, + !_chunk->requires_barriers(), _num_frames, p2i(f.sp()), p2i(_chunk->start_address()), p2i(_chunk->bottom_address())); + + if (_num_frames == 0) { + assert(f.pc() == _chunk->pc(), ""); + } + + if (_num_frames > 0 && !_callee_interpreted && f.is_interpreted()) { + log_develop_trace(continuations)("debug_verify_stack_chunk i2c"); + _num_i2c++; + } + + StackChunkVerifyOopsClosure oops_closure(_chunk); + f.iterate_oops(&oops_closure, map); + assert(oops_closure.count() == num_oops, "oops: %d oopmap->num_oops(): %d", oops_closure.count(), num_oops); + + _callee_interpreted = f.is_interpreted(); + _num_frames++; + return true; + } +}; + +template +class StackChunkVerifyBitmapClosure : public BitMapClosure { + stackChunkOop _chunk; + +public: + int _count; + + StackChunkVerifyBitmapClosure(stackChunkOop chunk) : _chunk(chunk), _count(0) {} + + bool do_bit(BitMap::idx_t index) override { + T* p = _chunk->address_for_bit(index); + _count++; + + oop obj = safe_load(p); + assert(obj == nullptr || dbg_is_good_oop(obj), + "p: " INTPTR_FORMAT " obj: " INTPTR_FORMAT " index: " SIZE_FORMAT, + p2i(p), p2i((oopDesc*)obj), index); + + return true; // continue processing + } +}; + +bool stackChunkOopDesc::verify(size_t* out_size, int* out_oops, int* out_frames, int* out_interpreted_frames) { + DEBUG_ONLY(if (!VerifyContinuations) return true;) + + assert(oopDesc::is_oop(this), ""); + + assert(stack_size() >= 0, ""); + assert(argsize() >= 0, ""); + assert(!has_bitmap() || is_gc_mode(), ""); + + if (is_empty()) { + assert(argsize() == 0, ""); + assert(max_thawing_size() == 0, ""); + } + + assert(oopDesc::is_oop_or_null(parent()), ""); + + const bool concurrent = !Thread::current()->is_Java_thread(); + + // If argsize == 0 and the chunk isn't mixed, the chunk contains the metadata (pc, fp -- frame::sender_sp_offset) + // for the top frame (below sp), and *not* for the bottom frame. + int size = stack_size() - argsize() - sp(); + assert(size >= 0, ""); + assert((size == 0) == is_empty(), ""); + + const StackChunkFrameStream first(this); + const bool has_safepoint_stub_frame = first.is_stub(); + + VerifyStackChunkFrameClosure closure(this, + has_safepoint_stub_frame ? 1 : 0, // Iterate_stack skips the safepoint stub + has_safepoint_stub_frame ? first.frame_size() : 0); + iterate_stack(&closure); + + assert(!is_empty() || closure._cb == nullptr, ""); + if (closure._cb != nullptr && closure._cb->is_compiled()) { + assert(argsize() == + (closure._cb->as_compiled_method()->method()->num_stack_arg_slots()*VMRegImpl::stack_slot_size) >>LogBytesPerWord, + "chunk argsize: %d bottom frame argsize: %d", argsize(), + (closure._cb->as_compiled_method()->method()->num_stack_arg_slots()*VMRegImpl::stack_slot_size) >>LogBytesPerWord); + } + + assert(closure._num_interpreted_frames == 0 || has_mixed_frames(), ""); + + if (!concurrent) { + assert(closure._size <= size + argsize() + frame::metadata_words, + "size: %d argsize: %d closure.size: %d end sp: " PTR_FORMAT " start sp: %d chunk size: %d", + size, argsize(), closure._size, closure._sp - start_address(), sp(), stack_size()); + assert(argsize() == closure._argsize, + "argsize(): %d closure.argsize: %d closure.callee_interpreted: %d", + argsize(), closure._argsize, closure._callee_interpreted); + + int calculated_max_size = closure._size + + closure._num_i2c * frame::align_wiggle + + closure._num_interpreted_frames * frame::align_wiggle; + assert(max_thawing_size() == calculated_max_size, + "max_size(): %d calculated_max_size: %d argsize: %d num_i2c: %d", + max_thawing_size(), calculated_max_size, closure._argsize, closure._num_i2c); + + if (out_size != nullptr) *out_size += size; + if (out_oops != nullptr) *out_oops += closure._num_oops; + if (out_frames != nullptr) *out_frames += closure._num_frames; + if (out_interpreted_frames != nullptr) *out_interpreted_frames += closure._num_interpreted_frames; + } else { + assert(out_size == nullptr, ""); + assert(out_oops == nullptr, ""); + assert(out_frames == nullptr, ""); + assert(out_interpreted_frames == nullptr, ""); + } + + if (has_bitmap()) { + assert(bitmap().size() == InstanceStackChunkKlass::bitmap_size_in_bits(stack_size()), + "bitmap().size(): %zu stack_size: %d", + bitmap().size(), stack_size()); + + int oop_count; + if (UseCompressedOops) { + StackChunkVerifyBitmapClosure bitmap_closure(this); + bitmap().iterate(&bitmap_closure, + bit_index_for((narrowOop*)(sp_address() - frame::metadata_words)), + bit_index_for((narrowOop*)end_address())); + oop_count = bitmap_closure._count; + } else { + StackChunkVerifyBitmapClosure bitmap_closure(this); + bitmap().iterate(&bitmap_closure, + bit_index_for((oop*)(sp_address() - frame::metadata_words)), + bit_index_for((oop*)end_address())); + oop_count = bitmap_closure._count; + } + assert(oop_count == closure._num_oops, + "bitmap_closure._count: %d closure._num_oops: %d", oop_count, closure._num_oops); + } + + return true; +} +#endif diff --git a/src/hotspot/share/oops/stackChunkOop.hpp b/src/hotspot/share/oops/stackChunkOop.hpp new file mode 100644 index 00000000000..29fcfaf4404 --- /dev/null +++ b/src/hotspot/share/oops/stackChunkOop.hpp @@ -0,0 +1,211 @@ +/* + * Copyright (c) 2021, 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_OOPS_STACKCHUNKOOP_HPP +#define SHARE_OOPS_STACKCHUNKOOP_HPP + + +#include "oops/instanceOop.hpp" +#include "runtime/handles.hpp" +#include "runtime/stackChunkFrameStream.hpp" +#include "utilities/bitMap.hpp" +#include "utilities/macros.hpp" + +class frame; +class MemRegion; +class RegisterMap; +class VMRegImpl; +typedef VMRegImpl* VMReg; + +// A continuation stack-chunk oop. +// See InstanceStackChunkKlass for a description of continuation stack-chunks. +// +// size and sp are in machine words +// max_size is the maximum space a thawed chunk would take up on the stack, *not* including top-most frame's metadata +class stackChunkOopDesc : public instanceOopDesc { +public: + enum class BarrierType { Load, Store }; + +private: + template friend class DoBarriersStackClosure; + + // Chunk flags. + // FLAG_HAS_INTERPRETED_FRAMES actually means "thaw slow because of some content-based chunk condition" + // It is set whenever we freeze slow, but generally signifies there might be interpreted/deoptimized/stub frames + static const uint8_t FLAG_HAS_INTERPRETED_FRAMES = 1 << 0; + static const uint8_t FLAG_CLAIM_RELATIVIZE = 1 << 1; // Only one thread claims relativization of derived pointers + static const uint8_t FLAG_NOTIFY_RELATIVIZE = 1 << 2; // Someone is waiting for relativization to complete + static const uint8_t FLAG_GC_MODE = 1 << 3; // Once true it and FLAG_HAS_INTERPRETED_FRAMES can't change + static const uint8_t FLAG_HAS_BITMAP = 1 << 4; // Can only be true if FLAG_GC_MODE is true + + bool try_acquire_relativization(); + void release_relativization(); + +public: + static inline stackChunkOop cast(oop obj); + + inline stackChunkOop parent() const; + inline void set_parent(stackChunkOop value); + template + inline bool is_parent_null() const; + template + inline void set_parent_raw(oop value); + + inline int stack_size() const; + + inline int sp() const; + inline void set_sp(int value); + + inline address pc() const; + inline void set_pc(address value); + + inline int argsize() const; + inline void set_argsize(int value); + + inline uint8_t flags() const; + inline uint8_t flags_acquire() const; + inline void set_flags(uint8_t value); + inline void release_set_flags(uint8_t value); + + inline int max_thawing_size() const; + inline void set_max_thawing_size(int value); + + inline oop cont() const; + template inline oop cont() const; + inline void set_cont(oop value); + template + inline void set_cont_raw(oop value); + + inline int bottom() const; + + inline HeapWord* start_of_stack() const; + + inline intptr_t* start_address() const; + inline intptr_t* end_address() const; + inline intptr_t* bottom_address() const; // = end_address - argsize + inline intptr_t* sp_address() const; + + + inline int to_offset(intptr_t* p) const; + inline intptr_t* from_offset(int offset) const; + + inline bool is_empty() const; + inline bool is_in_chunk(void* p) const; + inline bool is_usable_in_chunk(void* p) const; + + inline bool is_flag(uint8_t flag) const; + inline bool is_flag_acquire(uint8_t flag) const; + inline void set_flag(uint8_t flag, bool value); + inline bool try_set_flags(uint8_t prev_flags, uint8_t new_flags); + inline void clear_flags(); + + inline bool has_mixed_frames() const; + inline void set_has_mixed_frames(bool value); + + inline bool is_gc_mode() const; + inline bool is_gc_mode_acquire() const; + inline void set_gc_mode(bool value); + + inline bool has_bitmap() const; + inline void set_has_bitmap(bool value); + + inline bool has_thaw_slowpath_condition() const; + + inline bool requires_barriers(); + + template + void do_barriers(); + + template + inline void do_barriers(const StackChunkFrameStream& f, const RegisterMapT* map); + + template + void fix_thawed_frame(const frame& f, const RegisterMapT* map); + + template + inline void iterate_stack(StackChunkFrameClosureType* closure); + + // Derived pointers are relativized, with respect to their oop base. + void relativize_derived_pointers_concurrently(); + void transform(); + + inline BitMapView bitmap() const; + inline BitMap::idx_t bit_index_for(intptr_t* p) const; + inline intptr_t* address_for_bit(BitMap::idx_t index) const; + template inline BitMap::idx_t bit_index_for(OopT* p) const; + template inline OopT* address_for_bit(BitMap::idx_t index) const; + + MemRegion range(); + + // Returns a relative frame (with offset_sp, offset_unextended_sp, and offset_fp) that can be held + // during safepoints. This is orthogonal to the relativizing of the actual content of interpreted frames. + // To be used, frame objects need to be derelativized with `derelativize`. + inline frame relativize(frame fr) const; + inline frame derelativize(frame fr) const; + + frame top_frame(RegisterMap* map); + frame sender(const frame& fr, RegisterMap* map); + + inline int relativize_usp_offset(const frame& fr, const int usp_offset_in_bytes) const; + inline address usp_offset_to_location(const frame& fr, const int usp_offset_in_bytes) const; + inline address reg_to_location(const frame& fr, const RegisterMap* map, VMReg reg) const; + + // Access to relativized interpreter frames (both the frame object and frame content are relativized) + inline Method* interpreter_frame_method(const frame& fr); + inline address interpreter_frame_bcp(const frame& fr); + inline intptr_t* interpreter_frame_expression_stack_at(const frame& fr, int index) const; + inline intptr_t* interpreter_frame_local_at(const frame& fr, int index) const; + + int num_java_frames() const; + + inline void copy_from_stack_to_chunk(intptr_t* from, intptr_t* to, int size); + inline void copy_from_chunk_to_stack(intptr_t* from, intptr_t* to, int size); + + using oopDesc::print_on; + void print_on(bool verbose, outputStream* st) const; + + // Verifies the consistency of the chunk's data + bool verify(size_t* out_size = NULL, int* out_oops = NULL, + int* out_frames = NULL, int* out_interpreted_frames = NULL) NOT_DEBUG({ return true; }); + +private: + template + void do_barriers0(const StackChunkFrameStream& f, const RegisterMapT* map); + + template + inline void iterate_stack(StackChunkFrameClosureType* closure); + + inline intptr_t* relative_base() const; + + inline intptr_t* derelativize_address(int offset) const; + int relativize_address(intptr_t* p) const; + + inline void relativize_frame(frame& fr) const; + inline void derelativize_frame(frame& fr) const; + + inline void relativize_frame_pd(frame& fr) const; + inline void derelativize_frame_pd(frame& fr) const; +}; + +#endif // SHARE_OOPS_STACKCHUNKOOP_HPP diff --git a/src/hotspot/share/oops/stackChunkOop.inline.hpp b/src/hotspot/share/oops/stackChunkOop.inline.hpp new file mode 100644 index 00000000000..8d7b181b6e3 --- /dev/null +++ b/src/hotspot/share/oops/stackChunkOop.inline.hpp @@ -0,0 +1,374 @@ +/* + * Copyright (c) 2021, 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_OOPS_STACKCHUNKOOP_INLINE_HPP +#define SHARE_OOPS_STACKCHUNKOOP_INLINE_HPP + +#include "oops/stackChunkOop.hpp" + +#include "gc/shared/collectedHeap.hpp" +#include "memory/memRegion.hpp" +#include "memory/universe.hpp" +#include "oops/instanceStackChunkKlass.inline.hpp" +#include "runtime/frame.inline.hpp" +#include "runtime/handles.inline.hpp" +#include "runtime/registerMap.hpp" +#include "runtime/smallRegisterMap.inline.hpp" +#include "utilities/macros.hpp" +#include CPU_HEADER_INLINE(stackChunkOop) + +DEF_HANDLE_CONSTR(stackChunk, is_stackChunk_noinline) + +inline stackChunkOop stackChunkOopDesc::cast(oop obj) { + assert(obj == nullptr || obj->is_stackChunk(), "Wrong type"); + return stackChunkOop(obj); +} + +inline stackChunkOop stackChunkOopDesc::parent() const { return stackChunkOopDesc::cast(jdk_internal_vm_StackChunk::parent(as_oop())); } +template +inline bool stackChunkOopDesc::is_parent_null() const { return jdk_internal_vm_StackChunk::is_parent_null

(as_oop()); } +inline void stackChunkOopDesc::set_parent(stackChunkOop value) { jdk_internal_vm_StackChunk::set_parent(this, value); } +template +inline void stackChunkOopDesc::set_parent_raw(oop value) { jdk_internal_vm_StackChunk::set_parent_raw

(this, value); } + +inline int stackChunkOopDesc::stack_size() const { return jdk_internal_vm_StackChunk::size(as_oop()); } + +inline int stackChunkOopDesc::sp() const { return jdk_internal_vm_StackChunk::sp(as_oop()); } +inline void stackChunkOopDesc::set_sp(int value) { jdk_internal_vm_StackChunk::set_sp(this, value); } + +inline address stackChunkOopDesc::pc() const { return (address)jdk_internal_vm_StackChunk::pc(as_oop()); } +inline void stackChunkOopDesc::set_pc(address value) { jdk_internal_vm_StackChunk::set_pc(this, (intptr_t)value); } + +inline int stackChunkOopDesc::argsize() const { return jdk_internal_vm_StackChunk::argsize(as_oop()); } +inline void stackChunkOopDesc::set_argsize(int value) { jdk_internal_vm_StackChunk::set_argsize(as_oop(), value); } + +inline uint8_t stackChunkOopDesc::flags() const { return jdk_internal_vm_StackChunk::flags(as_oop()); } +inline void stackChunkOopDesc::set_flags(uint8_t value) { jdk_internal_vm_StackChunk::set_flags(this, value); } + +inline uint8_t stackChunkOopDesc::flags_acquire() const { return jdk_internal_vm_StackChunk::flags_acquire(as_oop()); } + +inline void stackChunkOopDesc::release_set_flags(uint8_t value) { + jdk_internal_vm_StackChunk::release_set_flags(this, value); +} + +inline bool stackChunkOopDesc::try_set_flags(uint8_t prev_flags, uint8_t new_flags) { + return jdk_internal_vm_StackChunk::try_set_flags(this, prev_flags, new_flags); +} + +inline int stackChunkOopDesc::max_thawing_size() const { return jdk_internal_vm_StackChunk::maxThawingSize(as_oop()); } +inline void stackChunkOopDesc::set_max_thawing_size(int value) { + assert(value >= 0, "size must be >= 0"); + jdk_internal_vm_StackChunk::set_maxThawingSize(this, (jint)value); +} + +inline oop stackChunkOopDesc::cont() const { return UseCompressedOops ? cont() : cont(); /* jdk_internal_vm_StackChunk::cont(as_oop()); */ } +template +inline oop stackChunkOopDesc::cont() const { + oop obj = jdk_internal_vm_StackChunk::cont_raw

(as_oop()); + obj = (oop)NativeAccess<>::oop_load(&obj); + return obj; +} +inline void stackChunkOopDesc::set_cont(oop value) { jdk_internal_vm_StackChunk::set_cont(this, value); } +template +inline void stackChunkOopDesc::set_cont_raw(oop value) { jdk_internal_vm_StackChunk::set_cont_raw

(this, value); } + +inline int stackChunkOopDesc::bottom() const { return stack_size() - argsize(); } + +inline HeapWord* stackChunkOopDesc::start_of_stack() const { + return (HeapWord*)(cast_from_oop(as_oop()) + InstanceStackChunkKlass::offset_of_stack()); +} + +inline intptr_t* stackChunkOopDesc::start_address() const { return (intptr_t*)start_of_stack(); } +inline intptr_t* stackChunkOopDesc::end_address() const { return start_address() + stack_size(); } +inline intptr_t* stackChunkOopDesc::bottom_address() const { return start_address() + bottom(); } +inline intptr_t* stackChunkOopDesc::sp_address() const { return start_address() + sp(); } + +inline int stackChunkOopDesc::to_offset(intptr_t* p) const { + assert(is_in_chunk(p) + || (p >= start_address() && (p - start_address()) <= stack_size() + frame::metadata_words), + "p: " INTPTR_FORMAT " start: " INTPTR_FORMAT " end: " INTPTR_FORMAT, p2i(p), p2i(start_address()), p2i(bottom_address())); + return p - start_address(); +} + +inline intptr_t* stackChunkOopDesc::from_offset(int offset) const { + assert(offset <= stack_size(), ""); + return start_address() + offset; +} + +inline bool stackChunkOopDesc::is_empty() const { + assert(sp() <= stack_size(), ""); + assert((sp() == stack_size()) == (sp() >= stack_size() - argsize()), + "sp: %d size: %d argsize: %d", sp(), stack_size(), argsize()); + return sp() == stack_size(); +} + +inline bool stackChunkOopDesc::is_in_chunk(void* p) const { + HeapWord* start = (HeapWord*)start_address(); + HeapWord* end = start + stack_size(); + return (HeapWord*)p >= start && (HeapWord*)p < end; +} + +bool stackChunkOopDesc::is_usable_in_chunk(void* p) const { +#if (defined(X86) || defined(AARCH64)) && !defined(ZERO) + HeapWord* start = (HeapWord*)start_address() + sp() - frame::sender_sp_offset; +#else + Unimplemented(); + HeapWord* start = NULL; +#endif + HeapWord* end = start + stack_size(); + return (HeapWord*)p >= start && (HeapWord*)p < end; +} + +inline bool stackChunkOopDesc::is_flag(uint8_t flag) const { + return (flags() & flag) != 0; +} +inline bool stackChunkOopDesc::is_flag_acquire(uint8_t flag) const { + return (flags_acquire() & flag) != 0; +} +inline void stackChunkOopDesc::set_flag(uint8_t flag, bool value) { + uint32_t flags = this->flags(); + set_flags((uint8_t)(value ? flags |= flag : flags &= ~flag)); +} +inline void stackChunkOopDesc::clear_flags() { + set_flags(0); +} + +inline bool stackChunkOopDesc::has_mixed_frames() const { return is_flag(FLAG_HAS_INTERPRETED_FRAMES); } +inline void stackChunkOopDesc::set_has_mixed_frames(bool value) { + assert((flags() & ~FLAG_HAS_INTERPRETED_FRAMES) == 0, "other flags should not be set"); + set_flag(FLAG_HAS_INTERPRETED_FRAMES, value); +} + +inline bool stackChunkOopDesc::is_gc_mode() const { return is_flag(FLAG_GC_MODE); } +inline bool stackChunkOopDesc::is_gc_mode_acquire() const { return is_flag_acquire(FLAG_GC_MODE); } +inline void stackChunkOopDesc::set_gc_mode(bool value) { set_flag(FLAG_GC_MODE, value); } + +inline bool stackChunkOopDesc::has_bitmap() const { return is_flag(FLAG_HAS_BITMAP); } +inline void stackChunkOopDesc::set_has_bitmap(bool value) { set_flag(FLAG_HAS_BITMAP, value); } + +inline bool stackChunkOopDesc::has_thaw_slowpath_condition() const { return flags() != 0; } + +inline bool stackChunkOopDesc::requires_barriers() { + return Universe::heap()->requires_barriers(this); +} + +template +void stackChunkOopDesc::do_barriers(const StackChunkFrameStream& f, const RegisterMapT* map) { + if (frame_kind == ChunkFrames::Mixed) { + // we could freeze deopted frames in slow mode. + f.handle_deopted(); + } + do_barriers0(f, map); +} + +template +inline void stackChunkOopDesc::iterate_stack(StackChunkFrameClosureType* closure) { + has_mixed_frames() ? iterate_stack(closure) + : iterate_stack(closure); +} + +template +inline void stackChunkOopDesc::iterate_stack(StackChunkFrameClosureType* closure) { + const SmallRegisterMap* map = SmallRegisterMap::instance; + assert(!map->in_cont(), ""); + + StackChunkFrameStream f(this); + bool should_continue = true; + + if (f.is_stub()) { + RegisterMap full_map((JavaThread*)nullptr, true, false, true); + full_map.set_include_argument_oops(false); + + f.next(&full_map); + + assert(!f.is_done(), ""); + assert(f.is_compiled(), ""); + + should_continue = closure->do_frame(f, &full_map); + f.next(map); + f.handle_deopted(); // the stub caller might be deoptimized (as it's not at a call) + } + assert(!f.is_stub(), ""); + + for(; should_continue && !f.is_done(); f.next(map)) { + if (frame_kind == ChunkFrames::Mixed) { + // in slow mode we might freeze deoptimized frames + f.handle_deopted(); + } + should_continue = closure->do_frame(f, map); + } +} + +inline frame stackChunkOopDesc::relativize(frame fr) const { relativize_frame(fr); return fr; } +inline frame stackChunkOopDesc::derelativize(frame fr) const { derelativize_frame(fr); return fr; } + +inline BitMapView stackChunkOopDesc::bitmap() const { + int stack_sz = stack_size(); + + // The bitmap is located after the stack. + HeapWord* bitmap_addr = start_of_stack() + stack_sz; + size_t bitmap_size_in_bits = InstanceStackChunkKlass::bitmap_size_in_bits(stack_sz); + + BitMapView bitmap((BitMap::bm_word_t*)bitmap_addr, bitmap_size_in_bits); + + DEBUG_ONLY(bitmap.verify_range(bit_index_for(start_address()), bit_index_for(end_address()));) + + return bitmap; +} + +inline BitMap::idx_t stackChunkOopDesc::bit_index_for(intptr_t* p) const { + return UseCompressedOops ? bit_index_for((narrowOop*)p) : bit_index_for((oop*)p); +} + +template +inline BitMap::idx_t stackChunkOopDesc::bit_index_for(OopT* p) const { + assert(p >= (OopT*)start_address(), "Address not in chunk"); + return p - (OopT*)start_address(); +} + +inline intptr_t* stackChunkOopDesc::address_for_bit(BitMap::idx_t index) const { + return UseCompressedOops ? (intptr_t*)address_for_bit(index) : (intptr_t*)address_for_bit(index); +} + +template +inline OopT* stackChunkOopDesc::address_for_bit(BitMap::idx_t index) const { + return (OopT*)start_address() + index; +} + +inline MemRegion stackChunkOopDesc::range() { + return MemRegion((HeapWord*)this, size()); +} + +inline int stackChunkOopDesc::relativize_usp_offset(const frame& fr, const int usp_offset_in_bytes) const { + assert(fr.is_compiled_frame() || fr.cb()->is_safepoint_stub(), ""); + assert(is_in_chunk(fr.unextended_sp()), ""); + + intptr_t* base = fr.real_fp(); // equal to the caller's sp + intptr_t* loc = (intptr_t*)((address)fr.unextended_sp() + usp_offset_in_bytes); + assert(base > loc, ""); + return (int)(base - loc); +} + +inline address stackChunkOopDesc::usp_offset_to_location(const frame& fr, const int usp_offset_in_bytes) const { + assert(fr.is_compiled_frame(), ""); + return (address)derelativize_address(fr.offset_unextended_sp()) + usp_offset_in_bytes; +} + +inline address stackChunkOopDesc::reg_to_location(const frame& fr, const RegisterMap* map, VMReg reg) const { + assert(fr.is_compiled_frame(), ""); + assert(map != nullptr, ""); + assert(map->stack_chunk() == as_oop(), ""); + + // the offsets are saved in the map after going through relativize_usp_offset, so they are sp - loc, in words + intptr_t offset = (intptr_t)map->location(reg, nullptr); // see usp_offset_to_index for the chunk case + intptr_t* base = derelativize_address(fr.offset_sp()); + return (address)(base - offset); +} + +inline Method* stackChunkOopDesc::interpreter_frame_method(const frame& fr) { + return derelativize(fr).interpreter_frame_method(); +} + +inline address stackChunkOopDesc::interpreter_frame_bcp(const frame& fr) { + return derelativize(fr).interpreter_frame_bcp(); +} + +inline intptr_t* stackChunkOopDesc::interpreter_frame_expression_stack_at(const frame& fr, int index) const { + frame heap_frame = derelativize(fr); + assert(heap_frame.is_heap_frame(), "must be"); + return heap_frame.interpreter_frame_expression_stack_at(index); +} + +inline intptr_t* stackChunkOopDesc::interpreter_frame_local_at(const frame& fr, int index) const { + frame heap_frame = derelativize(fr); + assert(heap_frame.is_heap_frame(), "must be"); + return heap_frame.interpreter_frame_local_at(index); +} + +inline void stackChunkOopDesc::copy_from_stack_to_chunk(intptr_t* from, intptr_t* to, int size) { + log_develop_trace(continuations)("Copying from v: " INTPTR_FORMAT " - " INTPTR_FORMAT " (%d words, %d bytes)", + p2i(from), p2i(from + size), size, size << LogBytesPerWord); + log_develop_trace(continuations)("Copying to h: " INTPTR_FORMAT "(" INTPTR_FORMAT "," INTPTR_FORMAT ") - " INTPTR_FORMAT "(" INTPTR_FORMAT "," INTPTR_FORMAT ") (%d words, %d bytes)", + p2i(to), to - start_address(), relative_base() - to, p2i(to + size), to + size - start_address(), + relative_base() - (to + size), size, size << LogBytesPerWord); + + assert(to >= start_address(), "Chunk underflow"); + assert(to + size <= end_address(), "Chunk overflow"); + + memcpy(to, from, size << LogBytesPerWord); +} + +inline void stackChunkOopDesc::copy_from_chunk_to_stack(intptr_t* from, intptr_t* to, int size) { + log_develop_trace(continuations)("Copying from h: " INTPTR_FORMAT "(" INTPTR_FORMAT "," INTPTR_FORMAT ") - " INTPTR_FORMAT "(" INTPTR_FORMAT "," INTPTR_FORMAT ") (%d words, %d bytes)", + p2i(from), from - start_address(), relative_base() - from, p2i(from + size), from + size - start_address(), + relative_base() - (from + size), size, size << LogBytesPerWord); + log_develop_trace(continuations)("Copying to v: " INTPTR_FORMAT " - " INTPTR_FORMAT " (%d words, %d bytes)", p2i(to), + p2i(to + size), size, size << LogBytesPerWord); + + assert(from >= start_address(), ""); + assert(from + size <= end_address(), ""); + +#if !defined(AMD64) || !defined(AARCH64) || defined(ZERO) + // Suppress compilation error from dummy function (somewhere). + if (to != nullptr) +#endif + memcpy(to, from, size << LogBytesPerWord); +} + +inline intptr_t* stackChunkOopDesc::relative_base() const { + // we relativize with respect to end rather than start because GC might compact the chunk + return end_address() + frame::metadata_words; +} + +inline intptr_t* stackChunkOopDesc::derelativize_address(int offset) const { + intptr_t* base = relative_base(); + intptr_t* p = base - offset; + assert(start_address() <= p && p <= base, "start_address: " PTR_FORMAT " p: " PTR_FORMAT " base: " PTR_FORMAT, + p2i(start_address()), p2i(p), p2i(base)); + return p; +} + +inline int stackChunkOopDesc::relativize_address(intptr_t* p) const { + intptr_t* base = relative_base(); + intptr_t offset = base - p; + assert(start_address() <= p && p <= base, "start_address: " PTR_FORMAT " p: " PTR_FORMAT " base: " PTR_FORMAT, + p2i(start_address()), p2i(p), p2i(base)); + assert(0 <= offset && offset <= std::numeric_limits::max(), "offset: " PTR_FORMAT, offset); + return offset; +} + +inline void stackChunkOopDesc::relativize_frame(frame& fr) const { + fr.set_offset_sp(relativize_address(fr.sp())); + fr.set_offset_unextended_sp(relativize_address(fr.unextended_sp())); + relativize_frame_pd(fr); +} + +inline void stackChunkOopDesc::derelativize_frame(frame& fr) const { + fr.set_sp(derelativize_address(fr.offset_sp())); + fr.set_unextended_sp(derelativize_address(fr.offset_unextended_sp())); + derelativize_frame_pd(fr); + fr.set_frame_index(-1); // for the sake of assertions in frame +} + +#endif // SHARE_OOPS_STACKCHUNKOOP_INLINE_HPP diff --git a/src/hotspot/share/opto/buildOopMap.cpp b/src/hotspot/share/opto/buildOopMap.cpp index 8c9d45947eb..fdcf1f0ffc3 100644 --- a/src/hotspot/share/opto/buildOopMap.cpp +++ b/src/hotspot/share/opto/buildOopMap.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2002, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2002, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -367,11 +367,13 @@ OopMap *OopFlow::build_oop_map( Node *n, int max_reg, PhaseRegAlloc *regalloc, i #endif #ifdef ASSERT + bool has_derived_oops = false; for( OopMapStream oms1(omap); !oms1.is_done(); oms1.next()) { OopMapValue omv1 = oms1.current(); if (omv1.type() != OopMapValue::derived_oop_value) { continue; } + has_derived_oops = true; bool found = false; for( OopMapStream oms2(omap); !oms2.is_done(); oms2.next()) { OopMapValue omv2 = oms2.current(); @@ -383,8 +385,18 @@ OopMap *OopFlow::build_oop_map( Node *n, int max_reg, PhaseRegAlloc *regalloc, i break; } } + assert(has_derived_oops == omap->has_derived_oops(), ""); assert( found, "derived with no base in oopmap" ); } + + int num_oops = 0; + for (OopMapStream oms2(omap); !oms2.is_done(); oms2.next()) { + OopMapValue omv = oms2.current(); + if (omv.type() == OopMapValue::oop_value || omv.type() == OopMapValue::narrowoop_value) { + num_oops++; + } + } + assert(num_oops == omap->num_oops(), "num_oops: %d omap->num_oops(): %d", num_oops, omap->num_oops()); #endif return omap; diff --git a/src/hotspot/share/opto/bytecodeInfo.cpp b/src/hotspot/share/opto/bytecodeInfo.cpp index 36f6e93454a..6cb1eaeba5c 100644 --- a/src/hotspot/share/opto/bytecodeInfo.cpp +++ b/src/hotspot/share/opto/bytecodeInfo.cpp @@ -215,6 +215,13 @@ bool InlineTree::should_not_inline(ciMethod* callee_method, ciMethod* caller_met fail_msg = "don't inline by annotation"; } + // Don't inline a method that changes Thread.currentThread() except + // into another method that is annotated @ChangesCurrentThread. + if (callee_method->changes_current_thread() + && ! C->method()->changes_current_thread()) { + fail_msg = "method changes current thread"; + } + // one more inlining restriction if (fail_msg == NULL && callee_method->has_unloaded_classes_in_signature()) { fail_msg = "unloaded signature classes"; diff --git a/src/hotspot/share/opto/c2compiler.cpp b/src/hotspot/share/opto/c2compiler.cpp index 9bfcb9688ea..075191ff912 100644 --- a/src/hotspot/share/opto/c2compiler.cpp +++ b/src/hotspot/share/opto/c2compiler.cpp @@ -638,10 +638,13 @@ bool C2Compiler::is_intrinsic_supported(const methodHandle& method, bool is_virt case vmIntrinsics::_storeFence: case vmIntrinsics::_storeStoreFence: case vmIntrinsics::_fullFence: + case vmIntrinsics::_currentCarrierThread: case vmIntrinsics::_currentThread: + case vmIntrinsics::_setCurrentThread: + case vmIntrinsics::_extentLocalCache: + case vmIntrinsics::_setExtentLocalCache: #ifdef JFR_HAVE_INTRINSICS case vmIntrinsics::_counterTime: - case vmIntrinsics::_getClassId: case vmIntrinsics::_getEventWriter: #endif case vmIntrinsics::_currentTimeMillis: @@ -709,6 +712,7 @@ bool C2Compiler::is_intrinsic_supported(const methodHandle& method, bool is_virt case vmIntrinsics::_Preconditions_checkIndex: case vmIntrinsics::_Preconditions_checkLongIndex: case vmIntrinsics::_getObjectSize: + case vmIntrinsics::_Continuation_doYield: break; case vmIntrinsics::_VectorUnaryOp: diff --git a/src/hotspot/share/opto/compile.cpp b/src/hotspot/share/opto/compile.cpp index f626215036a..0d923b5b8ca 100644 --- a/src/hotspot/share/opto/compile.cpp +++ b/src/hotspot/share/opto/compile.cpp @@ -997,6 +997,7 @@ void Compile::Init(int aliaslevel) { set_do_scheduling(OptoScheduling); set_do_vector_loop(false); + set_has_monitors(false); if (AllowVectorizeOnDemand) { if (has_method() && (_directive->VectorizeOption || _directive->VectorizeDebugOption)) { diff --git a/src/hotspot/share/opto/compile.hpp b/src/hotspot/share/opto/compile.hpp index b663d17bd4b..e101c70c03d 100644 --- a/src/hotspot/share/opto/compile.hpp +++ b/src/hotspot/share/opto/compile.hpp @@ -337,6 +337,7 @@ class Compile : public Phase { bool _has_irreducible_loop; // Found irreducible loops // JSR 292 bool _has_method_handle_invokes; // True if this method has MethodHandle invokes. + bool _has_monitors; // Metadata transfered to nmethod to enable Continuations lock-detection fastpath RTMState _rtm_state; // State of Restricted Transactional Memory usage int _loop_opts_cnt; // loop opts round bool _clinit_barrier_on_entry; // True if clinit barrier is needed on nmethod entry @@ -631,6 +632,8 @@ class Compile : public Phase { void set_max_node_limit(uint n) { _max_node_limit = n; } bool clinit_barrier_on_entry() { return _clinit_barrier_on_entry; } void set_clinit_barrier_on_entry(bool z) { _clinit_barrier_on_entry = z; } + bool has_monitors() const { return _has_monitors; } + void set_has_monitors(bool v) { _has_monitors = v; } // check the CompilerOracle for special behaviours for this compile bool method_has_option(enum CompileCommand option) { diff --git a/src/hotspot/share/opto/library_call.cpp b/src/hotspot/share/opto/library_call.cpp index 7e5ba508494..6ad28ff237b 100644 --- a/src/hotspot/share/opto/library_call.cpp +++ b/src/hotspot/share/opto/library_call.cpp @@ -57,10 +57,6 @@ #include "utilities/macros.hpp" #include "utilities/powerOfTwo.hpp" -#if INCLUDE_JFR -#include "jfr/jfr.hpp" -#endif - //---------------------------make_vm_intrinsic---------------------------- CallGenerator* Compile::make_vm_intrinsic(ciMethod* m, bool is_virtual) { vmIntrinsicID id = m->intrinsic_id(); @@ -472,11 +468,15 @@ bool LibraryCallKit::try_to_inline(int predicate) { case vmIntrinsics::_onSpinWait: return inline_onspinwait(); + case vmIntrinsics::_currentCarrierThread: return inline_native_currentCarrierThread(); case vmIntrinsics::_currentThread: return inline_native_currentThread(); + case vmIntrinsics::_setCurrentThread: return inline_native_setCurrentThread(); + + case vmIntrinsics::_extentLocalCache: return inline_native_extentLocalCache(); + case vmIntrinsics::_setExtentLocalCache: return inline_native_setExtentLocalCache(); #ifdef JFR_HAVE_INTRINSICS - case vmIntrinsics::_counterTime: return inline_native_time_funcs(CAST_FROM_FN_PTR(address, JFR_TIME_FUNCTION), "counterTime"); - case vmIntrinsics::_getClassId: return inline_native_classID(); + case vmIntrinsics::_counterTime: return inline_native_time_funcs(CAST_FROM_FN_PTR(address, JfrTime::time_function()), "counterTime"); case vmIntrinsics::_getEventWriter: return inline_native_getEventWriter(); #endif case vmIntrinsics::_currentTimeMillis: return inline_native_time_funcs(CAST_FROM_FN_PTR(address, os::javaTimeMillis), "currentTimeMillis"); @@ -631,6 +631,9 @@ bool LibraryCallKit::try_to_inline(int predicate) { case vmIntrinsics::_fmaF: return inline_fma(intrinsic_id()); + case vmIntrinsics::_Continuation_doYield: + return inline_continuation_do_yield(); + case vmIntrinsics::_isDigit: case vmIntrinsics::_isLowerCase: case vmIntrinsics::_isUpperCase: @@ -889,18 +892,41 @@ void LibraryCallKit::generate_string_range_check(Node* array, Node* offset, Node } } -//--------------------------generate_current_thread-------------------- -Node* LibraryCallKit::generate_current_thread(Node* &tls_output) { - ciKlass* thread_klass = env()->Thread_klass(); - const Type* thread_type = TypeOopPtr::make_from_klass(thread_klass)->cast_to_ptr_type(TypePtr::NotNull); +Node* LibraryCallKit::current_thread_helper(Node*& tls_output, ByteSize handle_offset, + bool is_immutable) { + ciKlass* thread_klass = env()->Thread_klass(); + const Type* thread_type + = TypeOopPtr::make_from_klass(thread_klass)->cast_to_ptr_type(TypePtr::NotNull); + Node* thread = _gvn.transform(new ThreadLocalNode()); - Node* p = basic_plus_adr(top()/*!oop*/, thread, in_bytes(JavaThread::threadObj_offset())); + Node* p = basic_plus_adr(top()/*!oop*/, thread, in_bytes(handle_offset)); tls_output = thread; - Node* thread_obj_handle = LoadNode::make(_gvn, NULL, immutable_memory(), p, p->bottom_type()->is_ptr(), TypeRawPtr::NOTNULL, T_ADDRESS, MemNode::unordered); + + Node* thread_obj_handle + = (is_immutable + ? LoadNode::make(_gvn, NULL, immutable_memory(), p, p->bottom_type()->is_ptr(), + TypeRawPtr::NOTNULL, T_ADDRESS, MemNode::unordered) + : make_load(NULL, p, p->bottom_type()->is_ptr(), T_ADDRESS, MemNode::unordered)); thread_obj_handle = _gvn.transform(thread_obj_handle); - return access_load(thread_obj_handle, thread_type, T_OBJECT, IN_NATIVE | C2_IMMUTABLE_MEMORY); + + DecoratorSet decorators = IN_NATIVE; + if (is_immutable) { + decorators |= C2_IMMUTABLE_MEMORY; + } + return access_load(thread_obj_handle, thread_type, T_OBJECT, decorators); } +//--------------------------generate_current_thread-------------------- +Node* LibraryCallKit::generate_current_thread(Node* &tls_output) { + return current_thread_helper(tls_output, JavaThread::threadObj_offset(), + /*is_immutable*/false); +} + +//--------------------------generate_virtual_thread-------------------- +Node* LibraryCallKit::generate_virtual_thread(Node* tls_output) { + return current_thread_helper(tls_output, JavaThread::vthread_offset(), + !C->method()->changes_current_thread()); +} //------------------------------make_string_method_node------------------------ // Helper method for String intrinsic functions. This version is called with @@ -2935,7 +2961,7 @@ bool LibraryCallKit::inline_native_classID() { Node* kls_trace_id_addr = basic_plus_adr(kls, in_bytes(KLASS_TRACE_ID_OFFSET)); Node* kls_trace_id_raw = ideal.load(ideal.ctrl(), kls_trace_id_addr,TypeLong::LONG, T_LONG, Compile::AliasIdxRaw); - Node* epoch_address = makecon(TypeRawPtr::make(Jfr::epoch_address())); + Node* epoch_address = makecon(TypeRawPtr::make(JfrIntrinsicSupport::epoch_address())); Node* epoch = ideal.load(ideal.ctrl(), epoch_address, TypeInt::BOOL, T_BOOLEAN, Compile::AliasIdxRaw); epoch = _gvn.transform(new LShiftLNode(longcon(1), epoch)); Node* mask = _gvn.transform(new LShiftLNode(epoch, intcon(META_SHIFT))); @@ -2946,9 +2972,9 @@ bool LibraryCallKit::inline_native_classID() { __ if_then(kls_trace_id_raw_and_mask, BoolTest::ne, epoch, unlikely); { sync_kit(ideal); make_runtime_call(RC_LEAF, - OptoRuntime::get_class_id_intrinsic_Type(), - CAST_FROM_FN_PTR(address, Jfr::get_class_id_intrinsic), - "get_class_id_intrinsic", + OptoRuntime::class_id_load_barrier_Type(), + CAST_FROM_FN_PTR(address, JfrIntrinsicSupport::load_barrier), + "class id load barrier", TypePtr::BOTTOM, kls); ideal.sync_kit(this); @@ -2969,7 +2995,7 @@ bool LibraryCallKit::inline_native_classID() { ideal.set(result, _gvn.transform(longcon(LAST_TYPE_ID + 1))); } __ end_if(); - Node* signaled_flag_address = makecon(TypeRawPtr::make(Jfr::signal_address())); + Node* signaled_flag_address = makecon(TypeRawPtr::make(JfrIntrinsicSupport::signal_address())); Node* signaled = ideal.load(ideal.ctrl(), signaled_flag_address, TypeInt::BOOL, T_BOOLEAN, Compile::AliasIdxRaw, true, MemNode::acquire); __ if_then(signaled, BoolTest::ne, ideal.ConI(1)); { ideal.store(ideal.ctrl(), signaled_flag_address, ideal.ConI(1), T_BOOLEAN, Compile::AliasIdxRaw, MemNode::release, true); @@ -2982,49 +3008,496 @@ bool LibraryCallKit::inline_native_classID() { return true; } +/* + * The intrinsic is a model of this pseudo-code: + * + * JfrThreadLocal* const tl = Thread::jfr_thread_local() + * jobject h_event_writer = tl->java_event_writer(); + * if (h_event_writer == NULL) { + * return NULL; + * } + * oop threadObj = Thread::threadObj(); + * oop vthread = java_lang_Thread::vthread(threadObj); + * traceid tid; + * bool excluded; + * if (vthread != threadObj) { // i.e. current thread is virtual + * tid = java_lang_Thread::tid(vthread); + * u2 vthread_epoch_raw = java_lang_Thread::jfr_epoch(vthread); + * excluded = vthread_epoch_raw & excluded_mask; + * if (!excluded) { + * traceid current_epoch = JfrTraceIdEpoch::current_generation(); + * u2 vthread_epoch = vthread_epoch_raw & epoch_mask; + * if (vthread_epoch != current_epoch) { + * write_checkpoint(); + * } + * } + * } else { + * tid = java_lang_Thread::tid(threadObj); + * u2 thread_epoch_raw = java_lang_Thread::jfr_epoch(threadObj); + * excluded = thread_epoch_raw & excluded_mask; + * } + * oop event_writer = JNIHandles::resolve_non_null(h_event_writer); + * traceid tid_in_event_writer = getField(event_writer, "threadID"); + * if (tid_in_event_writer != tid) { + * setField(event_writer, "threadID", tid); + * setField(event_writer, "excluded", excluded); + * } + * return event_writer + */ bool LibraryCallKit::inline_native_getEventWriter() { + enum { _true_path = 1, _false_path = 2, PATH_LIMIT }; + + // Save input memory and i_o state. + Node* input_memory_state = reset_memory(); + set_all_memory(input_memory_state); + Node* input_io_state = i_o(); + + Node* excluded_mask = _gvn.intcon(32768); + Node* epoch_mask = _gvn.intcon(32767); + + // TLS Node* tls_ptr = _gvn.transform(new ThreadLocalNode()); - Node* jobj_ptr = basic_plus_adr(top(), tls_ptr, - in_bytes(THREAD_LOCAL_WRITER_OFFSET_JFR)); + // Load the address of java event writer jobject handle from the jfr_thread_local structure. + Node* jobj_ptr = basic_plus_adr(top(), tls_ptr, in_bytes(THREAD_LOCAL_WRITER_OFFSET_JFR)); + // Load the eventwriter jobject handle. Node* jobj = make_load(control(), jobj_ptr, TypeRawPtr::BOTTOM, T_ADDRESS, MemNode::unordered); - Node* jobj_cmp_null = _gvn.transform( new CmpPNode(jobj, null()) ); - Node* test_jobj_eq_null = _gvn.transform( new BoolNode(jobj_cmp_null, BoolTest::eq) ); + // Null check the jobject handle. + Node* jobj_cmp_null = _gvn.transform(new CmpPNode(jobj, null())); + Node* test_jobj_not_equal_null = _gvn.transform(new BoolNode(jobj_cmp_null, BoolTest::ne)); + IfNode* iff_jobj_not_equal_null = create_and_map_if(control(), test_jobj_not_equal_null, PROB_MAX, COUNT_UNKNOWN); - IfNode* iff_jobj_null = - create_and_map_if(control(), test_jobj_eq_null, PROB_MIN, COUNT_UNKNOWN); + // False path, jobj is null. + Node* jobj_is_null = _gvn.transform(new IfFalseNode(iff_jobj_not_equal_null)); - enum { _normal_path = 1, - _null_path = 2, - PATH_LIMIT }; + // True path, jobj is not null. + Node* jobj_is_not_null = _gvn.transform(new IfTrueNode(iff_jobj_not_equal_null)); - RegionNode* result_rgn = new RegionNode(PATH_LIMIT); - PhiNode* result_val = new PhiNode(result_rgn, TypeInstPtr::BOTTOM); - - Node* jobj_is_null = _gvn.transform(new IfTrueNode(iff_jobj_null)); - result_rgn->init_req(_null_path, jobj_is_null); - result_val->init_req(_null_path, null()); - - Node* jobj_is_not_null = _gvn.transform(new IfFalseNode(iff_jobj_null)); set_control(jobj_is_not_null); - Node* res = access_load(jobj, TypeInstPtr::NOTNULL, T_OBJECT, - IN_NATIVE | C2_CONTROL_DEPENDENT_LOAD); - result_rgn->init_req(_normal_path, control()); - result_val->init_req(_normal_path, res); - set_result(result_rgn, result_val); + // Load the threadObj for the CarrierThread. + Node* threadObj = generate_current_thread(tls_ptr); + // Load the vthread. + Node* vthread = generate_virtual_thread(tls_ptr); + + // If vthread != threadObj, this is a virtual thread. + Node* vthread_cmp_threadObj = _gvn.transform(new CmpPNode(vthread, threadObj)); + Node* test_vthread_not_equal_threadObj = _gvn.transform(new BoolNode(vthread_cmp_threadObj, BoolTest::ne)); + IfNode* iff_vthread_not_equal_threadObj = + create_and_map_if(jobj_is_not_null, test_vthread_not_equal_threadObj, PROB_FAIR, COUNT_UNKNOWN); + + // False branch, fallback to threadObj. + Node* vthread_equal_threadObj = _gvn.transform(new IfFalseNode(iff_vthread_not_equal_threadObj)); + set_control(vthread_equal_threadObj); + + // Load the tid field from the vthread object. + Node* thread_obj_tid = load_field_from_object(threadObj, "tid", "J"); + + // Load the raw epoch value from the threadObj. + Node* threadObj_epoch_offset = basic_plus_adr(threadObj, java_lang_Thread::jfr_epoch_offset()); + Node* threadObj_epoch_raw = access_load_at(threadObj, threadObj_epoch_offset, TypeRawPtr::BOTTOM, TypeInt::CHAR, T_CHAR, + IN_HEAP | MO_UNORDERED | C2_MISMATCHED | C2_CONTROL_DEPENDENT_LOAD); + + // Mask off the excluded information from the epoch. + Node * threadObj_is_excluded = _gvn.transform(new AndINode(threadObj_epoch_raw, excluded_mask)); + + // True branch, this is a virtual thread. + Node* vthread_not_equal_threadObj = _gvn.transform(new IfTrueNode(iff_vthread_not_equal_threadObj)); + set_control(vthread_not_equal_threadObj); + + // Load the tid field from the vthread object. + Node* vthread_tid = load_field_from_object(vthread, "tid", "J"); + + // Load the raw epoch value from the vthread. + Node* vthread_epoch_offset = basic_plus_adr(vthread, java_lang_Thread::jfr_epoch_offset()); + Node* vthread_epoch_raw = access_load_at(vthread, vthread_epoch_offset, TypeRawPtr::BOTTOM, TypeInt::CHAR, T_CHAR, + IN_HEAP | MO_UNORDERED | C2_MISMATCHED | C2_CONTROL_DEPENDENT_LOAD); + + // Mask off the excluded information from the epoch. + Node * vthread_is_excluded = _gvn.transform(new AndINode(vthread_epoch_raw, _gvn.transform(excluded_mask))); + + // Branch on excluded to conditionalize updating the epoch for the virtual thread. + Node* is_excluded_cmp = _gvn.transform(new CmpINode(vthread_is_excluded, _gvn.transform(excluded_mask))); + Node* test_not_excluded = _gvn.transform(new BoolNode(is_excluded_cmp, BoolTest::ne)); + IfNode* iff_not_excluded = create_and_map_if(control(), test_not_excluded, PROB_MAX, COUNT_UNKNOWN); + + // False branch, vthread is excluded, no need to write epoch info. + Node* excluded = _gvn.transform(new IfFalseNode(iff_not_excluded)); + + // True branch, vthread is included, update epoch info. + Node* included = _gvn.transform(new IfTrueNode(iff_not_excluded)); + set_control(included); + + // Get epoch value. + Node* epoch = _gvn.transform(new AndINode(vthread_epoch_raw, _gvn.transform(epoch_mask))); + + // Load the current epoch generation. The value is unsigned 16-bit, so we type it as T_CHAR. + Node* epoch_generation_address = makecon(TypeRawPtr::make(JfrIntrinsicSupport::epoch_generation_address())); + Node* current_epoch_generation = make_load(control(), epoch_generation_address, TypeInt::CHAR, T_CHAR, MemNode::unordered); + + // Compare the epoch in the vthread to the current epoch generation. + Node* const epoch_cmp = _gvn.transform(new CmpUNode(current_epoch_generation, epoch)); + Node* test_epoch_not_equal = _gvn.transform(new BoolNode(epoch_cmp, BoolTest::ne)); + IfNode* iff_epoch_not_equal = create_and_map_if(control(), test_epoch_not_equal, PROB_FAIR, COUNT_UNKNOWN); + + // False path, epoch is equal, checkpoint information is valid. + Node* epoch_is_equal = _gvn.transform(new IfFalseNode(iff_epoch_not_equal)); + + // True path, epoch is not equal, write a checkpoint for the vthread. + Node* epoch_is_not_equal = _gvn.transform(new IfTrueNode(iff_epoch_not_equal)); + + set_control(epoch_is_not_equal); + + // Make a runtime call, which can safepoint, to write a checkpoint for the vthread for this epoch. + // The call also updates the native thread local thread id and the vthread with the current epoch. + Node* call_write_checkpoint = make_runtime_call(RC_NO_LEAF, + OptoRuntime::jfr_write_checkpoint_Type(), + StubRoutines::jfr_write_checkpoint(), + "write_checkpoint", TypePtr::BOTTOM); + Node* call_write_checkpoint_control = _gvn.transform(new ProjNode(call_write_checkpoint, TypeFunc::Control)); + + // vthread epoch != current epoch + RegionNode* epoch_compare_rgn = new RegionNode(PATH_LIMIT); + record_for_igvn(epoch_compare_rgn); + PhiNode* epoch_compare_mem = new PhiNode(epoch_compare_rgn, Type::MEMORY, TypePtr::BOTTOM); + record_for_igvn(epoch_compare_mem); + PhiNode* epoch_compare_io = new PhiNode(epoch_compare_rgn, Type::ABIO); + record_for_igvn(epoch_compare_io); + + // Update control and phi nodes. + epoch_compare_rgn->init_req(_true_path, call_write_checkpoint_control); + epoch_compare_rgn->init_req(_false_path, epoch_is_equal); + epoch_compare_mem->init_req(_true_path, _gvn.transform(reset_memory())); + epoch_compare_mem->init_req(_false_path, input_memory_state); + epoch_compare_io->init_req(_true_path, i_o()); + epoch_compare_io->init_req(_false_path, input_io_state); + + // excluded != true + RegionNode* exclude_compare_rgn = new RegionNode(PATH_LIMIT); + record_for_igvn(exclude_compare_rgn); + PhiNode* exclude_compare_mem = new PhiNode(exclude_compare_rgn, Type::MEMORY, TypePtr::BOTTOM); + record_for_igvn(exclude_compare_mem); + PhiNode* exclude_compare_io = new PhiNode(exclude_compare_rgn, Type::ABIO); + record_for_igvn(exclude_compare_io); + + // Update control and phi nodes. + exclude_compare_rgn->init_req(_true_path, _gvn.transform(epoch_compare_rgn)); + exclude_compare_rgn->init_req(_false_path, excluded); + exclude_compare_mem->init_req(_true_path, _gvn.transform(epoch_compare_mem)); + exclude_compare_mem->init_req(_false_path, input_memory_state); + exclude_compare_io->init_req(_true_path, _gvn.transform(epoch_compare_io)); + exclude_compare_io->init_req(_false_path, input_io_state); + + // vthread != threadObj + RegionNode* vthread_compare_rgn = new RegionNode(PATH_LIMIT); + record_for_igvn(vthread_compare_rgn); + PhiNode* vthread_compare_mem = new PhiNode(vthread_compare_rgn, Type::MEMORY, TypePtr::BOTTOM); + PhiNode* vthread_compare_io = new PhiNode(vthread_compare_rgn, Type::ABIO); + record_for_igvn(vthread_compare_io); + PhiNode* tid = new PhiNode(vthread_compare_rgn, TypeLong::LONG); + record_for_igvn(tid); + PhiNode* exclusion = new PhiNode(vthread_compare_rgn, TypeInt::BOOL); + record_for_igvn(exclusion); + + // Update control and phi nodes. + vthread_compare_rgn->init_req(_true_path, _gvn.transform(exclude_compare_rgn)); + vthread_compare_rgn->init_req(_false_path, vthread_equal_threadObj); + vthread_compare_mem->init_req(_true_path, _gvn.transform(exclude_compare_mem)); + vthread_compare_mem->init_req(_false_path, input_memory_state); + vthread_compare_io->init_req(_true_path, _gvn.transform(exclude_compare_io)); + vthread_compare_io->init_req(_false_path, input_io_state); + tid->init_req(_true_path, _gvn.transform(vthread_tid)); + tid->init_req(_false_path, _gvn.transform(thread_obj_tid)); + exclusion->init_req(_true_path, _gvn.transform(vthread_is_excluded)); + exclusion->init_req(_false_path, _gvn.transform(threadObj_is_excluded)); + + // Update branch state. + set_control(_gvn.transform(vthread_compare_rgn)); + set_all_memory(_gvn.transform(vthread_compare_mem)); + set_i_o(_gvn.transform(vthread_compare_io)); + + // Load the event writer oop by dereferencing the jobject handle. + ciKlass* klass_EventWriter = env()->find_system_klass(ciSymbol::make("jdk/jfr/internal/EventWriter")); + assert(klass_EventWriter->is_loaded(), "invariant"); + ciInstanceKlass* const instklass_EventWriter = klass_EventWriter->as_instance_klass(); + const TypeKlassPtr* const aklass = TypeKlassPtr::make(instklass_EventWriter); + const TypeOopPtr* const xtype = aklass->as_instance_type(); + Node* event_writer = access_load(jobj, xtype, T_OBJECT, IN_NATIVE | C2_CONTROL_DEPENDENT_LOAD); + + // Load the current thread id from the event writer object. + Node* const event_writer_tid = load_field_from_object(event_writer, "threadID", "J"); + // Get the field offset to, conditionally, store an updated tid value later. + Node* const event_writer_tid_field = field_address_from_object(event_writer, "threadID", "J", false); + const TypePtr* event_writer_tid_field_type = _gvn.type(event_writer_tid_field)->isa_ptr(); + // Get the field offset to, conditionally, store an updated exclusion value later. + Node* const event_writer_excluded_field = field_address_from_object(event_writer, "excluded", "Z", false); + const TypePtr* event_writer_excluded_field_type = _gvn.type(event_writer_excluded_field)->isa_ptr(); + + RegionNode* event_writer_tid_compare_rgn = new RegionNode(PATH_LIMIT); + record_for_igvn(event_writer_tid_compare_rgn); + PhiNode* event_writer_tid_compare_mem = new PhiNode(event_writer_tid_compare_rgn, Type::MEMORY, TypePtr::BOTTOM); + record_for_igvn(event_writer_tid_compare_mem); + PhiNode* event_writer_tid_compare_io = new PhiNode(event_writer_tid_compare_rgn, Type::ABIO); + record_for_igvn(event_writer_tid_compare_io); + + // Compare the current tid from the thread object to what is currently stored in the event writer object. + Node* const tid_cmp = _gvn.transform(new CmpLNode(event_writer_tid, _gvn.transform(tid))); + Node* test_tid_not_equal = _gvn.transform(new BoolNode(tid_cmp, BoolTest::ne)); + IfNode* iff_tid_not_equal = create_and_map_if(_gvn.transform(vthread_compare_rgn), test_tid_not_equal, PROB_FAIR, COUNT_UNKNOWN); + + // False path, tids are the same. + Node* tid_is_equal = _gvn.transform(new IfFalseNode(iff_tid_not_equal)); + + // True path, tid is not equal, need to update the tid in the event writer. + Node* tid_is_not_equal = _gvn.transform(new IfTrueNode(iff_tid_not_equal)); + record_for_igvn(tid_is_not_equal); + + // Store the exclusion state to the event writer. + store_to_memory(tid_is_not_equal, event_writer_excluded_field, _gvn.transform(exclusion), T_BOOLEAN, event_writer_excluded_field_type, MemNode::unordered); + + // Store the tid to the event writer. + store_to_memory(tid_is_not_equal, event_writer_tid_field, tid, T_LONG, event_writer_tid_field_type, MemNode::unordered); + + // Update control and phi nodes. + event_writer_tid_compare_rgn->init_req(_true_path, tid_is_not_equal); + event_writer_tid_compare_rgn->init_req(_false_path, tid_is_equal); + event_writer_tid_compare_mem->init_req(_true_path, _gvn.transform(reset_memory())); + event_writer_tid_compare_mem->init_req(_false_path, _gvn.transform(vthread_compare_mem)); + event_writer_tid_compare_io->init_req(_true_path, _gvn.transform(i_o())); + event_writer_tid_compare_io->init_req(_false_path, _gvn.transform(vthread_compare_io)); + + // Result of top level CFG, Memory, IO and Value. + RegionNode* result_rgn = new RegionNode(PATH_LIMIT); + PhiNode* result_mem = new PhiNode(result_rgn, Type::MEMORY, TypePtr::BOTTOM); + PhiNode* result_io = new PhiNode(result_rgn, Type::ABIO); + PhiNode* result_value = new PhiNode(result_rgn, TypeInstPtr::BOTTOM); + + // Result control. + result_rgn->init_req(_true_path, _gvn.transform(event_writer_tid_compare_rgn)); + result_rgn->init_req(_false_path, jobj_is_null); + + // Result memory. + result_mem->init_req(_true_path, _gvn.transform(event_writer_tid_compare_mem)); + result_mem->init_req(_false_path, _gvn.transform(input_memory_state)); + + // Result IO. + result_io->init_req(_true_path, _gvn.transform(event_writer_tid_compare_io)); + result_io->init_req(_false_path, _gvn.transform(input_io_state)); + + // Result value. + result_value->init_req(_true_path, _gvn.transform(event_writer)); // return event writer oop + result_value->init_req(_false_path, null()); // return NULL + + // Set output state. + set_control(_gvn.transform(result_rgn)); + set_all_memory(_gvn.transform(result_mem)); + set_i_o(_gvn.transform(result_io)); + set_result(result_rgn, result_value); return true; } +/* + * The intrinsic is a model of this pseudo-code: + * + * JfrThreadLocal* const tl = thread->jfr_thread_local(); + * if (carrierThread != thread) { // is virtual thread + * const u2 vthread_epoch_raw = java_lang_Thread::jfr_epoch(thread); + * bool excluded = vthread_epoch_raw & excluded_mask; + * Atomic::store(&tl->_contextual_tid, java_lang_Thread::tid(thread)); + * Atomic::store(&tl->_contextual_thread_excluded, is_excluded); + * if (!excluded) { + * const u2 vthread_epoch = vthread_epoch_raw & epoch_mask; + * Atomic::store(&tl->_vthread_epoch, vthread_epoch); + * } + * Atomic::release_store(&tl->_vthread, true); + * return; + * } + * Atomic::release_store(&tl->_vthread, false); + */ +void LibraryCallKit::extend_setCurrentThread(Node* jt, Node* thread) { + enum { _true_path = 1, _false_path = 2, PATH_LIMIT }; + + Node* input_memory_state = reset_memory(); + set_all_memory(input_memory_state); + + Node* excluded_mask = _gvn.intcon(32768); + Node* epoch_mask = _gvn.intcon(32767); + + Node* const carrierThread = generate_current_thread(jt); + // If thread != carrierThread, this is a virtual thread. + Node* thread_cmp_carrierThread = _gvn.transform(new CmpPNode(thread, carrierThread)); + Node* test_thread_not_equal_carrierThread = _gvn.transform(new BoolNode(thread_cmp_carrierThread, BoolTest::ne)); + IfNode* iff_thread_not_equal_carrierThread = + create_and_map_if(control(), test_thread_not_equal_carrierThread, PROB_FAIR, COUNT_UNKNOWN); + + Node* vthread_offset = basic_plus_adr(jt, in_bytes(THREAD_LOCAL_OFFSET_JFR + VTHREAD_OFFSET_JFR)); + + // False branch, is carrierThread. + Node* thread_equal_carrierThread = _gvn.transform(new IfFalseNode(iff_thread_not_equal_carrierThread)); + // Store release + Node* vthread_false_memory = store_to_memory(thread_equal_carrierThread, vthread_offset, _gvn.intcon(0), T_BOOLEAN, Compile::AliasIdxRaw, MemNode::release, true); + + set_all_memory(input_memory_state); + + // True branch, is virtual thread. + Node* thread_not_equal_carrierThread = _gvn.transform(new IfTrueNode(iff_thread_not_equal_carrierThread)); + set_control(thread_not_equal_carrierThread); + + // Load the raw epoch value from the vthread. + Node* epoch_offset = basic_plus_adr(thread, java_lang_Thread::jfr_epoch_offset()); + Node* epoch_raw = access_load_at(thread, epoch_offset, TypeRawPtr::BOTTOM, TypeInt::CHAR, T_CHAR, + IN_HEAP | MO_UNORDERED | C2_MISMATCHED | C2_CONTROL_DEPENDENT_LOAD); + + // Mask off the excluded information from the epoch. + Node * const is_excluded = _gvn.transform(new AndINode(epoch_raw, _gvn.transform(excluded_mask))); + + // Load the tid field from the thread. + Node* tid = load_field_from_object(thread, "tid", "J"); + + // Store the vthread tid to the jfr thread local. + Node* thread_id_offset = basic_plus_adr(jt, in_bytes(THREAD_LOCAL_OFFSET_JFR + VTHREAD_ID_OFFSET_JFR)); + Node* tid_memory = store_to_memory(control(), thread_id_offset, tid, T_LONG, Compile::AliasIdxRaw, MemNode::unordered, true); + + // Branch is_excluded to conditionalize updating the epoch . + Node* excluded_cmp = _gvn.transform(new CmpINode(is_excluded, _gvn.transform(excluded_mask))); + Node* test_excluded = _gvn.transform(new BoolNode(excluded_cmp, BoolTest::eq)); + IfNode* iff_excluded = create_and_map_if(control(), test_excluded, PROB_MIN, COUNT_UNKNOWN); + + // True branch, vthread is excluded, no need to write epoch info. + Node* excluded = _gvn.transform(new IfTrueNode(iff_excluded)); + set_control(excluded); + Node* vthread_is_excluded = _gvn.intcon(1); + + // False branch, vthread is included, update epoch info. + Node* included = _gvn.transform(new IfFalseNode(iff_excluded)); + set_control(included); + Node* vthread_is_included = _gvn.intcon(0); + + // Get epoch value. + Node* epoch = _gvn.transform(new AndINode(epoch_raw, _gvn.transform(epoch_mask))); + + // Store the vthread epoch to the jfr thread local. + Node* vthread_epoch_offset = basic_plus_adr(jt, in_bytes(THREAD_LOCAL_OFFSET_JFR + VTHREAD_EPOCH_OFFSET_JFR)); + Node* included_memory = store_to_memory(control(), vthread_epoch_offset, epoch, T_CHAR, Compile::AliasIdxRaw, MemNode::unordered, true); + + RegionNode* excluded_rgn = new RegionNode(PATH_LIMIT); + record_for_igvn(excluded_rgn); + PhiNode* excluded_mem = new PhiNode(excluded_rgn, Type::MEMORY, TypePtr::BOTTOM); + record_for_igvn(excluded_mem); + PhiNode* exclusion = new PhiNode(excluded_rgn, TypeInt::BOOL); + record_for_igvn(exclusion); + + // Merge the excluded control and memory. + excluded_rgn->init_req(_true_path, excluded); + excluded_rgn->init_req(_false_path, included); + excluded_mem->init_req(_true_path, tid_memory); + excluded_mem->init_req(_false_path, included_memory); + exclusion->init_req(_true_path, _gvn.transform(vthread_is_excluded)); + exclusion->init_req(_false_path, _gvn.transform(vthread_is_included)); + + // Set intermediate state. + set_control(_gvn.transform(excluded_rgn)); + set_all_memory(excluded_mem); + + // Store the vthread exclusion state to the jfr thread local. + Node* thread_local_excluded_offset = basic_plus_adr(jt, in_bytes(THREAD_LOCAL_OFFSET_JFR + VTHREAD_EXCLUDED_OFFSET_JFR)); + store_to_memory(control(), thread_local_excluded_offset, _gvn.transform(exclusion), T_BOOLEAN, Compile::AliasIdxRaw, MemNode::unordered, true); + + // Store release + Node * vthread_true_memory = store_to_memory(control(), vthread_offset, _gvn.intcon(1), T_BOOLEAN, Compile::AliasIdxRaw, MemNode::release, true); + + RegionNode* thread_compare_rgn = new RegionNode(PATH_LIMIT); + record_for_igvn(thread_compare_rgn); + PhiNode* thread_compare_mem = new PhiNode(thread_compare_rgn, Type::MEMORY, TypePtr::BOTTOM); + record_for_igvn(thread_compare_mem); + PhiNode* vthread = new PhiNode(thread_compare_rgn, TypeInt::BOOL); + record_for_igvn(vthread); + + // Merge the thread_compare control and memory. + thread_compare_rgn->init_req(_true_path, control()); + thread_compare_rgn->init_req(_false_path, thread_equal_carrierThread); + thread_compare_mem->init_req(_true_path, vthread_true_memory); + thread_compare_mem->init_req(_false_path, vthread_false_memory); + + // Set output state. + set_control(_gvn.transform(thread_compare_rgn)); + set_all_memory(_gvn.transform(thread_compare_mem)); +} + #endif // JFR_HAVE_INTRINSICS +//------------------------inline_native_currentCarrierThread------------------ +bool LibraryCallKit::inline_native_currentCarrierThread() { + Node* junk = NULL; + set_result(generate_current_thread(junk)); + return true; +} + //------------------------inline_native_currentThread------------------ bool LibraryCallKit::inline_native_currentThread() { Node* junk = NULL; - set_result(generate_current_thread(junk)); + set_result(generate_virtual_thread(junk)); + return true; +} + +//------------------------inline_native_setVthread------------------ +bool LibraryCallKit::inline_native_setCurrentThread() { + assert(C->method()->changes_current_thread(), + "method changes current Thread but is not annotated ChangesCurrentThread"); + Node* arr = argument(1); + Node* thread = _gvn.transform(new ThreadLocalNode()); + Node* p = basic_plus_adr(top()/*!oop*/, thread, in_bytes(JavaThread::vthread_offset())); + Node* thread_obj_handle + = make_load(NULL, p, p->bottom_type()->is_ptr(), T_OBJECT, MemNode::unordered); + thread_obj_handle = _gvn.transform(thread_obj_handle); + const TypePtr *adr_type = _gvn.type(thread_obj_handle)->isa_ptr(); + // Stores of oops to native memory not supported yet by BarrierSetC2::store_at_resolved + // access_store_at(NULL, thread_obj_handle, adr_type, arr, _gvn.type(arr), T_OBJECT, IN_NATIVE | MO_UNORDERED); + store_to_memory(control(), thread_obj_handle, arr, T_OBJECT, adr_type, MemNode::unordered); + JFR_ONLY(extend_setCurrentThread(thread, arr);) + return true; +} + +Node* LibraryCallKit::extentLocalCache_helper() { + ciKlass *objects_klass = ciObjArrayKlass::make(env()->Object_klass()); + const TypeOopPtr *etype = TypeOopPtr::make_from_klass(env()->Object_klass()); + + bool xk = etype->klass_is_exact(); + + Node* thread = _gvn.transform(new ThreadLocalNode()); + Node* p = basic_plus_adr(top()/*!oop*/, thread, in_bytes(JavaThread::extentLocalCache_offset())); + return _gvn.transform(LoadNode::make(_gvn, NULL, immutable_memory(), p, p->bottom_type()->is_ptr(), + TypeRawPtr::NOTNULL, T_ADDRESS, MemNode::unordered)); +} + +//------------------------inline_native_extentLocalCache------------------ +bool LibraryCallKit::inline_native_extentLocalCache() { + ciKlass *objects_klass = ciObjArrayKlass::make(env()->Object_klass()); + const TypeOopPtr *etype = TypeOopPtr::make_from_klass(env()->Object_klass()); + const TypeAry* arr0 = TypeAry::make(etype, TypeInt::POS); + + // Because we create the extentLocal cache lazily we have to make the + // type of the result BotPTR. + bool xk = etype->klass_is_exact(); + const Type* objects_type = TypeAryPtr::make(TypePtr::BotPTR, arr0, objects_klass, xk, 0); + Node* cache_obj_handle = extentLocalCache_helper(); + set_result(access_load(cache_obj_handle, objects_type, T_OBJECT, IN_NATIVE)); + + return true; +} + +//------------------------inline_native_setExtentLocalCache------------------ +bool LibraryCallKit::inline_native_setExtentLocalCache() { + Node* arr = argument(0); + Node* cache_obj_handle = extentLocalCache_helper(); + + const TypePtr *adr_type = _gvn.type(cache_obj_handle)->isa_ptr(); + store_to_memory(control(), cache_obj_handle, arr, T_OBJECT, adr_type, + MemNode::unordered); + return true; } @@ -5888,8 +6361,8 @@ bool LibraryCallKit::inline_reference_refersTo0(bool is_phantom) { Node* LibraryCallKit::load_field_from_object(Node* fromObj, const char* fieldName, const char* fieldTypeString, - DecoratorSet decorators = IN_HEAP, bool is_static = false, - ciInstanceKlass* fromKls = NULL) { + DecoratorSet decorators, bool is_static, + ciInstanceKlass* fromKls) { if (fromKls == NULL) { const TypeInstPtr* tinst = _gvn.type(fromObj)->isa_instptr(); assert(tinst != NULL, "obj is null"); @@ -5902,7 +6375,7 @@ Node* LibraryCallKit::load_field_from_object(Node* fromObj, const char* fieldNam ciSymbol::make(fieldTypeString), is_static); - assert (field != NULL, "undefined field"); + assert(field != NULL, "undefined field %s %s %s", fieldTypeString, fromKls->name()->as_utf8(), fieldName); if (field == NULL) return (Node *) NULL; if (is_static) { @@ -5937,8 +6410,8 @@ Node* LibraryCallKit::load_field_from_object(Node* fromObj, const char* fieldNam } Node * LibraryCallKit::field_address_from_object(Node * fromObj, const char * fieldName, const char * fieldTypeString, - bool is_exact = true, bool is_static = false, - ciInstanceKlass * fromKls = NULL) { + bool is_exact /* true */, bool is_static /* false */, + ciInstanceKlass * fromKls /* NULL */) { if (fromKls == NULL) { const TypeInstPtr* tinst = _gvn.type(fromObj)->isa_instptr(); assert(tinst != NULL, "obj is null"); @@ -7033,6 +7506,15 @@ Node* LibraryCallKit::inline_digestBase_implCompressMB_predicate(int predicate) return instof_false; // even if it is NULL } +bool LibraryCallKit::inline_continuation_do_yield() { + address call_addr = StubRoutines::cont_doYield(); + const TypeFunc* tf = OptoRuntime::continuation_doYield_Type(); + Node* call = make_runtime_call(RC_NO_LEAF, tf, call_addr, "doYield", TypeRawPtr::BOTTOM); + Node* result = _gvn.transform(new ProjNode(call, TypeFunc::Parms)); + set_result(result); + return true; +} + //-------------inline_fma----------------------------------- bool LibraryCallKit::inline_fma(vmIntrinsics::ID id) { Node *a = NULL; diff --git a/src/hotspot/share/opto/library_call.hpp b/src/hotspot/share/opto/library_call.hpp index 00ed88b106b..3135356e565 100644 --- a/src/hotspot/share/opto/library_call.hpp +++ b/src/hotspot/share/opto/library_call.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -134,7 +134,10 @@ class LibraryCallKit : public GraphKit { RegionNode* region); void generate_string_range_check(Node* array, Node* offset, Node* length, bool char_count); + Node* current_thread_helper(Node* &tls_output, ByteSize handle_offset, + bool is_immutable); Node* generate_current_thread(Node* &tls_output); + Node* generate_virtual_thread(Node* threadObj); Node* load_mirror_from_klass(Node* klass); Node* load_klass_from_mirror_common(Node* mirror, bool never_see_null, RegionNode* region, int null_path, @@ -181,8 +184,8 @@ class LibraryCallKit : public GraphKit { CallJavaNode* generate_method_call_virtual(vmIntrinsics::ID method_id) { return generate_method_call(method_id, true, false); } - Node* load_field_from_object(Node* fromObj, const char* fieldName, const char* fieldTypeString, DecoratorSet decorators, bool is_static, ciInstanceKlass* fromKls); - Node* field_address_from_object(Node* fromObj, const char* fieldName, const char* fieldTypeString, bool is_exact, bool is_static, ciInstanceKlass* fromKls); + Node* load_field_from_object(Node* fromObj, const char* fieldName, const char* fieldTypeString, DecoratorSet decorators = IN_HEAP, bool is_static = false, ciInstanceKlass* fromKls = NULL); + Node* field_address_from_object(Node* fromObj, const char* fieldName, const char* fieldTypeString, bool is_exact = true, bool is_static = false, ciInstanceKlass* fromKls = NULL); Node* make_string_method_node(int opcode, Node* str1_start, Node* cnt1, Node* str2_start, Node* cnt2, StrIntrinsicNode::ArgEnc ae); bool inline_string_compareTo(StrIntrinsicNode::ArgEnc ae); @@ -231,12 +234,20 @@ class LibraryCallKit : public GraphKit { bool inline_unsafe_writeback0(); bool inline_unsafe_writebackSync0(bool is_pre); bool inline_unsafe_copyMemory(); + + bool inline_native_currentCarrierThread(); bool inline_native_currentThread(); + bool inline_native_setCurrentThread(); + + bool inline_native_extentLocalCache(); + Node* extentLocalCache_helper(); + bool inline_native_setExtentLocalCache(); bool inline_native_time_funcs(address method, const char* funcName); #ifdef JFR_HAVE_INTRINSICS bool inline_native_classID(); bool inline_native_getEventWriter(); + void extend_setCurrentThread(Node* jt, Node* thread); #endif bool inline_native_Class_query(vmIntrinsics::ID id); bool inline_native_subtype_check(); @@ -313,6 +324,8 @@ class LibraryCallKit : public GraphKit { bool inline_profileBoolean(); bool inline_isCompileConstant(); + bool inline_continuation_do_yield(); + // Vector API support bool inline_vector_nary_operation(int n); bool inline_vector_frombits_coerced(); diff --git a/src/hotspot/share/opto/locknode.cpp b/src/hotspot/share/opto/locknode.cpp index 77b79eabdef..ea605b7a2cd 100644 --- a/src/hotspot/share/opto/locknode.cpp +++ b/src/hotspot/share/opto/locknode.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1999, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1999, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -179,6 +179,8 @@ void FastLockNode::create_rtm_lock_counter(JVMState* state) { void Parse::do_monitor_enter() { kill_dead_locals(); + C->set_has_monitors(true); + // Null check; get casted pointer. Node* obj = null_check(peek()); // Check for locking null object @@ -196,6 +198,10 @@ void Parse::do_monitor_enter() { void Parse::do_monitor_exit() { kill_dead_locals(); + // need to set it for monitor exit as well. + // OSR compiled methods can start with lock taken + C->set_has_monitors(true); + pop(); // Pop oop to unlock // Because monitors are guaranteed paired (else we bail out), we know // the matching Lock for this Unlock. Hence we know there is no need diff --git a/src/hotspot/share/opto/macro.cpp b/src/hotspot/share/opto/macro.cpp index a035f1e6ca4..5731c099d43 100644 --- a/src/hotspot/share/opto/macro.cpp +++ b/src/hotspot/share/opto/macro.cpp @@ -51,6 +51,7 @@ #include "opto/subtypenode.hpp" #include "opto/type.hpp" #include "prims/jvmtiExport.hpp" +#include "runtime/continuation.hpp" #include "runtime/sharedRuntime.hpp" #include "utilities/macros.hpp" #include "utilities/powerOfTwo.hpp" @@ -2208,9 +2209,30 @@ void PhaseMacroExpand::expand_lock_node(LockNode *lock) { _igvn.replace_node(_callprojs.fallthrough_proj, region); Node *memproj = transform_later(new ProjNode(call, TypeFunc::Memory)); - mem_phi->init_req(1, memproj ); + + Node* thread = transform_later(new ThreadLocalNode()); + if (Continuations::enabled()) { + // held_monitor_count increased in slowpath (complete_monitor_locking_C_inc_held_monitor_count), need compensate a decreament here + // this minimizes control flow changes here and add redundant count updates only in slowpath + Node* dec_count = make_load(slow_ctrl, memproj, thread, in_bytes(JavaThread::held_monitor_count_offset()), TypeInt::INT, TypeInt::INT->basic_type()); + Node* new_dec_count = transform_later(new SubINode(dec_count, intcon(1))); + Node *compensate_dec = make_store(slow_ctrl, memproj, thread, in_bytes(JavaThread::held_monitor_count_offset()), new_dec_count, T_INT); + mem_phi->init_req(1, compensate_dec); + } else { + mem_phi->init_req(1, memproj); + } transform_later(mem_phi); - _igvn.replace_node(_callprojs.fallthrough_memproj, mem_phi); + + if (Continuations::enabled()) { + // held_monitor_count increases in all path's post-dominate + Node* inc_count = make_load(region, mem_phi, thread, in_bytes(JavaThread::held_monitor_count_offset()), TypeInt::INT, TypeInt::INT->basic_type()); + Node* new_inc_count = transform_later(new AddINode(inc_count, intcon(1))); + Node *store = make_store(region, mem_phi, thread, in_bytes(JavaThread::held_monitor_count_offset()), new_inc_count, T_INT); + + _igvn.replace_node(_callprojs.fallthrough_memproj, store); + } else { + _igvn.replace_node(_callprojs.fallthrough_memproj, mem_phi); + } } //------------------------------expand_unlock_node---------------------- @@ -2264,7 +2286,16 @@ void PhaseMacroExpand::expand_unlock_node(UnlockNode *unlock) { mem_phi->init_req(1, memproj ); mem_phi->init_req(2, mem); transform_later(mem_phi); - _igvn.replace_node(_callprojs.fallthrough_memproj, mem_phi); + + if (Continuations::enabled()) { + Node* count = make_load(region, mem_phi, thread, in_bytes(JavaThread::held_monitor_count_offset()), TypeInt::INT, TypeInt::INT->basic_type()); + Node* newcount = transform_later(new SubINode(count, intcon(1))); + Node *store = make_store(region, mem_phi, thread, in_bytes(JavaThread::held_monitor_count_offset()), newcount, T_INT); + + _igvn.replace_node(_callprojs.fallthrough_memproj, store); + } else { + _igvn.replace_node(_callprojs.fallthrough_memproj, mem_phi); + } } void PhaseMacroExpand::expand_subtypecheck_node(SubTypeCheckNode *check) { diff --git a/src/hotspot/share/opto/memnode.cpp b/src/hotspot/share/opto/memnode.cpp index 2150fbbcf71..5a2eb30627d 100644 --- a/src/hotspot/share/opto/memnode.cpp +++ b/src/hotspot/share/opto/memnode.cpp @@ -850,12 +850,25 @@ void LoadNode::dump_spec(outputStream *st) const { //----------------------------is_immutable_value------------------------------- // Helper function to allow a raw load without control edge for some cases bool LoadNode::is_immutable_value(Node* adr) { - return (adr->is_AddP() && adr->in(AddPNode::Base)->is_top() && - adr->in(AddPNode::Address)->Opcode() == Op_ThreadLocal && - (adr->in(AddPNode::Offset)->find_intptr_t_con(-1) == - in_bytes(JavaThread::osthread_offset()) || - adr->in(AddPNode::Offset)->find_intptr_t_con(-1) == - in_bytes(JavaThread::threadObj_offset()))); + if (adr->is_AddP() && adr->in(AddPNode::Base)->is_top() && + adr->in(AddPNode::Address)->Opcode() == Op_ThreadLocal) { + + jlong offset = adr->in(AddPNode::Offset)->find_intptr_t_con(-1); + int offsets[] = { + in_bytes(JavaThread::osthread_offset()), + in_bytes(JavaThread::threadObj_offset()), + in_bytes(JavaThread::vthread_offset()), + in_bytes(JavaThread::extentLocalCache_offset()), + }; + + for (size_t i = 0; i < sizeof offsets / sizeof offsets[0]; i++) { + if (offset == offsets[i]) { + return true; + } + } + } + + return false; } #endif diff --git a/src/hotspot/share/opto/output.cpp b/src/hotspot/share/opto/output.cpp index fec9c5dcfd8..e1edf34b71d 100644 --- a/src/hotspot/share/opto/output.cpp +++ b/src/hotspot/share/opto/output.cpp @@ -3362,6 +3362,8 @@ void PhaseOutput::install_code(ciMethod* target, compiler, has_unsafe_access, SharedRuntime::is_wide_vector(C->max_vector_size()), + C->has_monitors(), + 0, C->rtm_state(), C->native_invokers()); diff --git a/src/hotspot/share/opto/parse1.cpp b/src/hotspot/share/opto/parse1.cpp index 836c9a1bea9..b66b5d620f8 100644 --- a/src/hotspot/share/opto/parse1.cpp +++ b/src/hotspot/share/opto/parse1.cpp @@ -424,6 +424,10 @@ Parse::Parse(JVMState* caller, ciMethod* parse_method, float expected_uses) C->set_has_reserved_stack_access(true); } + if (parse_method->is_synchronized()) { + C->set_has_monitors(true); + } + _tf = TypeFunc::make(method()); _iter.reset_to_method(method()); _flow = method()->get_flow_analysis(); diff --git a/src/hotspot/share/opto/runtime.cpp b/src/hotspot/share/opto/runtime.cpp index c20b55a1b33..825e4c8e7c8 100644 --- a/src/hotspot/share/opto/runtime.cpp +++ b/src/hotspot/share/opto/runtime.cpp @@ -144,7 +144,7 @@ bool OptoRuntime::generate(ciEnv* env) { gen(env, _multianewarray4_Java , multianewarray4_Type , multianewarray4_C , 0 , true, false); gen(env, _multianewarray5_Java , multianewarray5_Type , multianewarray5_C , 0 , true, false); gen(env, _multianewarrayN_Java , multianewarrayN_Type , multianewarrayN_C , 0 , true, false); - gen(env, _complete_monitor_locking_Java , complete_monitor_enter_Type , SharedRuntime::complete_monitor_locking_C, 0, false, false); + gen(env, _complete_monitor_locking_Java , complete_monitor_enter_Type , SharedRuntime::complete_monitor_locking_C_inc_held_monitor_count, 0, false, false); gen(env, _monitor_notify_Java , monitor_notify_Type , monitor_notify_C , 0 , false, false); gen(env, _monitor_notifyAll_Java , monitor_notify_Type , monitor_notifyAll_C , 0 , false, false); gen(env, _rethrow_Java , rethrow_Type , rethrow_C , 2 , true , true ); @@ -715,6 +715,60 @@ const TypeFunc* OptoRuntime::void_long_Type() { return TypeFunc::make(domain, range); } +const TypeFunc* OptoRuntime::void_void_Type() { + // create input type (domain) + const Type **fields = TypeTuple::fields(0); + const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+0, fields); + + // create result type (range) + fields = TypeTuple::fields(0); + const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+0, fields); + return TypeFunc::make(domain, range); + } + + const TypeFunc* OptoRuntime::continuation_doYield_Type() { + // create input type (domain) + const Type **fields = TypeTuple::fields(0); + const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+0, fields); + + // create result type (range) + fields = TypeTuple::fields(1); + fields[TypeFunc::Parms+0] = TypeInt::INT; + const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+1, fields); + + return TypeFunc::make(domain, range); + } + + const TypeFunc* OptoRuntime::continuation_jump_Type() { + // create input type (domain) + const Type **fields = TypeTuple::fields(6); + fields[TypeFunc::Parms+0] = TypeLong::LONG; + fields[TypeFunc::Parms+1] = Type::HALF; + fields[TypeFunc::Parms+2] = TypeLong::LONG; + fields[TypeFunc::Parms+3] = Type::HALF; + fields[TypeFunc::Parms+4] = TypeLong::LONG; + fields[TypeFunc::Parms+5] = Type::HALF; + const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+6, fields); + + // create result type (range) + fields = TypeTuple::fields(0); + const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+0, fields); + return TypeFunc::make(domain, range); + } + + + const TypeFunc* OptoRuntime::jfr_write_checkpoint_Type() { + // create input type (domain) + const Type **fields = TypeTuple::fields(0); + const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms, fields); + + // create result type (range) + fields = TypeTuple::fields(0); + const TypeTuple *range = TypeTuple::make(TypeFunc::Parms, fields); + return TypeFunc::make(domain, range); + } + + // arraycopy stub variations: enum ArrayCopyType { ac_fast, // void(ptr, ptr, size_t) @@ -1550,7 +1604,7 @@ const TypeFunc *OptoRuntime::register_finalizer_Type() { } #if INCLUDE_JFR -const TypeFunc *OptoRuntime::get_class_id_intrinsic_Type() { +const TypeFunc *OptoRuntime::class_id_load_barrier_Type() { // create input type (domain) const Type **fields = TypeTuple::fields(1); fields[TypeFunc::Parms+0] = TypeInstPtr::KLASS; diff --git a/src/hotspot/share/opto/runtime.hpp b/src/hotspot/share/opto/runtime.hpp index b056c2828c3..fb3c0e03772 100644 --- a/src/hotspot/share/opto/runtime.hpp +++ b/src/hotspot/share/opto/runtime.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1998, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1998, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -242,6 +242,11 @@ private: static const TypeFunc* modf_Type(); static const TypeFunc* l2f_Type(); static const TypeFunc* void_long_Type(); + static const TypeFunc* void_void_Type(); + static const TypeFunc* continuation_doYield_Type(); + static const TypeFunc* continuation_jump_Type(); + + static const TypeFunc* jfr_write_checkpoint_Type(); static const TypeFunc* flush_windows_Type(); @@ -288,7 +293,7 @@ private: static const TypeFunc* register_finalizer_Type(); - JFR_ONLY(static const TypeFunc* get_class_id_intrinsic_Type();) + JFR_ONLY(static const TypeFunc* class_id_load_barrier_Type();) // Dtrace support static const TypeFunc* dtrace_method_entry_exit_Type(); diff --git a/src/hotspot/share/prims/forte.cpp b/src/hotspot/share/prims/forte.cpp index af6dad64c20..de9b8b9f57d 100644 --- a/src/hotspot/share/prims/forte.cpp +++ b/src/hotspot/share/prims/forte.cpp @@ -92,8 +92,9 @@ static bool is_decipherable_interpreted_frame(JavaThread* thread, vframeStreamForte::vframeStreamForte(JavaThread *jt, frame fr, - bool stop_at_java_call_stub) : vframeStreamCommon(jt, false /* process_frames */) { - + bool stop_at_java_call_stub) + : vframeStreamCommon(RegisterMap(jt, false, false, false)) { + _reg_map.set_async(true); _stop_at_java_call_stub = stop_at_java_call_stub; _frame = fr; diff --git a/src/hotspot/share/prims/jni.cpp b/src/hotspot/share/prims/jni.cpp index 7e37c59dcb7..24ef171f6c2 100644 --- a/src/hotspot/share/prims/jni.cpp +++ b/src/hotspot/share/prims/jni.cpp @@ -2717,6 +2717,10 @@ JNI_ENTRY(jint, jni_MonitorEnter(JNIEnv *env, jobject jobj)) Handle obj(thread, JNIHandles::resolve_non_null(jobj)); ObjectSynchronizer::jni_enter(obj, thread); + if (!Continuation::pin(thread)) { + ObjectSynchronizer::jni_exit(obj(), CHECK_(JNI_ERR)); + THROW_(vmSymbols::java_lang_VirtualMachineError(), JNI_ERR); + } ret = JNI_OK; return ret; JNI_END @@ -2736,7 +2740,9 @@ JNI_ENTRY(jint, jni_MonitorExit(JNIEnv *env, jobject jobj)) Handle obj(THREAD, JNIHandles::resolve_non_null(jobj)); ObjectSynchronizer::jni_exit(obj(), CHECK_(JNI_ERR)); - + if (!Continuation::unpin(thread)) { + ShouldNotReachHere(); + } ret = JNI_OK; return ret; JNI_END @@ -3135,6 +3141,11 @@ JNI_ENTRY(jobject, jni_GetModule(JNIEnv* env, jclass clazz)) return Modules::get_module(clazz, THREAD); JNI_END +JNI_ENTRY(jboolean, jni_IsVirtualThread(JNIEnv* env, jobject obj)) + oop thread_obj = JNIHandles::resolve_external_guard(obj); + return java_lang_VirtualThread::is_instance(thread_obj) ? JNI_TRUE : JNI_FALSE; +JNI_END + // Structure containing all jni functions struct JNINativeInterface_ jni_NativeInterface = { @@ -3419,7 +3430,11 @@ struct JNINativeInterface_ jni_NativeInterface = { // Module features - jni_GetModule + jni_GetModule, + + // Virtual threads + + jni_IsVirtualThread }; @@ -3494,7 +3509,7 @@ static void post_thread_start_event(const JavaThread* jt) { assert(jt != NULL, "invariant"); EventThreadStart event; if (event.should_commit()) { - event.set_thread(JFR_THREAD_ID(jt)); + event.set_thread(JFR_JVM_THREAD_ID(jt)); event.set_parentThread((traceid)0); #if INCLUDE_JFR if (EventThreadStart::is_stacktrace_enabled()) { diff --git a/src/hotspot/share/prims/jniCheck.cpp b/src/hotspot/share/prims/jniCheck.cpp index c380ee07943..451f1adb128 100644 --- a/src/hotspot/share/prims/jniCheck.cpp +++ b/src/hotspot/share/prims/jniCheck.cpp @@ -2014,7 +2014,16 @@ JNI_ENTRY_CHECKED(jobject, checked_jni_GetModule(JNIEnv *env, jclass clazz)) functionEnter(thr); - jobject result = UNCHECKED()->GetModule(env,clazz); + jobject result = UNCHECKED()->GetModule(env, clazz); + functionExit(thr); + return result; +JNI_END + +JNI_ENTRY_CHECKED(jboolean, + checked_jni_IsVirtualThread(JNIEnv *env, + jobject obj)) + functionEnter(thr); + jboolean result = UNCHECKED()->IsVirtualThread(env, obj); functionExit(thr); return result; JNI_END @@ -2304,7 +2313,11 @@ struct JNINativeInterface_ checked_jni_NativeInterface = { // Module Features - checked_jni_GetModule + checked_jni_GetModule, + + // Virtual threads + + checked_jni_IsVirtualThread }; diff --git a/src/hotspot/share/prims/jvm.cpp b/src/hotspot/share/prims/jvm.cpp index 4283eb540d8..2757841c399 100644 --- a/src/hotspot/share/prims/jvm.cpp +++ b/src/hotspot/share/prims/jvm.cpp @@ -65,10 +65,11 @@ #include "oops/oop.inline.hpp" #include "prims/jvm_misc.hpp" #include "prims/jvmtiExport.hpp" -#include "prims/jvmtiThreadState.hpp" +#include "prims/jvmtiThreadState.inline.hpp" #include "prims/stackwalk.hpp" #include "runtime/arguments.hpp" #include "runtime/atomic.hpp" +#include "runtime/continuation.hpp" #include "runtime/globals_extension.hpp" #include "runtime/handles.inline.hpp" #include "runtime/init.hpp" @@ -85,6 +86,7 @@ #include "runtime/reflection.hpp" #include "runtime/synchronizer.hpp" #include "runtime/thread.inline.hpp" +#include "runtime/threadIdentifier.hpp" #include "runtime/threadSMR.hpp" #include "runtime/vframe.inline.hpp" #include "runtime/vmOperations.hpp" @@ -532,12 +534,12 @@ JVM_END // java.lang.StackTraceElement ////////////////////////////////////////////// -JVM_ENTRY(void, JVM_InitStackTraceElementArray(JNIEnv *env, jobjectArray elements, jobject throwable)) - Handle exception(THREAD, JNIHandles::resolve(throwable)); +JVM_ENTRY(void, JVM_InitStackTraceElementArray(JNIEnv *env, jobjectArray elements, jobject backtrace, jint depth)) + Handle backtraceh(THREAD, JNIHandles::resolve(backtrace)); objArrayOop st = objArrayOop(JNIHandles::resolve(elements)); objArrayHandle stack_trace(THREAD, st); // Fill in the allocated stack trace - java_lang_Throwable::get_stack_trace_elements(exception, stack_trace, CHECK); + java_lang_Throwable::get_stack_trace_elements(depth, backtraceh, stack_trace, CHECK); JVM_END @@ -552,14 +554,15 @@ JVM_END JVM_ENTRY(jobject, JVM_CallStackWalk(JNIEnv *env, jobject stackStream, jlong mode, - jint skip_frames, jint frame_count, jint start_index, - jobjectArray frames)) + jint skip_frames, jobject contScope, jobject cont, + jint frame_count, jint start_index, jobjectArray frames)) if (!thread->has_last_Java_frame()) { THROW_MSG_(vmSymbols::java_lang_InternalError(), "doStackWalk: no stack trace", NULL); } Handle stackStream_h(THREAD, JNIHandles::resolve_non_null(stackStream)); - + Handle contScope_h(THREAD, JNIHandles::resolve(contScope)); + Handle cont_h(THREAD, JNIHandles::resolve(cont)); // frames array is a Class[] array when only getting caller reference, // and a StackFrameInfo[] array (or derivative) otherwise. It should never // be null. @@ -571,8 +574,8 @@ JVM_ENTRY(jobject, JVM_CallStackWalk(JNIEnv *env, jobject stackStream, jlong mod THROW_MSG_(vmSymbols::java_lang_IllegalArgumentException(), "not enough space in buffers", NULL); } - oop result = StackWalk::walk(stackStream_h, mode, skip_frames, frame_count, - start_index, frames_array_h, CHECK_NULL); + oop result = StackWalk::walk(stackStream_h, mode, skip_frames, contScope_h, cont_h, + frame_count, start_index, frames_array_h, CHECK_NULL); return JNIHandles::make_local(THREAD, result); JVM_END @@ -593,7 +596,16 @@ JVM_ENTRY(jint, JVM_MoreStackWalk(JNIEnv *env, jobject stackStream, jlong mode, Handle stackStream_h(THREAD, JNIHandles::resolve_non_null(stackStream)); return StackWalk::fetchNextBatch(stackStream_h, mode, anchor, frame_count, - start_index, frames_array_h, THREAD); + start_index, frames_array_h, THREAD); +JVM_END + +JVM_ENTRY(void, JVM_SetStackWalkContinuation(JNIEnv *env, jobject stackStream, jlong anchor, jobjectArray frames, jobject cont)) + objArrayOop fa = objArrayOop(JNIHandles::resolve_non_null(frames)); + objArrayHandle frames_array_h(THREAD, fa); + Handle stackStream_h(THREAD, JNIHandles::resolve_non_null(stackStream)); + Handle cont_h(THREAD, JNIHandles::resolve_non_null(cont)); + + StackWalk::setContinuation(stackStream_h, anchor, frames_array_h, cont_h, THREAD); JVM_END // java.lang.Object /////////////////////////////////////////////// @@ -694,6 +706,12 @@ JVM_LEAF(jboolean, JVM_IsFinalizationEnabled(JNIEnv * env)) return InstanceKlass::is_finalization_enabled(); JVM_END +// jdk.internal.vm.Continuation ///////////////////////////////////////////////////// + +JVM_ENTRY(void, JVM_RegisterContinuationMethods(JNIEnv *env, jclass cls)) + CONT_RegisterNativeMethods(env, cls); +JVM_END + // java.io.File /////////////////////////////////////////////////////////////// JVM_LEAF(char*, JVM_NativePath(char* path)) @@ -2956,14 +2974,7 @@ JVM_ENTRY(void, JVM_StartThread(JNIEnv* env, jobject jthread)) os::native_thread_creation_failed_msg()); } -#if INCLUDE_JFR - if (Jfr::is_recording() && EventThreadStart::is_enabled() && - EventThreadStart::is_stacktrace_enabled()) { - JfrThreadLocal* tl = native_thread->jfr_thread_local(); - // skip Thread.start() and Thread.start0() - tl->set_cached_stack_trace_id(JfrStackTraceRepository::record(thread, 2)); - } -#endif + JFR_ONLY(Jfr::on_java_thread_start(thread, native_thread);) Thread::start(native_thread); @@ -3061,13 +3072,6 @@ JVM_LEAF(void, JVM_Yield(JNIEnv *env, jclass threadClass)) os::naked_yield(); JVM_END -static void post_thread_sleep_event(EventThreadSleep* event, jlong millis) { - assert(event != NULL, "invariant"); - assert(event->should_commit(), "invariant"); - event->set_time(millis); - event->commit(); -} - JVM_ENTRY(void, JVM_Sleep(JNIEnv* env, jclass threadClass, jlong millis)) if (millis < 0) { THROW_MSG(vmSymbols::java_lang_IllegalArgumentException(), "timeout value is negative"); @@ -3082,7 +3086,6 @@ JVM_ENTRY(void, JVM_Sleep(JNIEnv* env, jclass threadClass, jlong millis)) JavaThreadSleepState jtss(thread); HOTSPOT_THREAD_SLEEP_BEGIN(millis); - EventThreadSleep event; if (millis == 0) { os::naked_yield(); @@ -3093,9 +3096,6 @@ JVM_ENTRY(void, JVM_Sleep(JNIEnv* env, jclass threadClass, jlong millis)) // An asynchronous exception (e.g., ThreadDeathException) could have been thrown on // us while we were sleeping. We do not overwrite those. if (!HAS_PENDING_EXCEPTION) { - if (event.should_commit()) { - post_thread_sleep_event(&event, millis); - } HOTSPOT_THREAD_SLEEP_END(1); // TODO-FIXME: THROW_MSG returns which means we will not call set_state() @@ -3105,18 +3105,32 @@ JVM_ENTRY(void, JVM_Sleep(JNIEnv* env, jclass threadClass, jlong millis)) } thread->osthread()->set_state(old_state); } - if (event.should_commit()) { - post_thread_sleep_event(&event, millis); - } HOTSPOT_THREAD_SLEEP_END(0); JVM_END -JVM_ENTRY(jobject, JVM_CurrentThread(JNIEnv* env, jclass threadClass)) +JVM_ENTRY(jobject, JVM_CurrentCarrierThread(JNIEnv* env, jclass threadClass)) oop jthread = thread->threadObj(); - assert(jthread != NULL, "no current thread!"); + assert(jthread != NULL, "no current carrier thread!"); return JNIHandles::make_local(THREAD, jthread); JVM_END +JVM_ENTRY(jobject, JVM_CurrentThread(JNIEnv* env, jclass threadClass)) + oop theThread = thread->vthread(); + assert(theThread != (oop)NULL, "no current thread!"); + return JNIHandles::make_local(THREAD, theThread); +JVM_END + +JVM_ENTRY(void, JVM_SetCurrentThread(JNIEnv* env, jobject thisThread, + jobject theThread)) + oop threadObj = JNIHandles::resolve(theThread); + thread->set_vthread(threadObj); + JFR_ONLY(Jfr::on_set_current_thread(thread, threadObj);) +JVM_END + +JVM_ENTRY(jlong, JVM_GetNextThreadIdOffset(JNIEnv* env, jclass threadClass)) + return ThreadIdentifier::unsafe_offset(); +JVM_END + JVM_ENTRY(void, JVM_Interrupt(JNIEnv* env, jobject jthread)) ThreadsListHandle tlh(thread); JavaThread* receiver = NULL; @@ -3127,7 +3141,6 @@ JVM_ENTRY(void, JVM_Interrupt(JNIEnv* env, jobject jthread)) } JVM_END - // Return true iff the current thread has locked the object passed in JVM_ENTRY(jboolean, JVM_HoldsLock(JNIEnv* env, jclass threadClass, jobject obj)) @@ -3138,6 +3151,10 @@ JVM_ENTRY(jboolean, JVM_HoldsLock(JNIEnv* env, jclass threadClass, jobject obj)) return ObjectSynchronizer::current_thread_holds_lock(thread, h_obj); JVM_END +JVM_ENTRY(jobject, JVM_GetStackTrace(JNIEnv *env, jobject jthread)) + oop trace = java_lang_Thread::async_get_stack_trace(JNIHandles::resolve(jthread), THREAD); + return JNIHandles::make_local(THREAD, trace); +JVM_END JVM_ENTRY(void, JVM_DumpAllStacks(JNIEnv* env, jclass)) VM_PrintThreads op; @@ -3162,6 +3179,24 @@ JVM_ENTRY(void, JVM_SetNativeThreadName(JNIEnv* env, jobject jthread, jstring na } JVM_END +JVM_ENTRY(jobject, JVM_ExtentLocalCache(JNIEnv* env, jclass threadClass)) + oop theCache = thread->extentLocalCache(); + if (theCache) { + arrayOop objs = arrayOop(theCache); + assert(objs->length() == ExtentLocalCacheSize * 2, "wrong length"); + } + return JNIHandles::make_local(THREAD, theCache); +JVM_END + +JVM_ENTRY(void, JVM_SetExtentLocalCache(JNIEnv* env, jclass threadClass, + jobject theCache)) + arrayOop objs = arrayOop(JNIHandles::resolve(theCache)); + if (objs != NULL) { + assert(objs->length() == ExtentLocalCacheSize * 2, "wrong length"); + } + thread->set_extentLocalCache(objs); +JVM_END + // java.lang.SecurityManager /////////////////////////////////////////////////////////////////////// JVM_ENTRY(jobjectArray, JVM_GetClassContext(JNIEnv *env)) @@ -3452,6 +3487,11 @@ JVM_LEAF(jboolean, JVM_IsSupportedJNIVersion(jint version)) JVM_END +JVM_LEAF(jboolean, JVM_IsPreviewEnabled(JNIEnv *env)) + return Arguments::enable_preview() ? JNI_TRUE : JNI_FALSE; +JVM_END + + // String support /////////////////////////////////////////////////////////////////////////// JVM_ENTRY(jstring, JVM_InternString(JNIEnv *env, jstring str)) @@ -3895,3 +3935,109 @@ JVM_LEAF(jint, JVM_FindSignal(const char *name)) return os::get_signal_number(name); JVM_END +JVM_ENTRY(void, JVM_VirtualThreadMountBegin(JNIEnv* env, jobject vthread, jboolean first_mount)) +#if INCLUDE_JVMTI + if (!DoJVMTIVirtualThreadTransitions) { + assert(!JvmtiExport::can_support_virtual_threads(), "sanity check"); + return; + } + JvmtiVTMSTransitionDisabler::start_VTMS_transition(vthread, /* is_mount */ true); +#else + fatal("Should only be called with JVMTI enabled"); +#endif +JVM_END + +JVM_ENTRY(void, JVM_VirtualThreadMountEnd(JNIEnv* env, jobject vthread, jboolean first_mount)) +#if INCLUDE_JVMTI + if (!DoJVMTIVirtualThreadTransitions) { + assert(!JvmtiExport::can_support_virtual_threads(), "sanity check"); + return; + } + oop vt = JNIHandles::resolve(vthread); + + thread->rebind_to_jvmti_thread_state_of(vt); + + { + MutexLocker mu(JvmtiThreadState_lock); + JvmtiThreadState* state = thread->jvmti_thread_state(); + if (state != NULL && state->is_pending_interp_only_mode()) { + JvmtiEventController::enter_interp_only_mode(); + } + } + assert(thread->is_in_VTMS_transition(), "sanity check"); + JvmtiVTMSTransitionDisabler::finish_VTMS_transition(vthread, /* is_mount */ true); + if (first_mount) { + // thread start + if (JvmtiExport::can_support_virtual_threads()) { + JvmtiEventController::thread_started(thread); + if (JvmtiExport::should_post_vthread_start()) { + JvmtiExport::post_vthread_start(vthread); + } + } else { // compatibility for vthread unaware agents: legacy thread_start + if (PostVirtualThreadCompatibleLifecycleEvents && + JvmtiExport::should_post_thread_life()) { + // JvmtiEventController::thread_started is called here + JvmtiExport::post_thread_start(thread); + } + } + } + if (JvmtiExport::should_post_vthread_mount()) { + JvmtiExport::post_vthread_mount(vthread); + } +#else + fatal("Should only be called with JVMTI enabled"); +#endif +JVM_END + +JVM_ENTRY(void, JVM_VirtualThreadUnmountBegin(JNIEnv* env, jobject vthread, jboolean last_unmount)) +#if INCLUDE_JVMTI + if (!DoJVMTIVirtualThreadTransitions) { + assert(!JvmtiExport::can_support_virtual_threads(), "sanity check"); + return; + } + HandleMark hm(thread); + Handle ct(thread, thread->threadObj()); + + if (JvmtiExport::should_post_vthread_unmount()) { + JvmtiExport::post_vthread_unmount(vthread); + } + if (last_unmount) { + if (JvmtiExport::can_support_virtual_threads()) { + if (JvmtiExport::should_post_vthread_end()) { + JvmtiExport::post_vthread_end(vthread); + } + } else { // compatibility for vthread unaware agents: legacy thread_end + if (PostVirtualThreadCompatibleLifecycleEvents && + JvmtiExport::should_post_thread_life()) { + JvmtiExport::post_thread_end(thread); + } + } + } + + assert(!thread->is_in_VTMS_transition(), "sanity check"); + JvmtiVTMSTransitionDisabler::start_VTMS_transition(vthread, /* is_mount */ false); + + if (last_unmount && thread->jvmti_thread_state() != NULL) { + JvmtiExport::cleanup_thread(thread); + thread->set_jvmti_thread_state(NULL); + oop vt = JNIHandles::resolve(vthread); + java_lang_Thread::set_jvmti_thread_state(vt, NULL); + } + thread->rebind_to_jvmti_thread_state_of(ct()); +#else + fatal("Should only be called with JVMTI enabled"); +#endif +JVM_END + +JVM_ENTRY(void, JVM_VirtualThreadUnmountEnd(JNIEnv* env, jobject vthread, jboolean last_unmount)) +#if INCLUDE_JVMTI + if (!DoJVMTIVirtualThreadTransitions) { + assert(!JvmtiExport::can_support_virtual_threads(), "sanity check"); + return; + } + assert(thread->is_in_VTMS_transition(), "sanity check"); + JvmtiVTMSTransitionDisabler::finish_VTMS_transition(vthread, /* is_mount */ false); +#else + fatal("Should only be called with JVMTI enabled"); +#endif +JVM_END diff --git a/src/hotspot/share/prims/jvmti.xml b/src/hotspot/share/prims/jvmti.xml index 4857e0a0eac..ef6a7052e47 100644 --- a/src/hotspot/share/prims/jvmti.xml +++ b/src/hotspot/share/prims/jvmti.xml @@ -134,6 +134,8 @@ + + @@ -158,6 +160,7 @@ + + These functions provide information about threads and allow an agent to + suspend and resume threads. +

+ The jthread specified to these functions can be a JNI + reference to a + platform thread + or virtual thread. + Some functions are not supported on virtual threads and return + JVMTI_ERROR_UNSUPPORTED_OPERATION when called with a reference + to a virtual thread. @@ -1222,9 +1235,9 @@ jvmtiEnv *jvmti; Thread is runnable. - Thread is waiting to enter a synchronization block/method or, + Thread is waiting to enter a synchronized block/method or, after an Object.wait(), waiting to re-enter a - synchronization block/method. + synchronized block/method. Thread is waiting. @@ -1238,7 +1251,7 @@ jvmtiEnv *jvmti; For example, Object.wait(long). - Thread is sleeping -- Thread.sleep(long). + Thread is sleeping -- Thread.sleep. Thread is waiting on an object monitor -- Object.wait. @@ -1246,6 +1259,8 @@ jvmtiEnv *jvmti; Thread is parked, for example: LockSupport.park, LockSupport.parkUtil and LockSupport.parkNanos. + A virtual thread that is sleeping, in Thread.sleep, may + have this state flag set instead of JVMTI_THREAD_STATE_SLEEPING. Thread suspended. @@ -1513,16 +1528,15 @@ jvmtiEnv *jvmti; Get All Threads - Get all live threads. - The threads are Java programming language threads; - that is, threads that are attached to the VM. + Get all live platform threads that are attached to the VM. + The list of threads includes + agent threads. + It does not include virtual threads. A thread is live if java.lang.Thread.isAlive() would return true, that is, the thread has - been started and has not yet died. + been started and has not yet terminated. The universe of threads is determined by the context of the environment, which typically is all threads attached to the VM. - Note that this includes agent threads - (see ). jvmdi @@ -1531,14 +1545,14 @@ jvmtiEnv *jvmti; - On return, points to the number of running threads. + On return, points to the number of threads. On return, points to an array of references, one - for each running thread. + for each thread. @@ -1561,7 +1575,7 @@ jvmtiEnv *jvmti; - + The thread to suspend. @@ -1594,7 +1608,7 @@ jvmtiEnv *jvmti; Java programming language threads. A thread is live if java.lang.Thread.isAlive() would return true, that is, the thread has - been started and has not yet died. + been started and has not yet terminated. The universe of threads is determined by the context of the environment, which, typically, is all threads attached to the VM, @@ -1712,6 +1726,53 @@ jvmtiEnv *jvmti; + + Suspend All Virtual Threads + + SuspendAllVirtualThreads is a preview API of the Java platform. + Preview features may be removed in a future release, or upgraded to + permanent features of the Java platform. +

+ Suspend all virtual threads except those in the exception list. + Virtual threads that are currently suspended do not change state. + Virtual threads may be resumed with + or + or + . + + new + + + + + + + + + The number of threads in the list of threads not to be suspended. + + + + + + not an error if except_count == 0 + + + The list of threads not to be suspended. + + + + + + A thread in was invalid. + + + Both was NULL + and was non-zero. + + + + Resume Thread @@ -1729,7 +1790,7 @@ jvmtiEnv *jvmti; - + The thread to resume. @@ -1793,13 +1854,57 @@ jvmtiEnv *jvmti; + + Resume All Virtual Threads + + ResumeAllVirtualThreads is a preview API of the Java platform. + Preview features may be removed in a future release, or upgraded to + permanent features of the Java platform. +

+ Resume all virtual threads except those in the exception list. + Virtual threads that are currently resumed do not change state. + Virtual threads may be suspended with + or + or + . + + new + + + + + + + + + The number of threads in the list of threads not to be resumed. + + + + + + not an error if except_count == 0 + + + The list of threads not to be resumed. + + + + + + A thread in was invalid. + + + Both was NULL + and was non-zero. + + + + Stop Thread - Send the specified asynchronous exception to the specified thread. - Normally, this function is used to kill the specified thread with an - instance of the exception ThreadDeath, similar to - java.lang.Thread.stop. + Send the specified asynchronous exception to the specified platform thread. jvmdi @@ -1807,10 +1912,12 @@ jvmtiEnv *jvmti; - + The thread to stop. - + The thread may not be a virtual thread. Otherwise, the error code + will be returned. + @@ -1820,6 +1927,9 @@ jvmtiEnv *jvmti; + + is a virtual thread. + @@ -1859,20 +1969,22 @@ jvmtiEnv *jvmti; The thread priority. See the thread priority constants: - . + . The priority of a + virtual thread is always JVMTI_THREAD_NORM_PRIORITY. - Is this a daemon thread? + Is this a daemon thread? The daemon status of a virtual thread is + always JNI_TRUE. The thread group to which this thread belongs. - NULL if the thread has died. + NULL if the thread has terminated. @@ -1919,7 +2031,7 @@ jvmtiEnv *jvmti; - + The thread to query. @@ -1973,7 +2085,7 @@ jvmtiEnv *jvmti; - + The thread to query. @@ -2009,7 +2121,7 @@ jvmtiEnv *jvmti; - + The thread to query. @@ -2099,7 +2211,7 @@ jvmtiEnv *jvmti; If enabled, a event will be sent.

Since the thread has been started, the thread will be live when this function - returns, unless the thread has died immediately. + returns, unless the thread terminated immediately.

The thread group of the thread is ignored -- specifically, the thread is not added to the thread group and the thread is not seen on queries of the thread @@ -2123,7 +2235,9 @@ jvmtiEnv *jvmti; The thread to run. - + The thread may not be a virtual thread. Otherwise, the error code + will be returned. + @@ -2158,6 +2272,9 @@ jvmtiEnv *jvmti; or greater than + + is a virtual thread. + @@ -2182,7 +2299,7 @@ jvmtiEnv *jvmti; - + Store to this thread. @@ -2323,7 +2440,8 @@ jvmtiEnv *jvmti; Get Thread Group Children - Get the live threads and active subgroups in this thread group. + Get the live platform threads and the child thread groups in this + thread group. Virtual threads are not returned by this function. jvmdi @@ -2350,13 +2468,13 @@ jvmtiEnv *jvmti; - On return, points to the number of active child thread groups + On return, points to the number of child thread groups - On return, points to an array of the active child thread groups. + On return, points to an array of the child thread groups. @@ -2484,7 +2602,7 @@ if (err == JVMTI_ERROR_NONE && count >= 1) { - + Fetch the stack trace of this thread. @@ -2543,8 +2661,11 @@ if (err == JVMTI_ERROR_NONE && count >= 1) { Get All Stack Traces - Get information about the stacks of all live threads - (including agent threads). + Get the stack traces of all live platform threads attached to the VM. + The list includes the stack traces of + agent threads. + It does not include the stack traces of virtual threads. +

If is less than the depth of a stack, the topmost frames are returned for that thread, otherwise the entire stack is returned. @@ -2768,7 +2889,7 @@ err = (*jvmti)->Deallocate(jvmti, stack_info); - + The thread to query. @@ -2840,7 +2961,7 @@ err = (*jvmti)->Deallocate(jvmti, stack_info); - + The thread whose current frame is to be popped. @@ -2851,6 +2972,10 @@ err = (*jvmti)->Deallocate(jvmti, stack_info); Called or calling method is a native method. The implementation is unable to pop this frame. + + The thread is a virtual thread and the implementation is unable + to pop this frame. + Thread was not suspended and was not the current thread. @@ -2872,7 +2997,7 @@ err = (*jvmti)->Deallocate(jvmti, stack_info); - + The thread of the frame to query. @@ -2921,7 +3046,7 @@ err = (*jvmti)->Deallocate(jvmti, stack_info); - + The thread of the frame for which the frame pop event will be generated. @@ -2996,7 +3121,7 @@ err = (*jvmti)->Deallocate(jvmti, stack_info); - + The thread whose current frame is to return early. @@ -3013,6 +3138,8 @@ err = (*jvmti)->Deallocate(jvmti, stack_info); Attempted to return early from a frame corresponding to a native method. + The thread is a virtual thread and the implementation is + unable to force its current frame to return. Or the implementation is unable to provide this functionality on this frame. @@ -3047,7 +3174,7 @@ err = (*jvmti)->Deallocate(jvmti, stack_info); - + The thread whose current frame is to return early. @@ -3063,6 +3190,8 @@ err = (*jvmti)->Deallocate(jvmti, stack_info); Attempted to return early from a frame corresponding to a native method. + The thread is a virtual thread and the implementation is + unable to force its current frame to return. Or the implementation is unable to provide this functionality on this frame. @@ -3093,7 +3222,7 @@ err = (*jvmti)->Deallocate(jvmti, stack_info); - + The thread whose current frame is to return early. @@ -3109,6 +3238,8 @@ err = (*jvmti)->Deallocate(jvmti, stack_info); Attempted to return early from a frame corresponding to a native method. + The thread is a virtual thread and the implementation is + unable to force its current frame to return. Or the implementation is unable to provide this functionality on this frame. @@ -3136,7 +3267,7 @@ err = (*jvmti)->Deallocate(jvmti, stack_info); - + The thread whose current frame is to return early. @@ -3152,6 +3283,8 @@ err = (*jvmti)->Deallocate(jvmti, stack_info); Attempted to return early from a frame corresponding to a native method. + The thread is a virtual thread and the implementation is + unable to force its current frame to return. Or the implementation is unable to provide this functionality on this frame. @@ -3179,7 +3312,7 @@ err = (*jvmti)->Deallocate(jvmti, stack_info); - + The thread whose current frame is to return early. @@ -3193,8 +3326,12 @@ err = (*jvmti)->Deallocate(jvmti, stack_info); - Attempted to return early from a frame corresponding to a native method. - Or the implementation is unable to provide this functionality on this frame. + Attempted to return early from a frame + corresponding to a native method. + The thread is a virtual thread and the implementation is + unable to force its current frame to return. + Or the implementation is unable to provide + this functionality on this frame. The result type of the called method is not double. @@ -3220,7 +3357,7 @@ err = (*jvmti)->Deallocate(jvmti, stack_info); - + The thread whose current frame is to return early. @@ -3230,6 +3367,8 @@ err = (*jvmti)->Deallocate(jvmti, stack_info); Attempted to return early from a frame corresponding to a native method. + The thread is a virtual thread and the implementation is + unable to force its current frame to return. Or the implementation is unable to provide this functionality on this frame. @@ -4963,9 +5102,9 @@ class C2 extends C1 implements I2 { These functions and data types were introduced in the original - version 1.0. They are deprecated and will be changed + version 1.0. They are deprecated and will be changed to return an error in a future release. They were superseded in - version 1.2 (Java SE 6) by more + version 1.2 (Java SE 6) by more powerful and flexible versions @@ -5682,6 +5821,13 @@ class C2 extends C1 implements I2 { The mapping of variables to slot numbers can be obtained with the function . +

+ The GetLocalXXX functions may be used to retrieve the value of + a local variable contained in the frame of a virtual thread. + The SetLocalXXX functions may be used to set the value of a + local variable in the topmost frame of a virtual thread suspended at a + breakpoint or single step event. An implementation may support setting locals + in other cases. @@ -5696,7 +5842,7 @@ class C2 extends C1 implements I2 { - + The thread of the frame containing the variable's value. @@ -5749,7 +5895,7 @@ class C2 extends C1 implements I2 { - + The thread of the frame containing the variable's value. @@ -5787,7 +5933,7 @@ class C2 extends C1 implements I2 { - + The thread of the frame containing the variable's value. @@ -5839,7 +5985,7 @@ class C2 extends C1 implements I2 { - + The thread of the frame containing the variable's value. @@ -5888,7 +6034,7 @@ class C2 extends C1 implements I2 { - + The thread of the frame containing the variable's value. @@ -5937,7 +6083,7 @@ class C2 extends C1 implements I2 { - + The thread of the frame containing the variable's value. @@ -5986,7 +6132,7 @@ class C2 extends C1 implements I2 { - + The thread of the frame containing the variable's value. @@ -6025,6 +6171,11 @@ class C2 extends C1 implements I2 { Not a visible frame + + The thread is a virtual thread and the implementation does not support + setting the value of locals in the frame of the given depth. + See Local Variables. + @@ -6042,7 +6193,7 @@ class C2 extends C1 implements I2 { - + The thread of the frame containing the variable's value. @@ -6079,6 +6230,11 @@ class C2 extends C1 implements I2 { Not a visible frame + + The thread is a virtual thread and the implementation does not support + setting the value of locals in the frame of the given depth. + See Local Variables. + @@ -6094,7 +6250,7 @@ class C2 extends C1 implements I2 { - + The thread of the frame containing the variable's value. @@ -6128,6 +6284,11 @@ class C2 extends C1 implements I2 { Not a visible frame + + The thread is a virtual thread and the implementation does not support + setting the value of locals in the frame of the given depth. + See Local Variables. + @@ -6143,7 +6304,7 @@ class C2 extends C1 implements I2 { - + The thread of the frame containing the variable's value. @@ -6177,6 +6338,11 @@ class C2 extends C1 implements I2 { Not a visible frame + + The thread is a virtual thread and the implementation does not support + setting the value of locals in the frame of the given depth. + See Local Variables. + @@ -6192,7 +6358,7 @@ class C2 extends C1 implements I2 { - + The thread of the frame containing the variable's value. @@ -6226,6 +6392,11 @@ class C2 extends C1 implements I2 { Not a visible frame + + The thread is a virtual thread and the implementation does not support + setting the value of locals in the frame of the given depth. + See Local Variables. + @@ -9534,7 +9705,8 @@ myInit() { is non-NULL and is not a valid thread. - is non-NULL and is not live (has not been started or is now dead). + is non-NULL and is not alive + (has not been started or has terminated). thread level control was attempted on events which do not @@ -10427,6 +10599,21 @@ myInit() { called and events can be generated. + + + can_support_virtual_threads is a preview API of the Java platform. + Preview features may be removed in a future release, or upgraded to + permanent features of the Java platform. +

+ Can support virtual threads. + If this capability is enabled then the following functions can be called: + , + , + and the following events can be enabled: + , + , + + @@ -10784,6 +10971,8 @@ myInit() { the current thread (see vs ). + The current thread may not be a virtual thread. Otherwise, the error code + will be returned. On many platforms this call will be equivalent to: GetThreadCpuTime(env, NULL, nanos_ptr) @@ -10817,6 +11006,9 @@ myInit() { + + Current thread is a virtual thread. + @@ -10878,13 +11070,15 @@ myInit() { - + The thread to query. - + The thread may not be a virtual thread. Otherwise, the error code + will be returned. + - + On return, points to the CPU time used by the specified thread in nanoseconds. @@ -10894,6 +11088,9 @@ myInit() { + + is a virtual thread. + @@ -11725,7 +11922,7 @@ myInit() { This operation requires the thread to be alive--that is, - it must be started and not yet have died. + it must be started and not yet terminated. The class has been loaded but not yet prepared. @@ -11863,6 +12060,9 @@ myInit() { A new class version has unsupported differences in class attributes. + + Functionality is unsupported in this implementation. + @@ -12691,8 +12891,14 @@ myInit() { - Thread start events are generated by a new thread before its initial - method executes. + A thread start event is generated by a new thread before its initial + method executes. The capability + + can_support_virtual_threads determines + if a new virtual thread generates a ThreadStart event or + a event. If disabled, a virtual + thread generates a ThreadStart event. If enabled, a virtual + thread generates a VirtualThreadStart event.

A thread may be listed in the array returned by @@ -12726,8 +12932,15 @@ myInit() { - Thread end events are generated by a terminating thread - after its initial method has finished execution. + A thread end event is generated by a terminating thread after its + initial method has finished execution. + The capability + + can_support_virtual_threads determines + if a terminating virtual thread generates a ThreadEnd + event or a event. If disabled, a + virtual thread generates a ThreadEnd event. If enabled, + a virtual thread generates a VirtualThreadEnd event.

A thread may be listed in the array returned by @@ -12758,6 +12971,134 @@ myInit() { + + + VirtualThreadStart is a preview API of the Java platform. + Preview features may be removed in a future release, or upgraded to + permanent features of the Java platform. +

+ A virtual thread start event is generated before its initial method executes. +

+ The event is sent on the . + + new + + Can support virtual threads + + + + + JNIEnv + + + The JNI environment of the event (current) thread. + + + + + + Virtual thread started for execution. + + + + + + + + VirtualThreadEnd is a preview API of the Java platform. + Preview features may be removed in a future release, or upgraded to + permanent features of the Java platform. +

+ A virtual thread end event is generated after its initial method has finished execution. +

+ The event is sent on the . + + new + + Can support virtual threads + + + + + JNIEnv + + + The JNI environment of the event (current) thread. + + + + + + Virtual thread being ended. + + + + + + + + + A virtual thread mount event is generated before its method continue to execute on the mounted thread. +

+ The event is sent on the the virtual thread is mounted to. + + new + + Can support virtual threads + + + + + JNIEnv + + + The JNI environment of the event (current) thread. + + + + + + Virtual thread that is mounted. + + + + + + + + + + A virtual thread unmount event is generated when the virtual thread is about to be unmounted from the carrier thread. +

+ The event is sent on the the virtual thread is unmounted from. + + new + + Can support virtual threads + + + + + JNIEnv + + + The JNI environment of the event (current) thread. + + + + + + Virtual thread that is unmounted. + + + + + + @@ -15015,6 +15356,15 @@ typedef void (JNICALL *jvmtiEventVMInit) - Add new error JVMTI_ERROR_UNSUPPORTED_REDEFINITION_CLASS_ATTRIBUTE_CHANGED that can be returned by RedefineClasses and RetransformClasses. + + Support for Low Overhead Heap Sampling: + - Add new capability: + - can_generate_sampled_object_alloc_events + - Add new function: + - SetHeapSamplingInterval + - Add new event type: + - JVMTI_EVENT_SAMPLED_OBJECT_ALLOC + Minor spec update for the capability "can_redefine_any_class". It now says: @@ -15039,11 +15389,24 @@ typedef void (JNICALL *jvmtiEventVMInit) Minor clarification in the section "Agent Shutdown" that says the implementation may choose to not call the Agent_OnUnload function - if the Agent_OnAttach/Agent_OnAttach_L function reported an error. + if the Agent_OnAttach/Agent_OnAttach_L function reported an error. Minor update to deprecate Heap functions 1.0. + + Support for virtual threads: + - Add new capability: + - can_support_virtual_threads + - Add new functions: + - SuspendAllVirtualThreads + - ResumeAllVirtualThreads + - Add new event types: + - JVMTI_EVENT_VIRTUAL_THREAD_START + - JVMTI_EVENT_VIRTUAL_THREAD_END + - Add new error code: + - JVMTI_ERROR_UNSUPPORTED_OPERATION + diff --git a/src/hotspot/share/prims/jvmti.xsl b/src/hotspot/share/prims/jvmti.xsl index c0673327ab0..7f64283ebbe 100644 --- a/src/hotspot/share/prims/jvmti.xsl +++ b/src/hotspot/share/prims/jvmti.xsl @@ -1,6 +1,6 @@