8209961: [AOT] crash in Graal stub when -XX:+VerifyOops is used

Reviewed-by: kvn

make/RunTests.gmk

@@ -172,6 +172,10 @@ define SetupAotModuleBody
     $1_JAOTC_OPTS += --compile-with-assertions
   endif
 
+  ifneq ($$(filter -XX:+VerifyOops, $$($1_VM_OPTIONS)), )
+    $1_JAOTC_OPTS += -J-Dgraal.AOTVerifyOops=true
+  endif
+
   $$($1_AOT_LIB): $$(JDK_UNDER_TEST)/release \
       $$(call DependOnVariable, $1_JAOTC_OPTS) \
       $$(call DependOnVariable, JDK_UNDER_TEST)

src/hotspot/share/aot/aotCodeHeap.cpp

@@ -566,6 +566,10 @@ void AOTCodeHeap::link_stub_routines_symbols() {
 
     SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_stub_routines_throw_delayed_StackOverflowError_entry", address, StubRoutines::_throw_delayed_StackOverflowError_entry);
 
+    SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_verify_oops", intptr_t, VerifyOops);
+    SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_verify_oop_count_address", jint *, &StubRoutines::_verify_oop_count);
+    SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_verify_oop_bits", intptr_t, Universe::verify_oop_bits());
+    SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_verify_oop_mask", intptr_t, Universe::verify_oop_mask());
 }
 
 void AOTCodeHeap::link_os_symbols() {

src/hotspot/share/jvmci/jvmciCodeInstaller.cpp

@@ -1386,6 +1386,10 @@ void CodeInstaller::site_Mark(CodeBuffer& buffer, jint pc_offset, JVMCIObject si
       case CRC_TABLE_ADDRESS:
       case LOG_OF_HEAP_REGION_GRAIN_BYTES:
       case INLINE_CONTIGUOUS_ALLOCATION_SUPPORTED:
+      case VERIFY_OOPS:
+      case VERIFY_OOP_BITS:
+      case VERIFY_OOP_MASK:
+      case VERIFY_OOP_COUNT_ADDRESS:
         break;
       default:
         JVMCI_ERROR("invalid mark id: %d", id);

src/hotspot/share/jvmci/jvmciCodeInstaller.hpp

@@ -160,6 +160,10 @@ private:
     LOG_OF_HEAP_REGION_GRAIN_BYTES,
     INLINE_CONTIGUOUS_ALLOCATION_SUPPORTED,
     DEOPT_MH_HANDLER_ENTRY,
+    VERIFY_OOPS,
+    VERIFY_OOP_BITS,
+    VERIFY_OOP_MASK,
+    VERIFY_OOP_COUNT_ADDRESS,
     INVOKE_INVALID                         = -1
   };
 

src/hotspot/share/jvmci/vmStructs_jvmci.cpp

@@ -472,6 +472,10 @@
   declare_constant(CodeInstaller::LOG_OF_HEAP_REGION_GRAIN_BYTES)         \
   declare_constant(CodeInstaller::INLINE_CONTIGUOUS_ALLOCATION_SUPPORTED) \
   declare_constant(CodeInstaller::DEOPT_MH_HANDLER_ENTRY)                 \
+  declare_constant(CodeInstaller::VERIFY_OOP_COUNT_ADDRESS)               \
+  declare_constant(CodeInstaller::VERIFY_OOPS)                            \
+  declare_constant(CodeInstaller::VERIFY_OOP_BITS)                        \
+  declare_constant(CodeInstaller::VERIFY_OOP_MASK)                        \
   declare_constant(CodeInstaller::INVOKE_INVALID)                         \
                                                                           \
   declare_constant(vmIntrinsics::FIRST_MH_SIG_POLY)                       \

src/jdk.aot/share/classes/jdk.tools.jaotc.binformat/src/jdk/tools/jaotc/binformat/BinaryContainer.java

@@ -464,6 +464,22 @@ public final class BinaryContainer implements SymbolTable {
         return "_aot_inline_contiguous_allocation_supported";
     }
 
+    public static String getVerifyOopsSymbolName() {
+        return "_aot_verify_oops";
+    }
+
+    public static String getVerifyOopCountAddressSymbolName() {
+        return "_aot_verify_oop_count_address";
+    }
+
+    public static String getVerifyOopBitsSymbolName() {
+        return "_aot_verify_oop_bits";
+    }
+
+    public static String getVerifyOopMaskSymbolName() {
+        return "_aot_verify_oop_mask";
+    }
+
     public int getCodeSegmentSize() {
         return codeSegmentSize;
     }
@@ -515,6 +531,10 @@ public final class BinaryContainer implements SymbolTable {
         createGotSymbol(getPollingPageSymbolName());
         createGotSymbol(getLogOfHeapRegionGrainBytesSymbolName());
         createGotSymbol(getInlineContiguousAllocationSupportedSymbolName());
+        createGotSymbol(getVerifyOopsSymbolName());
+        createGotSymbol(getVerifyOopCountAddressSymbolName());
+        createGotSymbol(getVerifyOopBitsSymbolName());
+        createGotSymbol(getVerifyOopMaskSymbolName());
 
         for (HashMap.Entry<String, String> entry : functionNamesToAOTSymbols.entrySet()) {
             createGotSymbol(entry.getValue());

src/jdk.aot/share/classes/jdk.tools.jaotc/src/jdk/tools/jaotc/Main.java

@@ -54,6 +54,7 @@ import org.graalvm.compiler.hotspot.HotSpotGraalOptionValues;
 import org.graalvm.compiler.hotspot.HotSpotGraalRuntime;
 import org.graalvm.compiler.hotspot.HotSpotGraalRuntime.HotSpotGC;
 import org.graalvm.compiler.hotspot.HotSpotHostBackend;
+import org.graalvm.compiler.hotspot.HotSpotMarkId;
 import org.graalvm.compiler.hotspot.meta.HotSpotInvokeDynamicPlugin;
 import org.graalvm.compiler.java.GraphBuilderPhase;
 import org.graalvm.compiler.nodes.graphbuilderconf.GraphBuilderConfiguration;
@@ -177,6 +178,15 @@ public final class Main {
             graalOptions = new OptionValues(graalOptions, GeneratePIC, true, ImmutableCode, true);
             GraalJVMCICompiler graalCompiler = HotSpotGraalCompilerFactory.createCompiler("JAOTC", JVMCI.getRuntime(), graalOptions, CompilerConfigurationFactory.selectFactory(null, graalOptions));
             HotSpotGraalRuntime runtime = (HotSpotGraalRuntime) graalCompiler.getGraalRuntime();
+            GraalHotSpotVMConfig graalHotSpotVMConfig = runtime.getVMConfig();
+
+            if (graalHotSpotVMConfig.verifyOops) {
+                if (!HotSpotMarkId.VERIFY_OOPS.isAvailable() || !HotSpotMarkId.VERIFY_OOP_COUNT_ADDRESS.isAvailable()) {
+                    System.err.println("Running jaotc with -XX:+VerifyOops is not supported by this JDK");
+                    return false;
+                }
+            }
+
             HotSpotHostBackend backend = (HotSpotHostBackend) runtime.getCapability(RuntimeProvider.class).getHostBackend();
             MetaAccessProvider metaAccess = backend.getProviders().getMetaAccess();
             filters = new GraalFilters(metaAccess);
@@ -207,12 +217,12 @@ public final class Main {
                     return false;
                 }
             };
+
             AOTBackend aotBackend = new AOTBackend(this, graalOptions, backend, indyPlugin);
             SnippetReflectionProvider snippetReflection = aotBackend.getProviders().getSnippetReflection();
             AOTCompiler compiler = new AOTCompiler(this, graalOptions, aotBackend, options.threads);
             classes = compiler.compileClasses(classes);
 
-            GraalHotSpotVMConfig graalHotSpotVMConfig = runtime.getVMConfig();
             PhaseSuite<HighTierContext> graphBuilderSuite = aotBackend.getGraphBuilderSuite();
             ListIterator<BasePhase<? super HighTierContext>> iterator = graphBuilderSuite.findPhase(GraphBuilderPhase.class);
             GraphBuilderConfiguration graphBuilderConfig = ((GraphBuilderPhase) iterator.previous()).getGraphBuilderConfig();

src/jdk.aot/share/classes/jdk.tools.jaotc/src/jdk/tools/jaotc/MarkProcessor.java

@@ -69,6 +69,10 @@ final class MarkProcessor {
             case NARROW_OOP_BASE_ADDRESS:
             case CRC_TABLE_ADDRESS:
             case LOG_OF_HEAP_REGION_GRAIN_BYTES:
+            case VERIFY_OOPS:
+            case VERIFY_OOP_BITS:
+            case VERIFY_OOP_MASK:
+            case VERIFY_OOP_COUNT_ADDRESS:
                 String vmSymbolName;
                 switch (markId) {
                     case POLL_FAR:
@@ -90,6 +94,18 @@ final class MarkProcessor {
                     case LOG_OF_HEAP_REGION_GRAIN_BYTES:
                         vmSymbolName = BinaryContainer.getLogOfHeapRegionGrainBytesSymbolName();
                         break;
+                    case VERIFY_OOPS:
+                        vmSymbolName = BinaryContainer.getVerifyOopsSymbolName();
+                        break;
+                    case VERIFY_OOP_COUNT_ADDRESS:
+                        vmSymbolName = BinaryContainer.getVerifyOopCountAddressSymbolName();
+                        break;
+                    case VERIFY_OOP_BITS:
+                        vmSymbolName = BinaryContainer.getVerifyOopBitsSymbolName();
+                        break;
+                    case VERIFY_OOP_MASK:
+                        vmSymbolName = BinaryContainer.getVerifyOopMaskSymbolName();
+                        break;
                     default:
                         throw new InternalError("Unhandled mark: " + mark);
                 }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/GraalOptions.java

@@ -198,6 +198,9 @@ public final class GraalOptions {
     @Option(help = "Generate position independent code", type = OptionType.Expert)
     public static final OptionKey<Boolean> GeneratePIC = new OptionKey<>(false);
 
+    @Option(help = "Generate verify oop checks in AOT code", type = OptionType.Expert)
+    public static final OptionKey<Boolean> AOTVerifyOops = new OptionKey<>(false);
+
     // Runtime settings
     @Option(help = "", type = OptionType.Expert)
     public static final OptionKey<Boolean> SupportJsrBytecodes = new OptionKey<>(true);

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/GraalHotSpotVMConfig.java

@@ -27,6 +27,7 @@ package org.graalvm.compiler.hotspot;
 import java.lang.reflect.Field;
 import java.lang.reflect.Modifier;
 import java.util.List;
+import java.util.Map;
 
 import org.graalvm.compiler.api.replacements.Fold;
 import org.graalvm.compiler.api.replacements.Fold.InjectedParameter;
@@ -64,6 +65,7 @@ public class GraalHotSpotVMConfig extends GraalHotSpotVMConfigAccess {
 
         assert check();
         reportErrors();
+        populateMarkConstants();
     }
 
     private final CompressEncoding oopEncoding;
@@ -893,25 +895,6 @@ public class GraalHotSpotVMConfig extends GraalHotSpotVMConfigAccess {
     private static final boolean JDK_8245443 = ((JDK == 11 && JDK_UPDATE >= 8) || JDK >= 15);
 
     // Checkstyle: stop
-    public final int VERIFIED_ENTRY = getConstant("CodeInstaller::VERIFIED_ENTRY", Integer.class);
-    public final int UNVERIFIED_ENTRY = getConstant("CodeInstaller::UNVERIFIED_ENTRY", Integer.class);
-    public final int OSR_ENTRY = getConstant("CodeInstaller::OSR_ENTRY", Integer.class);
-    public final int EXCEPTION_HANDLER_ENTRY = getConstant("CodeInstaller::EXCEPTION_HANDLER_ENTRY", Integer.class);
-    public final int DEOPT_HANDLER_ENTRY = getConstant("CodeInstaller::DEOPT_HANDLER_ENTRY", Integer.class);
-    public final int DEOPT_MH_HANDLER_ENTRY = getConstant("CodeInstaller::DEOPT_MH_HANDLER_ENTRY", Integer.class, -1, (JVMCI ? jvmciGE(JVMCI_20_2_b01) : JDK >= 16));
-    public final int FRAME_COMPLETE = getConstant("CodeInstaller::FRAME_COMPLETE", Integer.class, -1, (JVMCI ? jvmciGE(JVMCI_20_1_b01) : JDK_8245443));
-    public final int INVOKEINTERFACE = getConstant("CodeInstaller::INVOKEINTERFACE", Integer.class);
-    public final int INVOKEVIRTUAL = getConstant("CodeInstaller::INVOKEVIRTUAL", Integer.class);
-    public final int INVOKESTATIC = getConstant("CodeInstaller::INVOKESTATIC", Integer.class);
-    public final int INVOKESPECIAL = getConstant("CodeInstaller::INVOKESPECIAL", Integer.class);
-    public final int INLINE_INVOKE = getConstant("CodeInstaller::INLINE_INVOKE", Integer.class);
-    public final int POLL_NEAR = getConstant("CodeInstaller::POLL_NEAR", Integer.class);
-    public final int POLL_RETURN_NEAR = getConstant("CodeInstaller::POLL_RETURN_NEAR", Integer.class);
-    public final int POLL_FAR = getConstant("CodeInstaller::POLL_FAR", Integer.class);
-    public final int POLL_RETURN_FAR = getConstant("CodeInstaller::POLL_RETURN_FAR", Integer.class);
-    public final int CARD_TABLE_SHIFT = getConstant("CodeInstaller::CARD_TABLE_SHIFT", Integer.class);
-    public final int CARD_TABLE_ADDRESS = getConstant("CodeInstaller::CARD_TABLE_ADDRESS", Integer.class);
-    public final int INVOKE_INVALID = getConstant("CodeInstaller::INVOKE_INVALID", Integer.class);
     public final int VMINTRINSIC_FIRST_MH_SIG_POLY = getConstant("vmIntrinsics::FIRST_MH_SIG_POLY", Integer.class, -1, (JVMCI ? jvmciGE(JVMCI_20_2_b01) : JDK >= 16));
     public final int VMINTRINSIC_LAST_MH_SIG_POLY = getConstant("vmIntrinsics::LAST_MH_SIG_POLY", Integer.class, -1, (JVMCI ? jvmciGE(JVMCI_20_2_b01) : JDK >= 16));
     public final int VMINTRINSIC_INVOKE_GENERIC = getConstant("vmIntrinsics::_invokeGeneric", Integer.class, -1, (JVMCI ? jvmciGE(JVMCI_20_2_b01) : JDK >= 16));
@@ -920,17 +903,51 @@ public class GraalHotSpotVMConfig extends GraalHotSpotVMConfigAccess {
     public final boolean CPU_HAS_INTEL_JCC_ERRATUM = getFieldValue("VM_Version::_has_intel_jcc_erratum", Boolean.class, "bool",
                     true, "amd64".equals(osArch) && (JVMCI ? jvmciGE(JVMCI_20_1_b01) : JDK >= 15));
 
-    /**
-     * The following constants are given default values here since they are missing in the native
-     * JVMCI-8 code but are still required for {@link HotSpotMarkId} to work in a JDK8 environment.
-     */
-    public final int NARROW_KLASS_BASE_ADDRESS = getConstant("CodeInstaller::NARROW_KLASS_BASE_ADDRESS", Integer.class, 19, JDK > 9);
-    public final int NARROW_OOP_BASE_ADDRESS = getConstant("CodeInstaller::NARROW_OOP_BASE_ADDRESS", Integer.class, 20, JDK > 9);
-    public final int CRC_TABLE_ADDRESS = getConstant("CodeInstaller::CRC_TABLE_ADDRESS", Integer.class, 21, JDK > 9);
-    public final int LOG_OF_HEAP_REGION_GRAIN_BYTES = getConstant("CodeInstaller::LOG_OF_HEAP_REGION_GRAIN_BYTES", Integer.class, 22, JDK > 9);
-
     // Checkstyle: resume
 
+    private static void checkForMissingRequiredValue(HotSpotMarkId markId, boolean required) {
+        if (!markId.isAvailable() && required) {
+            GraalHotSpotVMConfigAccess.reportError("Unsupported Mark " + markId);
+        }
+    }
+
+    private void populateMarkConstants() {
+        boolean jdk13JvmciBackport = (JVMCI && JDK > 8) ? jvmciGE(JVMCI_19_3_b03) : JDK > 9;
+        boolean verifyOopsMarkSupported = JDK >= 16;
+        Map<String, Long> constants = getStore().getConstants();
+        for (HotSpotMarkId markId : HotSpotMarkId.values()) {
+            Integer value = null;
+            String key = "CodeInstaller::" + markId.name();
+            Long result = constants.get(key);
+            if (result != null) {
+                value = result.intValue();
+            }
+            markId.setValue(value);
+            switch (markId) {
+                case FRAME_COMPLETE:
+                    checkForMissingRequiredValue(markId, JVMCI ? jvmciGE(JVMCI_20_1_b01) : JDK_8245443);
+                    break;
+                case DEOPT_MH_HANDLER_ENTRY:
+                    checkForMissingRequiredValue(markId, JVMCI ? jvmciGE(JVMCI_20_2_b01) : JDK >= 16);
+                    break;
+                case NARROW_KLASS_BASE_ADDRESS:
+                case CRC_TABLE_ADDRESS:
+                case NARROW_OOP_BASE_ADDRESS:
+                case LOG_OF_HEAP_REGION_GRAIN_BYTES:
+                    checkForMissingRequiredValue(markId, jdk13JvmciBackport);
+                    break;
+                case VERIFY_OOPS:
+                case VERIFY_OOP_BITS:
+                case VERIFY_OOP_MASK:
+                case VERIFY_OOP_COUNT_ADDRESS:
+                    checkForMissingRequiredValue(markId, verifyOopsMarkSupported);
+                    break;
+                default:
+                    checkForMissingRequiredValue(markId, true);
+            }
+        }
+    }
+
     protected boolean check() {
         for (Field f : getClass().getDeclaredFields()) {
             int modifiers = f.getModifiers();
@@ -952,7 +969,7 @@ public class GraalHotSpotVMConfig extends GraalHotSpotVMConfigAccess {
     }
 
     public boolean supportsMethodHandleDeoptimizationEntry() {
-        return DEOPT_MH_HANDLER_ENTRY != -1 && VMINTRINSIC_FIRST_MH_SIG_POLY != -1 && VMINTRINSIC_LAST_MH_SIG_POLY != -1 && VMINTRINSIC_INVOKE_GENERIC != -1 &&
+        return HotSpotMarkId.DEOPT_MH_HANDLER_ENTRY.isAvailable() && VMINTRINSIC_FIRST_MH_SIG_POLY != -1 && VMINTRINSIC_LAST_MH_SIG_POLY != -1 && VMINTRINSIC_INVOKE_GENERIC != -1 &&
                         VMINTRINSIC_COMPILED_LAMBDA_FORM != -1;
     }
 

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/GraalHotSpotVMConfigAccess.java

@@ -26,6 +26,7 @@ package org.graalvm.compiler.hotspot;
 
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Formatter;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -127,6 +128,7 @@ public class GraalHotSpotVMConfigAccess {
     public final String osArch;
 
     protected static final Version JVMCI_0_55 = new Version2(0, 55);
+    protected static final Version JVMCI_20_2_b04 = new Version3(20, 2, 4);
     protected static final Version JVMCI_20_2_b01 = new Version3(20, 2, 1);
     protected static final Version JVMCI_20_1_b01 = new Version3(20, 1, 1);
     protected static final Version JVMCI_20_0_b03 = new Version3(20, 0, 3);
@@ -135,7 +137,7 @@ public class GraalHotSpotVMConfigAccess {
     protected static final Version JVMCI_19_3_b07 = new Version3(19, 3, 7);
 
     public static boolean jvmciGE(Version v) {
-        return JVMCI_PRERELEASE || !JVMCI_VERSION.isLessThan(v);
+        return !JVMCI_VERSION.isLessThan(v);
     }
 
     public static final int JDK = JavaVersionUtil.JAVA_SPEC;
@@ -143,12 +145,12 @@ public class GraalHotSpotVMConfigAccess {
     public static final boolean IS_OPENJDK = getProperty("java.vm.name", "").startsWith("OpenJDK");
     public static final Version JVMCI_VERSION;
     public static final boolean JVMCI;
-    public static final boolean JVMCI_PRERELEASE;
+    public static final boolean JDK_PRERELEASE;
     static {
         String vmVersion = getProperty("java.vm.version");
         JVMCI_VERSION = Version.parse(vmVersion);
-        JVMCI_PRERELEASE = vmVersion.contains("SNAPSHOT") || vmVersion.contains("internal") || vmVersion.contains("-dev");
+        JDK_PRERELEASE = vmVersion.contains("SNAPSHOT") || vmVersion.contains("-dev");
-        JVMCI = JVMCI_VERSION != null || JVMCI_PRERELEASE;
+        JVMCI = JVMCI_VERSION != null;
     }
 
     private final List<String> missing = new ArrayList<>();
@@ -179,7 +181,7 @@ public class GraalHotSpotVMConfigAccess {
     private boolean deferErrors = this instanceof GraalHotSpotVMConfig;
 
     private void recordError(String name, List<String> list, String unexpectedValue) {
-        if (JVMCI_PRERELEASE) {
+        if (JDK_PRERELEASE) {
             return;
         }
         String message = name;
@@ -224,7 +226,25 @@ public class GraalHotSpotVMConfigAccess {
                 messages.add(String.format("VM config values not expected to be present in %s:%n    %s", runtime,
                                 unexpected.stream().sorted().collect(Collectors.joining(System.lineSeparator() + "    "))));
             }
-            throw new JVMCIError(String.join(System.lineSeparator(), messages));
+            reportError(String.join(System.lineSeparator(), messages));
+        }
+    }
+
+    static void reportError(String rawErrorMessage) {
+        String value = getProperty("JVMCI_CONFIG_CHECK");
+        Formatter errorMessage = new Formatter().format(rawErrorMessage);
+        String javaHome = getProperty("java.home");
+        String vmName = getProperty("java.vm.name");
+        errorMessage.format("%nSet the JVMCI_CONFIG_CHECK system property to \"ignore\" to suppress ");
+        errorMessage.format("this error or to \"warn\" to emit a warning and continue execution.%n");
+        errorMessage.format("Currently used Java home directory is %s.%n", javaHome);
+        errorMessage.format("Currently used VM configuration is: %s%n", vmName);
+        if ("ignore".equals(value)) {
+            return;
+        } else if ("warn".equals(value) || JDK_PRERELEASE) {
+            System.err.println(errorMessage.toString());
+        } else {
+            throw new JVMCIError(errorMessage.toString());
         }
     }
 

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotMarkId.java

@@ -25,12 +25,9 @@
 
 package org.graalvm.compiler.hotspot;
 
-import static org.graalvm.compiler.debug.GraalError.shouldNotReachHere;
-
 import org.graalvm.compiler.code.CompilationResult;
 
 import jdk.vm.ci.common.NativeImageReinitialize;
-import jdk.vm.ci.hotspot.HotSpotJVMCIRuntime;
 
 /**
  * Constants used to mark special positions in code being installed into the code cache by Graal C++
@@ -43,7 +40,7 @@ public enum HotSpotMarkId implements CompilationResult.MarkId {
     EXCEPTION_HANDLER_ENTRY(false),
     DEOPT_HANDLER_ENTRY(false),
     DEOPT_MH_HANDLER_ENTRY(false),
-    FRAME_COMPLETE(true, true),
+    FRAME_COMPLETE(true),
     INVOKEINTERFACE(false),
     INVOKEVIRTUAL(false),
     INVOKESTATIC(false),
@@ -57,31 +54,22 @@ public enum HotSpotMarkId implements CompilationResult.MarkId {
     NARROW_KLASS_BASE_ADDRESS(true),
     NARROW_OOP_BASE_ADDRESS(true),
     CRC_TABLE_ADDRESS(true),
-    LOG_OF_HEAP_REGION_GRAIN_BYTES(true);
+    LOG_OF_HEAP_REGION_GRAIN_BYTES(true),
+    VERIFY_OOPS(true),
+    VERIFY_OOP_BITS(true),
+    VERIFY_OOP_MASK(true),
+    VERIFY_OOP_COUNT_ADDRESS(true);
 
     private final boolean isMarkAfter;
     @NativeImageReinitialize private Integer value;
-    private final boolean optional;
 
     HotSpotMarkId(boolean isMarkAfter) {
-        this(isMarkAfter, false);
-    }
-
-    HotSpotMarkId(boolean isMarkAfter, boolean optional) {
         this.isMarkAfter = isMarkAfter;
-        this.optional = optional;
+        this.value = null;
     }
 
-    private Integer getValue() {
+    void setValue(Integer value) {
-        if (value == null) {
+        this.value = value;
-            Long result = HotSpotJVMCIRuntime.runtime().getConfigStore().getConstants().get("CodeInstaller::" + name());
-            if (result != null) {
-                this.value = result.intValue();
-            } else if (!optional) {
-                throw shouldNotReachHere("Unsupported Mark " + name());
-            }
-        }
-        return value;
     }
 
     @Override
@@ -92,7 +80,7 @@ public enum HotSpotMarkId implements CompilationResult.MarkId {
     @Override
     public Object getId() {
         assert isAvailable() : this;
-        return getValue();
+        return value;
     }
 
     @Override
@@ -101,14 +89,14 @@ public enum HotSpotMarkId implements CompilationResult.MarkId {
     }
 
     public boolean isAvailable() {
-        return getValue() != null;
+        return value != null;
     }
 
     @Override
     public String toString() {
         return "HotSpotCodeMark{" + name() +
-                        ", value=" + getValue() +
+                        ", value=" + value +
-                        ", optional=" + optional +
                         '}';
     }
+
 }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/nodes/GraalHotSpotVMConfigNode.java

@@ -25,6 +25,7 @@
 package org.graalvm.compiler.hotspot.nodes;
 
 import static org.graalvm.compiler.core.common.GraalOptions.GeneratePIC;
+import static org.graalvm.compiler.core.common.GraalOptions.AOTVerifyOops;
 import static org.graalvm.compiler.nodeinfo.NodeCycles.CYCLES_1;
 import static org.graalvm.compiler.nodeinfo.NodeSize.SIZE_1;
 
@@ -70,6 +71,7 @@ public class GraalHotSpotVMConfigNode extends FloatingNode implements LIRLowerab
         super(TYPE, stamp);
         this.config = config;
         this.markId = markId;
+        assert markId != null;
     }
 
     /**
@@ -83,6 +85,7 @@ public class GraalHotSpotVMConfigNode extends FloatingNode implements LIRLowerab
         super(TYPE, StampFactory.forKind(kind));
         this.config = config;
         this.markId = markId;
+        assert markId != null;
     }
 
     @Override
@@ -97,6 +100,9 @@ public class GraalHotSpotVMConfigNode extends FloatingNode implements LIRLowerab
     @NodeIntrinsic
     private static native int loadIntConfigValue(@ConstantNodeParameter HotSpotMarkId markId);
 
+    @NodeIntrinsic
+    private static native boolean loadBoolConfigValue(@ConstantNodeParameter HotSpotMarkId markId);
+
     public static long cardTableAddress() {
         return loadLongConfigValue(HotSpotMarkId.CARD_TABLE_ADDRESS);
     }
@@ -109,6 +115,22 @@ public class GraalHotSpotVMConfigNode extends FloatingNode implements LIRLowerab
         return loadIntConfigValue(HotSpotMarkId.LOG_OF_HEAP_REGION_GRAIN_BYTES);
     }
 
+    public static boolean verifyOops() {
+        return loadBoolConfigValue(HotSpotMarkId.VERIFY_OOPS);
+    }
+
+    public static long verifyOopBits() {
+        return loadLongConfigValue(HotSpotMarkId.VERIFY_OOP_BITS);
+    }
+
+    public static long verifyOopMask() {
+        return loadLongConfigValue(HotSpotMarkId.VERIFY_OOP_MASK);
+    }
+
+    public static long verifyOopCounterAddress() {
+        return loadLongConfigValue(HotSpotMarkId.VERIFY_OOP_COUNT_ADDRESS);
+    }
+
     public static boolean intrinsify(GraphBuilderContext b, @InjectedNodeParameter Stamp returnStamp, @InjectedNodeParameter GraalHotSpotVMConfig config, HotSpotMarkId mark) {
         if (b.getReplacements().isEncodingSnippets()) {
             // This plugin must be deferred so that these constants aren't embedded in libgraal
@@ -120,19 +142,36 @@ public class GraalHotSpotVMConfigNode extends FloatingNode implements LIRLowerab
 
     @Override
     public Node canonical(CanonicalizerTool tool) {
-        Boolean generatePIC = GeneratePIC.getValue(tool.getOptions());
+        boolean generatePIC = GeneratePIC.getValue(tool.getOptions());
-        if (markId == null) {
+        boolean aotVerifyOops = AOTVerifyOops.getValue(tool.getOptions());
-            return ConstantNode.forBoolean(!generatePIC);
+        if (!generatePIC || !markId.isAvailable()) {
-        } else if (!generatePIC) {
             if (markId == HotSpotMarkId.CARD_TABLE_ADDRESS) {
                 return ConstantNode.forLong(config.cardtableStartAddress);
             } else if (markId == HotSpotMarkId.CRC_TABLE_ADDRESS) {
                 return ConstantNode.forLong(config.crcTableAddress);
             } else if (markId == HotSpotMarkId.LOG_OF_HEAP_REGION_GRAIN_BYTES) {
                 return ConstantNode.forInt(config.logOfHRGrainBytes);
+            } else if (markId == HotSpotMarkId.VERIFY_OOPS) {
+                return ConstantNode.forBoolean(config.verifyOops);
+            } else if (markId == HotSpotMarkId.VERIFY_OOP_BITS) {
+                return ConstantNode.forLong(config.verifyOopBits);
+            } else if (markId == HotSpotMarkId.VERIFY_OOP_MASK) {
+                return ConstantNode.forLong(config.verifyOopMask);
+            } else if (markId == HotSpotMarkId.VERIFY_OOP_COUNT_ADDRESS) {
+                return ConstantNode.forLong(config.verifyOopCounterAddress);
             } else {
                 throw GraalError.shouldNotReachHere(markId.toString());
             }
+        } else if (generatePIC && !aotVerifyOops) {
+            if (markId == HotSpotMarkId.VERIFY_OOPS) {
+                return ConstantNode.forBoolean(false);
+            } else if (markId == HotSpotMarkId.VERIFY_OOP_BITS) {
+                return ConstantNode.forLong(0L);
+            } else if (markId == HotSpotMarkId.VERIFY_OOP_MASK) {
+                return ConstantNode.forLong(0L);
+            } else if (markId == HotSpotMarkId.VERIFY_OOP_COUNT_ADDRESS) {
+                return ConstantNode.forLong(0L);
+            }
         }
         return this;
     }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/HotSpotG1WriteBarrierSnippets.java

@@ -149,7 +149,7 @@ public final class HotSpotG1WriteBarrierSnippets extends G1WriteBarrierSnippets
 
     @Override
     protected boolean verifyOops() {
-        return HotSpotReplacementsUtil.verifyOops(INJECTED_VMCONFIG);
+        return GraalHotSpotVMConfigNode.verifyOops();
     }
 
     @Override

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/HotSpotReplacementsUtil.java

@@ -41,6 +41,7 @@ import org.graalvm.compiler.graph.Node.ConstantNodeParameter;
 import org.graalvm.compiler.graph.Node.NodeIntrinsic;
 import org.graalvm.compiler.graph.spi.CanonicalizerTool;
 import org.graalvm.compiler.hotspot.GraalHotSpotVMConfig;
+import org.graalvm.compiler.hotspot.nodes.GraalHotSpotVMConfigNode;
 import org.graalvm.compiler.hotspot.word.KlassPointer;
 import org.graalvm.compiler.nodes.CanonicalizableLocation;
 import org.graalvm.compiler.nodes.CompressionNode;
@@ -192,11 +193,6 @@ public class HotSpotReplacementsUtil {
         return config.useG1GC;
     }
 
-    @Fold
-    public static boolean verifyOops(@InjectedParameter GraalHotSpotVMConfig config) {
-        return config.verifyOops;
-    }
-
     /**
      * @see GraalHotSpotVMConfig#doingUnsafeAccessOffset
      */
@@ -726,7 +722,7 @@ public class HotSpotReplacementsUtil {
     }
 
     public static Object verifyOop(Object object) {
-        if (verifyOops(INJECTED_VMCONFIG)) {
+        if (GraalHotSpotVMConfigNode.verifyOops()) {
             verifyOopStub(VERIFY_OOP, object);
         }
         return object;

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/stubs/ForeignCallSnippets.java

@@ -29,7 +29,6 @@ import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfig.INJECTED_VMCONFI
 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.clearPendingException;
 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.getPendingException;
 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.loadHubIntrinsic;
-import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.verifyOops;
 import static org.graalvm.compiler.hotspot.stubs.StubUtil.fatal;
 
 import org.graalvm.compiler.api.replacements.Fold;
@@ -39,6 +38,7 @@ import org.graalvm.compiler.debug.DebugHandlersFactory;
 import org.graalvm.compiler.hotspot.GraalHotSpotVMConfig;
 import org.graalvm.compiler.hotspot.meta.HotSpotProviders;
 import org.graalvm.compiler.hotspot.nodes.DeoptimizeCallerNode;
+import org.graalvm.compiler.hotspot.nodes.GraalHotSpotVMConfigNode;
 import org.graalvm.compiler.hotspot.word.KlassPointer;
 import org.graalvm.compiler.nodes.NamedLocationIdentity;
 import org.graalvm.compiler.nodes.PiNode;
@@ -87,15 +87,15 @@ public class ForeignCallSnippets implements Snippets {
      */
     @Snippet
     public static Object verifyObject(Object object) {
-        if (verifyOops(INJECTED_VMCONFIG)) {
+        if (GraalHotSpotVMConfigNode.verifyOops()) {
-            Word verifyOopCounter = WordFactory.unsigned(verifyOopCounterAddress(INJECTED_VMCONFIG));
+            Word verifyOopCounter = WordFactory.unsigned(GraalHotSpotVMConfigNode.verifyOopCounterAddress());
             verifyOopCounter.writeInt(0, verifyOopCounter.readInt(0) + 1);
 
             Pointer oop = Word.objectToTrackedPointer(object);
             if (object != null) {
                 GuardingNode anchorNode = SnippetAnchorNode.anchor();
                 // make sure object is 'reasonable'
-                if (!oop.and(WordFactory.unsigned(verifyOopMask(INJECTED_VMCONFIG))).equal(WordFactory.unsigned(verifyOopBits(INJECTED_VMCONFIG)))) {
+                if (!oop.and(WordFactory.unsigned(GraalHotSpotVMConfigNode.verifyOopMask())).equal(WordFactory.unsigned(GraalHotSpotVMConfigNode.verifyOopBits()))) {
                     fatal("oop not in heap: %p", oop.rawValue());
                 }
 
@@ -108,11 +108,6 @@ public class ForeignCallSnippets implements Snippets {
         return object;
     }
 
-    @Fold
-    static long verifyOopCounterAddress(@InjectedParameter GraalHotSpotVMConfig config) {
-        return config.verifyOopCounterAddress;
-    }
-
     @Fold
     static long verifyOopMask(@InjectedParameter GraalHotSpotVMConfig config) {
         return config.verifyOopMask;

test/jtreg-ext/requires/VMProps.java

@@ -389,11 +389,6 @@ public class VMProps implements Callable<Map<String, String>> {
             return "false";
         }
 
-        if (WB.getBooleanVMFlag("VerifyOops")) {
-            // Should be enabled when JDK-8209961 is fixed
-            return "false";
-        }
-
         switch (GC.selected()) {
             case Serial:
             case Parallel: