8196626: [Graal] runtime/appcds/UseAppCDS.java crashes with "VM thread using lock Heap_lock (not allowed to block on)"

Print a message and abort CDS dumping when the allocation of metaspace has failed

Reviewed-by: iklam, mseledtsov

src/hotspot/share/interpreter/rewriter.cpp

@@ -111,12 +111,12 @@ void Rewriter::make_constant_pool_cache(TRAPS) {
   if (HAS_PENDING_EXCEPTION) {
     MetadataFactory::free_metadata(loader_data, cache);
     _pool->set_cache(NULL);  // so the verifier isn't confused
+  } else {
+    DEBUG_ONLY(
+    if (DumpSharedSpaces) {
+      cache->verify_just_initialized();
+    })
   }
-
-  DEBUG_ONLY(
-  if (DumpSharedSpaces) {
-    cache->verify_just_initialized();
-  })
 }
 
 

src/hotspot/share/memory/metaspace.cpp

@@ -4848,6 +4848,12 @@ MetaWord* Metaspace::allocate(ClassLoaderData* loader_data, size_t word_size,
   MetaWord* result = loader_data->metaspace_non_null()->allocate(word_size, mdtype);
 
   if (result == NULL) {
+    if (DumpSharedSpaces && THREAD->is_VM_thread()) {
+      tty->print_cr("Failed allocating metaspace object type %s of size " SIZE_FORMAT ". CDS dump aborted.",
+          MetaspaceObj::type_name(type), word_size * BytesPerWord);
+      vm_exit(1);
+    }
+
     tracer()->report_metaspace_allocation_failure(loader_data, word_size, type, mdtype);
 
     // Allocation failed.

src/hotspot/share/memory/metaspaceShared.cpp

@@ -1018,6 +1018,7 @@ public:
   VMOp_Type type() const { return VMOp_PopulateDumpSharedSpace; }
   void doit();   // outline because gdb sucks
   static void write_region(FileMapInfo* mapinfo, int region, DumpRegion* space, bool read_only,  bool allow_exec);
+  bool allow_nested_vm_operations() const { return true; }
 }; // class VM_PopulateDumpSharedSpace
 
 class SortedSymbolClosure: public SymbolClosure {

test/hotspot/jtreg/runtime/appcds/GraalWithLimitedMetaspace.java

@@ -0,0 +1,132 @@
+/*
+ * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+/*
+ * @test
+ * @summary Test dumping with limited metaspace with loading of JVMCI related classes.
+ *          VM should not crash but CDS dump will abort upon failure in allocating metaspace.
+ * @requires vm.cds & vm.graal.enabled
+ * @library /test/lib
+ * @modules java.base/jdk.internal.misc
+ *          java.management
+ *          jdk.jartool/sun.tools.jar
+ * @build UseAppCDS_Test
+ * @run driver ClassFileInstaller -jar test.jar UseAppCDS_Test
+ * @run main/othervm -XX:+UnlockExperimentalVMOptions -XX:+EnableJVMCI
+ *      -XX:+TieredCompilation -XX:+UseJVMCICompiler -Djvmci.Compiler=graal
+ *      GraalWithLimitedMetaspace
+ */
+
+import jdk.test.lib.cds.CDSTestUtils;
+import jdk.test.lib.process.OutputAnalyzer;
+import jdk.test.lib.process.ProcessTools;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.io.*;
+
+public class GraalWithLimitedMetaspace {
+
+    // Class UseAppCDS_Test is loaded by the App loader
+
+    static final String TEST_OUT = "UseAppCDS_Test.main--executed";
+
+    private static final String TESTJAR = "./test.jar";
+    private static final String TESTNAME = "UseAppCDS_Test";
+    private static final String TESTCLASS = TESTNAME + ".class";
+
+    private static final String CLASSLIST_FILE = "./GraalWithLimitedMetaspace.classlist";
+    private static final String ARCHIVE_FILE = "./GraalWithLimitedMetaspace.jsa";
+    private static final String BOOTCLASS = "java.lang.Class";
+
+    public static void main(String[] args) throws Exception {
+
+        // dump loaded classes into a classlist file
+        dumpLoadedClasses(new String[] { BOOTCLASS, TESTNAME });
+
+
+        // create an archive using the classlist
+        dumpArchive();
+
+    }
+
+    public static List<String> toClassNames(String filename) throws IOException {
+        ArrayList<String> classes = new ArrayList<>();
+        try (BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(filename)))) {
+            for (; ; ) {
+                String line = br.readLine();
+                if (line == null) {
+                    break;
+                }
+                classes.add(line.replaceAll("/", "."));
+            }
+        }
+        return classes;
+    }
+
+    static void dumpLoadedClasses(String[] expectedClasses) throws Exception {
+        ProcessBuilder pb = ProcessTools.createJavaProcessBuilder(true,
+          TestCommon.makeCommandLineForAppCDS(
+            "-XX:DumpLoadedClassList=" + CLASSLIST_FILE,
+            // trigger JVMCI runtime init so that JVMCI classes will be
+            // included in the classlist
+            "-XX:+EagerJVMCI",
+            "-cp",
+            TESTJAR,
+            "-XX:+UseAppCDS",
+            TESTNAME,
+            TEST_OUT));
+
+        OutputAnalyzer output = TestCommon.executeAndLog(pb, "dump-loaded-classes")
+            .shouldHaveExitValue(0)
+            .shouldContain(TEST_OUT);
+
+        List<String> dumpedClasses = toClassNames(CLASSLIST_FILE);
+
+        for (String clazz : expectedClasses) {
+            if (!dumpedClasses.contains(clazz)) {
+                throw new RuntimeException(clazz + " missing in " +
+                                           CLASSLIST_FILE);
+            }
+        }
+    }
+
+    static void dumpArchive() throws Exception {
+        ProcessBuilder pb = ProcessTools.createJavaProcessBuilder(true,
+          TestCommon.makeCommandLineForAppCDS(
+            "-cp",
+            TESTJAR,
+            "-XX:+UseAppCDS",
+            "-XX:SharedClassListFile=" + CLASSLIST_FILE,
+            "-XX:SharedArchiveFile=" + ARCHIVE_FILE,
+            "-Xlog:cds",
+            "-Xshare:dump",
+            "-XX:MetaspaceSize=12M",
+            "-XX:MaxMetaspaceSize=12M"));
+
+        OutputAnalyzer output = TestCommon.executeAndLog(pb, "dump-archive")
+            .shouldHaveExitValue(1)
+            .shouldContain("Failed allocating metaspace object type");
+    }
+}