8209389: SIGSEGV in WalkOopAndArchiveClosure::do_oop_work

Check the MetaspaceShared::archive_heap_object return value and handle failure accordingly.

Reviewed-by: iklam, coleenp

src/hotspot/share/classfile/javaClasses.cpp

@@ -1038,6 +1038,7 @@ void java_lang_Class::archive_basic_type_mirrors(TRAPS) {
     if (m != NULL) {
       // Update the field at _array_klass_offset to point to the relocated array klass.
       oop archived_m = MetaspaceShared::archive_heap_object(m, THREAD);
+      assert(archived_m != NULL, "sanity");
       Klass *ak = (Klass*)(archived_m->metadata_field(_array_klass_offset));
       assert(ak != NULL || t == T_VOID, "should not be NULL");
       if (ak != NULL) {

src/hotspot/share/memory/heapShared.cpp

@@ -352,7 +352,7 @@ class WalkOopAndArchiveClosure: public BasicOopIterateClosure {
       // A java.lang.Class instance can not be included in an archived
       // object sub-graph.
       if (java_lang_Class::is_instance(obj)) {
-        tty->print("Unknown java.lang.Class object is in the archived sub-graph\n");
+        log_error(cds, heap)("Unknown java.lang.Class object is in the archived sub-graph\n");
         vm_exit(1);
       }
 
@@ -392,6 +392,17 @@ class WalkOopAndArchiveClosure: public BasicOopIterateClosure {
       Thread* THREAD = Thread::current();
       // Archive the current oop before iterating through its references
       archived = MetaspaceShared::archive_heap_object(obj, THREAD);
+      if (archived == NULL) {
+        ResourceMark rm;
+        LogTarget(Error, cds, heap) log_err;
+        LogStream ls_err(log_err);
+        outputStream* out_err = &ls_err;
+        log_err.print("Failed to archive %s object ("
+                      PTR_FORMAT "), size[" SIZE_FORMAT "] in sub-graph",
+                      obj->klass()->external_name(), p2i(obj), (size_t)obj->size());
+        obj->print_on(out_err);
+        vm_exit(1);
+      }
       assert(MetaspaceShared::is_archive_object(archived), "must be archived");
       log.print("=== archiving oop " PTR_FORMAT " ==> " PTR_FORMAT,
                  p2i(obj), p2i(archived));
@@ -480,6 +491,15 @@ void HeapShared::archive_reachable_objects_from_static_field(Klass *k,
 
       // get the archived copy of the field referenced object
       oop af = MetaspaceShared::archive_heap_object(f, THREAD);
+      if (af == NULL) {
+        // Skip archiving the sub-graph referenced from the current entry field.
+        ResourceMark rm;
+        log_info(cds, heap)(
+          "Cannot archive the sub-graph referenced from %s object ("
+          PTR_FORMAT ") size[" SIZE_FORMAT "], skipped.",
+          f->klass()->external_name(), p2i(f), (size_t)f->size());
+        return;
+      }
       if (!MetaspaceShared::is_archive_object(f)) {
         WalkOopAndArchiveClosure walker(1, subgraph_info, f, af);
         f->oop_iterate(&walker);
@@ -492,6 +512,10 @@ void HeapShared::archive_reachable_objects_from_static_field(Klass *k,
       Klass *relocated_k = af->klass();
       Klass *orig_k = f->klass();
       subgraph_info->add_subgraph_object_klass(orig_k, relocated_k);
+      ResourceMark rm;
+      log_info(cds, heap)(
+          "Archived the sub-graph referenced from %s object " PTR_FORMAT,
+          f->klass()->external_name(), p2i(f));
     } else {
       // The field contains null, we still need to record the entry point,
       // so it can be restored at runtime.

src/hotspot/share/memory/metaspaceShared.cpp

@@ -1873,6 +1873,8 @@ oop MetaspaceShared::archive_heap_object(oop obj, Thread* THREAD) {
 
   int len = obj->size();
   if (G1CollectedHeap::heap()->is_archive_alloc_too_large(len)) {
+    log_debug(cds, heap)("Cannot archive, object (" PTR_FORMAT ") is too large: " SIZE_FORMAT,
+                         p2i(obj), (size_t)obj->size());
     return NULL;
   }
 
@@ -1883,9 +1885,14 @@ oop MetaspaceShared::archive_heap_object(oop obj, Thread* THREAD) {
     relocate_klass_ptr(archived_oop);
     ArchivedObjectCache* cache = MetaspaceShared::archive_object_cache();
     cache->put(obj, archived_oop);
+    log_debug(cds, heap)("Archived heap object " PTR_FORMAT " ==> " PTR_FORMAT,
+                         p2i(obj), p2i(archived_oop));
+  } else {
+    log_error(cds, heap)(
+      "Cannot allocate space for object " PTR_FORMAT " in archived heap region",
+      p2i(obj));
+    vm_exit(1);
   }
-  log_debug(cds, heap)("Archived heap object " PTR_FORMAT " ==> " PTR_FORMAT,
-                       p2i(obj), p2i(archived_oop));
   return archived_oop;
 }
 

src/hotspot/share/oops/constantPool.cpp

@@ -296,6 +296,11 @@ void ConstantPool::archive_resolved_references(Thread* THREAD) {
     }
 
     oop archived = MetaspaceShared::archive_heap_object(rr, THREAD);
+    // If the resolved references array is not archived (too large),
+    // the 'archived' object is NULL. No need to explicitly check
+    // the return value of archive_heap_object here. At runtime, the
+    // resolved references will be created using the normal process
+    // when there is no archived value.
     _cache->set_archived_references(archived);
     set_resolved_references(NULL);
   }

test/hotspot/jtreg/runtime/appcds/cacheObject/ArchivedIntegerCacheTest.java

@@ -137,5 +137,17 @@ public class ArchivedIntegerCacheTest {
                 "30000",
                 "false");
         TestCommon.checkExec(output);
+
+        // Test case 6)
+        // - Cache is too large to archive
+        output = TestCommon.dump(appJar,
+                TestCommon.list("CheckIntegerCacheApp"),
+                "-XX:AutoBoxCacheMax=2000000",
+                "-Xmx1g",
+                "-XX:NewSize=1g",
+                "-Xlog:cds+heap=info",
+                use_whitebox_jar);
+        TestCommon.checkDump(output,
+            "Cannot archive the sub-graph referenced from [Ljava.lang.Integer; object");
     }
 }