8293650: Shenandoah: Support archived heap objects

Reviewed-by: rkennke, wkemper, iklam

src/hotspot/share/cds/archiveHeapWriter.hpp

@@ -112,6 +112,12 @@ class ArchiveHeapWriter : AllStatic {
 public:
   static const intptr_t NOCOOPS_REQUESTED_BASE = 0x10000000;
 
+  // The minimum region size of all collectors that are supported by CDS in
+  // ArchiveHeapLoader::can_map() mode. Currently only G1 is supported. G1's region size
+  // depends on -Xmx, but can never be smaller than 1 * M.
+  // (TODO: Perhaps change to 256K to be compatible with Shenandoah)
+  static constexpr int MIN_GC_REGION_ALIGNMENT = 1 * M;
+
 private:
   class EmbeddedOopRelocator;
   struct NativePointerInfo {
@@ -119,12 +125,6 @@ private:
     int _field_offset;
   };
 
-  // The minimum region size of all collectors that are supported by CDS in
-  // ArchiveHeapLoader::can_map() mode. Currently only G1 is supported. G1's region size
-  // depends on -Xmx, but can never be smaller than 1 * M.
-  // (TODO: Perhaps change to 256K to be compatible with Shenandoah)
-  static constexpr int MIN_GC_REGION_ALIGNMENT = 1 * M;
-
   static GrowableArrayCHeap<u1, mtClassShared>* _buffer;
 
   // The number of bytes that have written into _buffer (may be smaller than _buffer->length()).

src/hotspot/share/cds/filemap.cpp

@@ -2025,7 +2025,7 @@ void FileMapInfo::map_or_load_heap_region() {
         // TODO - remove implicit knowledge of G1
         log_info(cds)("Cannot use CDS heap data. UseG1GC is required for -XX:-UseCompressedOops");
       } else {
-        log_info(cds)("Cannot use CDS heap data. UseEpsilonGC, UseG1GC, UseSerialGC or UseParallelGC are required.");
+        log_info(cds)("Cannot use CDS heap data. UseEpsilonGC, UseG1GC, UseSerialGC, UseParallelGC, or UseShenandoahGC are required.");
       }
     }
   }

src/hotspot/share/gc/shenandoah/shenandoahAsserts.cpp

@@ -254,7 +254,7 @@ void ShenandoahAsserts::assert_correct(void* interior_loc, oop obj, const char*
   // Do additional checks for special objects: their fields can hold metadata as well.
   // We want to check class loading/unloading did not corrupt them.
 
-  if (java_lang_Class::is_instance(obj)) {
+  if (Universe::is_fully_initialized() && java_lang_Class::is_instance(obj)) {
     Metadata* klass = obj->metadata_field(java_lang_Class::klass_offset());
     if (klass != nullptr && !Metaspace::contains(klass)) {
       print_failure(_safe_all, obj, interior_loc, nullptr, "Shenandoah assert_correct failed",
@@ -283,10 +283,12 @@ void ShenandoahAsserts::assert_in_correct_region(void* interior_loc, oop obj, co
   }
 
   size_t alloc_size = obj->size();
+  HeapWord* obj_end = cast_from_oop<HeapWord*>(obj) + alloc_size;
+
   if (ShenandoahHeapRegion::requires_humongous(alloc_size)) {
     size_t idx = r->index();
-    size_t num_regions = ShenandoahHeapRegion::required_regions(alloc_size * HeapWordSize);
-    for (size_t i = idx; i < idx + num_regions; i++) {
+    size_t end_idx = heap->heap_region_index_containing(obj_end - 1);
+    for (size_t i = idx; i < end_idx; i++) {
       ShenandoahHeapRegion* chain_reg = heap->get_region(i);
       if (i == idx && !chain_reg->is_humongous_start()) {
         print_failure(_safe_unknown, obj, interior_loc, nullptr, "Shenandoah assert_in_correct_region failed",
@@ -299,6 +301,12 @@ void ShenandoahAsserts::assert_in_correct_region(void* interior_loc, oop obj, co
                       file, line);
       }
     }
+  } else {
+    if (obj_end > r->top()) {
+      print_failure(_safe_unknown, obj, interior_loc, nullptr, "Shenandoah assert_in_correct_region failed",
+                    "Object end should be within the active area of the region",
+                    file, line);
+    }
   }
 }
 

src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp

@@ -72,6 +72,7 @@
 #include "gc/shenandoah/shenandoahJfrSupport.hpp"
 #endif
 
+#include "cds/archiveHeapWriter.hpp"
 #include "classfile/systemDictionary.hpp"
 #include "code/codeCache.hpp"
 #include "memory/classLoaderMetaspace.hpp"
@@ -2482,3 +2483,80 @@ bool ShenandoahHeap::requires_barriers(stackChunkOop obj) const {
 
   return false;
 }
+
+HeapWord* ShenandoahHeap::allocate_loaded_archive_space(size_t size) {
+#if INCLUDE_CDS_JAVA_HEAP
+  // CDS wants a continuous memory range to load a bunch of objects.
+  // This effectively bypasses normal allocation paths, and requires
+  // a bit of massaging to unbreak GC invariants.
+
+  ShenandoahAllocRequest req = ShenandoahAllocRequest::for_shared(size);
+
+  // Easy case: a single regular region, no further adjustments needed.
+  if (!ShenandoahHeapRegion::requires_humongous(size)) {
+    return allocate_memory(req);
+  }
+
+  // Hard case: the requested size would cause a humongous allocation.
+  // We need to make sure it looks like regular allocation to the rest of GC.
+
+  // CDS code would guarantee no objects straddle multiple regions, as long as
+  // regions are as large as MIN_GC_REGION_ALIGNMENT. It is impractical at this
+  // point to deal with case when Shenandoah runs with smaller regions.
+  // TODO: This check can be dropped once MIN_GC_REGION_ALIGNMENT agrees more with Shenandoah.
+  if (ShenandoahHeapRegion::region_size_bytes() < ArchiveHeapWriter::MIN_GC_REGION_ALIGNMENT) {
+    return nullptr;
+  }
+
+  HeapWord* mem = allocate_memory(req);
+  size_t start_idx = heap_region_index_containing(mem);
+  size_t num_regions = ShenandoahHeapRegion::required_regions(size * HeapWordSize);
+
+  // Flip humongous -> regular.
+  {
+    ShenandoahHeapLocker locker(lock(), false);
+    for (size_t c = start_idx; c < start_idx + num_regions; c++) {
+      get_region(c)->make_regular_bypass();
+    }
+  }
+
+  return mem;
+#else
+  assert(false, "Archive heap loader should not be available, should not be here");
+  return nullptr;
+#endif // INCLUDE_CDS_JAVA_HEAP
+}
+
+void ShenandoahHeap::complete_loaded_archive_space(MemRegion archive_space) {
+  // Nothing to do here, except checking that heap looks fine.
+#ifdef ASSERT
+  HeapWord* start = archive_space.start();
+  HeapWord* end = archive_space.end();
+
+  // No unclaimed space between the objects.
+  // Objects are properly allocated in correct regions.
+  HeapWord* cur = start;
+  while (cur < end) {
+    oop oop = cast_to_oop(cur);
+    shenandoah_assert_in_correct_region(nullptr, oop);
+    cur += oop->size();
+  }
+
+  // No unclaimed tail at the end of archive space.
+  assert(cur == end,
+         "Archive space should be fully used: " PTR_FORMAT " " PTR_FORMAT,
+         p2i(cur), p2i(end));
+
+  // Region bounds are good.
+  ShenandoahHeapRegion* begin_reg = heap_region_containing(start);
+  ShenandoahHeapRegion* end_reg = heap_region_containing(end);
+  assert(begin_reg->is_regular(), "Must be");
+  assert(end_reg->is_regular(), "Must be");
+  assert(begin_reg->bottom() == start,
+         "Must agree: archive-space-start: " PTR_FORMAT ", begin-region-bottom: " PTR_FORMAT,
+         p2i(start), p2i(begin_reg->bottom()));
+  assert(end_reg->top() == end,
+         "Must agree: archive-space-end: " PTR_FORMAT ", end-region-top: " PTR_FORMAT,
+         p2i(end), p2i(end_reg->top()));
+#endif
+}

src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp

@@ -539,6 +539,12 @@ public:
   void sync_pinned_region_status();
   void assert_pinned_region_status() NOT_DEBUG_RETURN;
 
+// ---------- CDS archive support
+
+  bool can_load_archived_objects() const override { return UseCompressedOops; }
+  HeapWord* allocate_loaded_archive_space(size_t size) override;
+  void complete_loaded_archive_space(MemRegion archive_space) override;
+
 // ---------- Allocation support
 //
 private:

src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp

@@ -100,8 +100,10 @@ void ShenandoahHeapRegion::make_regular_allocation() {
 
 void ShenandoahHeapRegion::make_regular_bypass() {
   shenandoah_assert_heaplocked();
-  assert (ShenandoahHeap::heap()->is_full_gc_in_progress() || ShenandoahHeap::heap()->is_degenerated_gc_in_progress(),
-          "only for full or degen GC");
+  assert (!Universe::is_fully_initialized() ||
+          ShenandoahHeap::heap()->is_full_gc_in_progress() ||
+          ShenandoahHeap::heap()->is_degenerated_gc_in_progress(),
+          "Only for STW GC or when Universe is initializing (CDS)");
 
   switch (_state) {
     case _empty_uncommitted:

test/hotspot/jtreg/runtime/cds/appcds/TestShenandoahWithCDS.java

@@ -0,0 +1,89 @@
+/*
+ * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test
+ * @bug 8293650
+ * @requires vm.cds
+ * @requires vm.bits == 64
+ * @requires vm.gc.Shenandoah
+ * @requires vm.gc.G1
+ * @requires vm.gc == null
+ * @library /test/lib /test/hotspot/jtreg/runtime/cds/appcds
+ * @compile test-classes/Hello.java
+ * @run driver TestShenandoahWithCDS
+ */
+
+import jdk.test.lib.Platform;
+import jdk.test.lib.process.OutputAnalyzer;
+
+public class TestShenandoahWithCDS {
+    public final static String HELLO = "Hello World";
+    static String helloJar;
+
+    public static void main(String... args) throws Exception {
+        helloJar = JarBuilder.build("hello", "Hello");
+
+        // Run with the variety of region sizes, and combinations
+        // of G1/Shenandoah at dump/exec times. "-1" means to use G1.
+        final int[] regionSizes = { -1, 256, 512, 1024, 2048 };
+
+        for (int dumpRegionSize : regionSizes) {
+            for (int execRegionSize : regionSizes) {
+                test(dumpRegionSize, execRegionSize);
+            }
+        }
+    }
+
+    static void test(int dumpRegionSize, int execRegionSize) throws Exception {
+        String exp = "-XX:+UnlockExperimentalVMOptions";
+        String optDumpGC = (dumpRegionSize != -1) ? "-XX:+UseShenandoahGC" : "-XX:+UseG1GC";
+        String optExecGC = (execRegionSize != -1) ? "-XX:+UseShenandoahGC" : "-XX:+UseG1GC";
+        String optDumpRegionSize = (dumpRegionSize != -1) ? "-XX:ShenandoahRegionSize=" + dumpRegionSize + "K" : exp;
+        String optExecRegionSize = (execRegionSize != -1) ? "-XX:ShenandoahRegionSize=" + execRegionSize + "K" : exp;
+        OutputAnalyzer out;
+
+        System.out.println("0. Dump with " + optDumpGC + " and " + optDumpRegionSize);
+        out = TestCommon.dump(helloJar,
+                              new String[] {"Hello"},
+                              exp,
+                              "-Xmx1g",
+                              optDumpGC,
+                              optDumpRegionSize,
+                              "-Xlog:cds");
+        out.shouldContain("Dumping shared data to file:");
+        out.shouldHaveExitValue(0);
+
+        System.out.println("1. Exec with " + optExecGC + " and " + optExecRegionSize);
+        out = TestCommon.exec(helloJar,
+                              exp,
+                              "-Xmx1g",
+                              optExecGC,
+                              optExecRegionSize,
+                              "-Xlog:cds",
+                              "Hello");
+        out.shouldContain(HELLO);
+        out.shouldHaveExitValue(0);
+    }
+}