8225573: Shenandoah: Enhance ShenandoahVerifier to ensure roots to-space invariant

Reviewed-by: shade

src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp

@@ -2178,6 +2178,11 @@ void ShenandoahHeap::op_final_updaterefs() {
     concurrent_mark()->update_thread_roots(ShenandoahPhaseTimings::final_update_refs_roots);
   }
 
+  // Has to be done before cset is clear
+  if (ShenandoahVerify) {
+    verifier()->verify_roots_in_to_space();
+  }
+
   ShenandoahGCPhase final_update_refs(ShenandoahPhaseTimings::final_update_refs_recycle);
 
   trash_cset_regions();
@@ -2185,7 +2190,6 @@ void ShenandoahHeap::op_final_updaterefs() {
   set_update_refs_in_progress(false);
 
   if (ShenandoahVerify) {
-    verifier()->verify_roots_no_forwarded();
     verifier()->verify_after_updaterefs();
   }
 

src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp

@@ -964,6 +964,44 @@ public:
   void do_oop(oop* p)       { do_oop_work(p); }
 };
 
+class ShenandoahVerifyInToSpaceClosure : public OopClosure {
+private:
+  template <class T>
+  void do_oop_work(T* p) {
+    T o = RawAccess<>::oop_load(p);
+    if (!CompressedOops::is_null(o)) {
+      oop obj = CompressedOops::decode_not_null(o);
+      ShenandoahHeap* heap = ShenandoahHeap::heap_no_check();
+
+      if (!heap->marking_context()->is_marked(obj)) {
+        ShenandoahAsserts::print_failure(ShenandoahAsserts::_safe_all, obj, p, NULL,
+                "Verify Roots In To-Space", "Should be marked", __FILE__, __LINE__);
+      }
+
+      if (heap->in_collection_set(obj)) {
+        ShenandoahAsserts::print_failure(ShenandoahAsserts::_safe_all, obj, p, NULL,
+                "Verify Roots In To-Space", "Should not be in collection set", __FILE__, __LINE__);
+      }
+
+      oop fwd = (oop) ShenandoahForwarding::get_forwardee_raw_unchecked(obj);
+      if (!oopDesc::equals_raw(obj, fwd)) {
+        ShenandoahAsserts::print_failure(ShenandoahAsserts::_safe_all, obj, p, NULL,
+                "Verify Roots In To-Space", "Should not be forwarded", __FILE__, __LINE__);
+      }
+    }
+  }
+
+public:
+  void do_oop(narrowOop* p) { do_oop_work(p); }
+  void do_oop(oop* p)       { do_oop_work(p); }
+};
+
+void ShenandoahVerifier::verify_roots_in_to_space() {
+  ShenandoahRootVerifier verifier;
+  ShenandoahVerifyInToSpaceClosure cl;
+  verifier.oops_do(&cl);
+}
+
 void ShenandoahVerifier::verify_roots_no_forwarded() {
   ShenandoahRootVerifier verifier;
   ShenandoahVerifyNoForwared cl;

src/hotspot/share/gc/shenandoah/shenandoahVerifier.hpp

@@ -189,6 +189,7 @@ public:
   void verify_generic(VerifyOption option);
 
   // Roots should only contain to-space oops
+  void verify_roots_in_to_space();
   void verify_roots_no_forwarded();
   void verify_roots_no_forwarded_except(ShenandoahRootVerifier::RootTypes types);
 };