8222992: Shenandoah: Pre-evacuate all roots

Reviewed-by: shade

src/hotspot/share/gc/shenandoah/shenandoahClosures.hpp

@@ -79,4 +79,16 @@ private:
   inline void do_oop_work(T* p);
 };
 
+#ifdef ASSERT
+class ShenandoahAssertNotForwardedClosure : public OopClosure {
+private:
+  template <class T>
+  inline void do_oop_work(T* p);
+
+public:
+  inline void do_oop(narrowOop* p);
+  inline void do_oop(oop* p);
+};
+#endif // ASSERT
+
 #endif // SHARE_GC_SHENANDOAH_SHENANDOAHCLOSURES_HPP

src/hotspot/share/gc/shenandoah/shenandoahClosures.inline.hpp

@@ -107,4 +107,18 @@ void ShenandoahEvacuateUpdateRootsClosure::do_oop(narrowOop* p) {
   do_oop_work(p);
 }
 
+#ifdef ASSERT
+template <class T>
+void ShenandoahAssertNotForwardedClosure::do_oop_work(T* p) {
+  T o = RawAccess<>::oop_load(p);
+  if (!CompressedOops::is_null(o)) {
+    oop obj = CompressedOops::decode_not_null(o);
+    shenandoah_assert_not_forwarded(p, obj);
+  }
+}
+
+void ShenandoahAssertNotForwardedClosure::do_oop(narrowOop* p) { do_oop_work(p); }
+void ShenandoahAssertNotForwardedClosure::do_oop(oop* p)       { do_oop_work(p); }
+#endif
+
 #endif // SHARE_GC_SHENANDOAH_SHENANDOAHCLOSURES_HPP

src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp

@@ -661,15 +661,13 @@ void ShenandoahConcurrentMark::weak_refs_work(bool full_gc) {
 // anything to them.
 void ShenandoahConcurrentMark::weak_roots_work() {
   WorkGang* workers = _heap->workers();
-  ShenandoahIsAliveSelector is_alive;
+  OopClosure* keep_alive = &do_nothing_cl;
-
+#ifdef ASSERT
-  if (_heap->has_forwarded_objects()) {
+  ShenandoahWeakAssertNotForwardedClosure verify_cl;
-    ShenandoahWeakUpdateClosure cl;
+  keep_alive = &verify_cl;
-    WeakProcessor::weak_oops_do(workers, is_alive.is_alive_closure(), &cl, 1);
+#endif
-  } else {
+  ShenandoahIsAliveClosure is_alive;
-    ShenandoahWeakAssertNotForwardedClosure cl;
+  WeakProcessor::weak_oops_do(workers, &is_alive, keep_alive, 1);
-    WeakProcessor::weak_oops_do(workers, is_alive.is_alive_closure(), &cl, 1);
-  }
 }
 
 void ShenandoahConcurrentMark::weak_refs_work_doit(bool full_gc) {

src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp

@@ -1487,8 +1487,13 @@ void ShenandoahHeap::op_final_mark() {
   if (!cancelled_gc()) {
     concurrent_mark()->finish_mark_from_roots(/* full_gc = */ false);
 
-    if (has_forwarded_objects()) {
+    // Degen may be caused by failed evacuation of roots
-      concurrent_mark()->update_roots(ShenandoahPhaseTimings::update_roots);
+    if (is_degenerated_gc_in_progress() && has_forwarded_objects()) {
+      concurrent_mark()->update_roots(ShenandoahPhaseTimings::degen_gc_update_roots);
+    }
+
+    if (ShenandoahVerify) {
+      verifier()->verify_roots_no_forwarded();
     }
 
     stop_concurrent_marking();
@@ -1540,6 +1545,7 @@ void ShenandoahHeap::op_final_mark() {
       }
 
       if (ShenandoahVerify) {
+        verifier()->verify_roots_no_forwarded();
         verifier()->verify_during_evacuation();
       }
     } else {
@@ -2176,8 +2182,8 @@ void ShenandoahHeap::op_final_updaterefs() {
   assert(!cancelled_gc(), "Should have been done right before");
 
   concurrent_mark()->update_roots(is_degenerated_gc_in_progress() ?
-                                 ShenandoahPhaseTimings::degen_gc_update_roots:
+                                  ShenandoahPhaseTimings::degen_gc_update_roots:
-                                 ShenandoahPhaseTimings::final_update_refs_roots);
+                                  ShenandoahPhaseTimings::final_update_refs_roots);
 
   ShenandoahGCPhase final_update_refs(ShenandoahPhaseTimings::final_update_refs_recycle);
 
@@ -2186,6 +2192,7 @@ void ShenandoahHeap::op_final_updaterefs() {
   set_update_refs_in_progress(false);
 
   if (ShenandoahVerify) {
+    verifier()->verify_roots_no_forwarded();
     verifier()->verify_after_updaterefs();
   }
 

src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.cpp

@@ -42,26 +42,6 @@
 #include "runtime/thread.hpp"
 #include "services/management.hpp"
 
-struct PhaseMap {
-  WeakProcessorPhases::Phase            _weak_processor_phase;
-  ShenandoahPhaseTimings::GCParPhases   _shenandoah_phase;
-};
-
-static const struct PhaseMap phase_mapping[] = {
-#if INCLUDE_JVMTI
-  {WeakProcessorPhases::jvmti,                 ShenandoahPhaseTimings::JVMTIWeakRoots},
-#endif
-#if INCLUDE_JFR
-  {WeakProcessorPhases::jfr,                   ShenandoahPhaseTimings::JFRWeakRoots},
-#endif
-  {WeakProcessorPhases::jni,                   ShenandoahPhaseTimings::JNIWeakRoots},
-  {WeakProcessorPhases::stringtable,           ShenandoahPhaseTimings::StringTableRoots},
-  {WeakProcessorPhases::resolved_method_table, ShenandoahPhaseTimings::ResolvedMethodTableRoots},
-  {WeakProcessorPhases::vm,                    ShenandoahPhaseTimings::VMWeakRoots}
-};
-
-STATIC_ASSERT(sizeof(phase_mapping) / sizeof(PhaseMap) == WeakProcessorPhases::phase_count);
-
 ShenandoahRootProcessor::ShenandoahRootProcessor(ShenandoahHeap* heap, uint n_workers,
                                                  ShenandoahPhaseTimings::Phase phase) :
   _process_strong_tasks(new SubTasksDone(SHENANDOAH_RP_PS_NumElements)),
@@ -84,32 +64,16 @@ ShenandoahRootProcessor::~ShenandoahRootProcessor() {
     StringDedup::gc_epilogue();
   }
 
-  ShenandoahWorkerTimings* worker_times = ShenandoahHeap::heap()->phase_timings()->worker_times();
-
   if (_processed_weak_roots) {
     assert(_weak_processor_timings.max_threads() == n_workers(), "Must match");
-    for (uint index = 0; index < WeakProcessorPhases::phase_count; index ++) {
+    ShenandoahWorkerTimings* worker_times = ShenandoahHeap::heap()->phase_timings()->worker_times();
-      weak_processor_timing_to_shenandoah_timing(phase_mapping[index]._weak_processor_phase,
+    ShenandoahTimingConverter::weak_processing_timing_to_shenandoah_timing(&_weak_processor_timings,
-                                                 phase_mapping[index]._shenandoah_phase,
+                                                                           worker_times);
-                                                 worker_times);
-    }
   }
 
   ShenandoahHeap::heap()->phase_timings()->record_workers_end(_phase);
 }
 
-void ShenandoahRootProcessor::weak_processor_timing_to_shenandoah_timing(const WeakProcessorPhases::Phase wpp,
-                                                                         const ShenandoahPhaseTimings::GCParPhases spp,
-                                                                         ShenandoahWorkerTimings* worker_times) const {
-  if (WeakProcessorPhases::is_serial(wpp)) {
-    worker_times->record_time_secs(spp, 0, _weak_processor_timings.phase_time_sec(wpp));
-  } else {
-    for (uint index = 0; index < _weak_processor_timings.max_threads(); index ++) {
-      worker_times->record_time_secs(spp, index, _weak_processor_timings.worker_time_sec(index, wpp));
-    }
-  }
-}
-
 void ShenandoahRootProcessor::process_all_roots_slow(OopClosure* oops) {
   CLDToOopClosure clds(oops, ClassLoaderData::_claim_strong);
   CodeBlobToOopClosure blobs(oops, !CodeBlobToOopClosure::FixRelocations);
@@ -251,7 +215,9 @@ ShenandoahRootEvacuator::ShenandoahRootEvacuator(ShenandoahHeap* heap, uint n_wo
   _evacuation_tasks(new SubTasksDone(SHENANDOAH_EVAC_NumElements)),
   _srs(n_workers),
   _phase(phase),
-  _coderoots_cset_iterator(ShenandoahCodeRoots::cset_iterator()) {
+  _coderoots_cset_iterator(ShenandoahCodeRoots::cset_iterator()),
+  _weak_processor_timings(n_workers),
+  _weak_processor_task(&_weak_processor_timings, n_workers) {
   heap->phase_timings()->record_workers_start(_phase);
   if (ShenandoahStringDedup::is_enabled()) {
     StringDedup::gc_prologue(false);
@@ -263,6 +229,12 @@ ShenandoahRootEvacuator::~ShenandoahRootEvacuator() {
   if (ShenandoahStringDedup::is_enabled()) {
     StringDedup::gc_epilogue();
   }
+
+  ShenandoahWorkerTimings* worker_times = ShenandoahHeap::heap()->phase_timings()->worker_times();
+  assert(_weak_processor_timings.max_threads() == n_workers(), "Must match");
+  ShenandoahTimingConverter::weak_processing_timing_to_shenandoah_timing(&_weak_processor_timings,
+                                                                         worker_times);
+
   ShenandoahHeap::heap()->phase_timings()->record_workers_end(_phase);
 }
 
@@ -270,23 +242,28 @@ void ShenandoahRootEvacuator::process_evacuate_roots(OopClosure* oops,
                                                      CodeBlobClosure* blobs,
                                                      uint worker_id) {
 
+  AlwaysTrueClosure always_true;
   ShenandoahWorkerTimings* worker_times = ShenandoahHeap::heap()->phase_timings()->worker_times();
   {
     bool is_par = n_workers() > 1;
     ResourceMark rm;
     ShenandoahWorkerTimingsTracker timer(worker_times, ShenandoahPhaseTimings::ThreadRoots, worker_id);
-
     Threads::possibly_parallel_oops_do(is_par, oops, NULL);
   }
 
+  {
+    ShenandoahWorkerTimingsTracker timer(worker_times, ShenandoahPhaseTimings::CLDGRoots, worker_id);
+    CLDToOopClosure clds(oops, ClassLoaderData::_claim_strong);
+    _cld_iterator.root_cld_do(&clds, &clds);
+  }
+
   if (blobs != NULL) {
     ShenandoahWorkerTimingsTracker timer(worker_times, ShenandoahPhaseTimings::CodeCacheRoots, worker_id);
     _coderoots_cset_iterator.possibly_parallel_blobs_do(blobs);
   }
 
   if (ShenandoahStringDedup::is_enabled()) {
-    ShenandoahForwardedIsAliveClosure is_alive;
+    ShenandoahStringDedup::parallel_oops_do(&always_true, oops, worker_id);
-    ShenandoahStringDedup::parallel_oops_do(&is_alive, oops, worker_id);
   }
 
   if (_evacuation_tasks->try_claim_task(SHENANDOAH_EVAC_Universe_oops_do)) {
@@ -306,6 +283,11 @@ void ShenandoahRootEvacuator::process_evacuate_roots(OopClosure* oops,
     JvmtiExport::weak_oops_do(&is_alive, oops);
   }
 
+  if (_evacuation_tasks->try_claim_task(SHENANDOAH_EVAC_JNIHandles_oops_do)) {
+    ShenandoahWorkerTimingsTracker timer(worker_times, ShenandoahPhaseTimings::JNIRoots, worker_id);
+    JNIHandles::oops_do(oops);
+  }
+
   if (_evacuation_tasks->try_claim_task(SHENANDOAH_EVAC_SystemDictionary_oops_do)) {
     ShenandoahWorkerTimingsTracker timer(worker_times, ShenandoahPhaseTimings::SystemDictionaryRoots, worker_id);
     SystemDictionary::oops_do(oops);
@@ -316,6 +298,7 @@ void ShenandoahRootEvacuator::process_evacuate_roots(OopClosure* oops,
     ObjectSynchronizer::oops_do(oops);
   }
 
+  _weak_processor_task.work<AlwaysTrueClosure, OopClosure>(worker_id, &always_true, oops);
 }
 
 uint ShenandoahRootEvacuator::n_workers() const {

src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.hpp

@@ -119,6 +119,9 @@ class ShenandoahRootEvacuator : public StackObj {
   StrongRootsScope _srs;
   ShenandoahPhaseTimings::Phase _phase;
   ShenandoahCsetCodeRootsIterator _coderoots_cset_iterator;
+  ParallelCLDRootIterator _cld_iterator;
+  WeakProcessorPhaseTimes _weak_processor_timings;
+  WeakProcessor::Task     _weak_processor_task;
 
   enum Shenandoah_evacuate_roots_tasks {
     SHENANDOAH_EVAC_Universe_oops_do,
@@ -126,6 +129,7 @@ class ShenandoahRootEvacuator : public StackObj {
     SHENANDOAH_EVAC_Management_oops_do,
     SHENANDOAH_EVAC_SystemDictionary_oops_do,
     SHENANDOAH_EVAC_jvmti_oops_do,
+    SHENANDOAH_EVAC_JNIHandles_oops_do,
     // Leave this one last.
     SHENANDOAH_EVAC_NumElements
   };

src/hotspot/share/gc/shenandoah/shenandoahUtils.cpp

@@ -183,3 +183,47 @@ ShenandoahWorkerSession::~ShenandoahWorkerSession() {
   ShenandoahThreadLocalData::set_worker_id(thr, ShenandoahThreadLocalData::INVALID_WORKER_ID);
 #endif
 }
+
+struct PhaseMap {
+   WeakProcessorPhases::Phase            _weak_processor_phase;
+   ShenandoahPhaseTimings::GCParPhases   _shenandoah_phase;
+};
+
+static const struct PhaseMap phase_mapping[] = {
+#if INCLUDE_JVMTI
+  {WeakProcessorPhases::jvmti,                 ShenandoahPhaseTimings::JVMTIWeakRoots},
+#endif
+#if INCLUDE_JFR
+  {WeakProcessorPhases::jfr,                   ShenandoahPhaseTimings::JFRWeakRoots},
+#endif
+  {WeakProcessorPhases::jni,                   ShenandoahPhaseTimings::JNIWeakRoots},
+  {WeakProcessorPhases::stringtable,           ShenandoahPhaseTimings::StringTableRoots},
+  {WeakProcessorPhases::resolved_method_table, ShenandoahPhaseTimings::ResolvedMethodTableRoots},
+  {WeakProcessorPhases::vm,                    ShenandoahPhaseTimings::VMWeakRoots}
+};
+
+STATIC_ASSERT(sizeof(phase_mapping) / sizeof(PhaseMap) == WeakProcessorPhases::phase_count);
+
+void ShenandoahTimingConverter::weak_processing_timing_to_shenandoah_timing(WeakProcessorPhaseTimes* weak_processing_timings,
+                                                                            ShenandoahWorkerTimings* sh_worker_times) {
+  assert(weak_processing_timings->max_threads() == weak_processing_timings->max_threads(), "Must match");
+  for (uint index = 0; index < WeakProcessorPhases::phase_count; index ++) {
+    weak_processing_phase_to_shenandoah_phase(phase_mapping[index]._weak_processor_phase,
+                                              weak_processing_timings,
+                                              phase_mapping[index]._shenandoah_phase,
+                                              sh_worker_times);
+  }
+}
+
+void ShenandoahTimingConverter::weak_processing_phase_to_shenandoah_phase(WeakProcessorPhases::Phase wpp,
+                                                                          WeakProcessorPhaseTimes* weak_processing_timings,
+                                                                          ShenandoahPhaseTimings::GCParPhases spp,
+                                                                          ShenandoahWorkerTimings* sh_worker_times) {
+  if (WeakProcessorPhases::is_serial(wpp)) {
+    sh_worker_times->record_time_secs(spp, 0, weak_processing_timings->phase_time_sec(wpp));
+  } else {
+    for (uint index = 0; index < weak_processing_timings->max_threads(); index ++) {
+      sh_worker_times->record_time_secs(spp, index, weak_processing_timings->worker_time_sec(index, wpp));
+    }
+  }
+}

src/hotspot/share/gc/shenandoah/shenandoahUtils.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2018, Red Hat, Inc. All rights reserved.
+ * Copyright (c) 2017, 2019, Red Hat, Inc. All rights reserved.
  *
  * This code is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License version 2 only, as
@@ -24,14 +24,14 @@
 #ifndef SHARE_GC_SHENANDOAH_SHENANDOAHUTILS_HPP
 #define SHARE_GC_SHENANDOAH_SHENANDOAHUTILS_HPP
 
-#include "jfr/jfrEvents.hpp"
-
 #include "gc/shared/gcCause.hpp"
+#include "gc/shared/gcVMOperations.hpp"
 #include "gc/shared/isGCActiveMark.hpp"
 #include "gc/shared/suspendibleThreadSet.hpp"
-#include "gc/shared/gcVMOperations.hpp"
+#include "gc/shared/weakProcessorPhaseTimes.hpp"
 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
+#include "jfr/jfrEvents.hpp"
 #include "memory/allocation.hpp"
 #include "runtime/safepoint.hpp"
 #include "runtime/vmThread.hpp"
@@ -175,4 +175,15 @@ public:
   }
 };
 
+class ShenandoahTimingConverter : public AllStatic {
+public:
+  static void weak_processing_timing_to_shenandoah_timing(WeakProcessorPhaseTimes* weak_processing_timings,
+                                                          ShenandoahWorkerTimings* sh_worker_times);
+private:
+  static void weak_processing_phase_to_shenandoah_phase(WeakProcessorPhases::Phase wpp,
+                                                        WeakProcessorPhaseTimes* weak_processing_timings,
+                                                        ShenandoahPhaseTimings::GCParPhases spp,
+                                                        ShenandoahWorkerTimings* sh_worker_times);
+};
+
 #endif // SHARE_GC_SHENANDOAH_SHENANDOAHUTILS_HPP

src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp

@@ -919,3 +919,31 @@ void ShenandoahVerifier::verify_after_fullgc() {
           _verify_gcstate_stable       // full gc cleaned up everything
   );
 }
+
+class ShenandoahVerifyNoForwared : public OopClosure {
+private:
+  template <class T>
+  void do_oop_work(T* p) {
+    T o = RawAccess<>::oop_load(p);
+    if (!CompressedOops::is_null(o)) {
+      oop obj = CompressedOops::decode_not_null(o);
+      oop fwd = (oop) ShenandoahBrooksPointer::get_raw_unchecked(obj);
+      if (!oopDesc::equals_raw(obj, fwd)) {
+        ShenandoahAsserts::print_failure(ShenandoahAsserts::_safe_all, obj, p, NULL,
+                                         "Verify Roots", "Should not be forwarded", __FILE__, __LINE__);
+      }
+    }
+  }
+
+public:
+  void do_oop(narrowOop* p) { do_oop_work(p); }
+  void do_oop(oop* p)       { do_oop_work(p); }
+};
+
+void ShenandoahVerifier::verify_roots_no_forwarded() {
+  guarantee(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "only when nothing else happens");
+  ShenandoahRootProcessor rp(_heap, 1, ShenandoahPhaseTimings::_num_phases); // no need for stats
+  ShenandoahVerifyNoForwared cl;
+  rp.process_all_roots_slow(&cl);
+}
+

src/hotspot/share/gc/shenandoah/shenandoahVerifier.hpp

@@ -186,6 +186,9 @@ public:
   void verify_after_traversal();
   void verify_after_degenerated();
   void verify_generic(VerifyOption option);
+
+  // Roots should only contain to-space oops
+  void verify_roots_no_forwarded();
 };
 
 #endif // SHARE_GC_SHENANDOAH_SHENANDOAHVERIFIER_HPP