stan/jdk-24
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

8247845: Shenandoah: refactor TLAB/GCLAB retirement code

Browse Source 
Reviewed-by: rkennke
This commit is contained in:
Aleksey Shipilev 2020-06-29 07:16:53 +02:00
parent 5ad963cf61
commit a793293464
5 changed files with 92 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

125
src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp
View File

@ -1044,28 +1044,79 @@ void ShenandoahHeap::trash_humongous_region_at(ShenandoahHeapRegion* start) {
}
}
class ShenandoahRetireGCLABClosure : public ThreadClosure {
class ShenandoahCheckCleanGCLABClosure : public ThreadClosure {
public:
ShenandoahCheckCleanGCLABClosure() {}
void do_thread(Thread* thread) {
PLAB* gclab = ShenandoahThreadLocalData::gclab(thread);
assert(gclab != NULL, "GCLAB should be initialized for %s", thread->name());
assert(gclab->words_remaining() == 0, "GCLAB should not need retirement");
}
};
class ShenandoahRetireGCLABClosure : public ThreadClosure {
private:
bool const _resize;
public:
ShenandoahRetireGCLABClosure(bool resize) : _resize(resize) {}
void do_thread(Thread* thread) {
PLAB* gclab = ShenandoahThreadLocalData::gclab(thread);
assert(gclab != NULL, "GCLAB should be initialized for %s", thread->name());
gclab->retire();
if (_resize && ShenandoahThreadLocalData::gclab_size(thread) > 0) {
ShenandoahThreadLocalData::set_gclab_size(thread, 0);
}
}
};
void ShenandoahHeap::make_parsable(bool retire_tlabs) {
if (UseTLAB) {
CollectedHeap::ensure_parsability(retire_tlabs);
void ShenandoahHeap::labs_make_parsable() {
assert(UseTLAB, "Only call with UseTLAB");
ShenandoahRetireGCLABClosure cl(false);
for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
ThreadLocalAllocBuffer& tlab = t->tlab();
tlab.make_parsable();
cl.do_thread(t);
}
ShenandoahRetireGCLABClosure cl;
workers()->threads_do(&cl);
}
void ShenandoahHeap::tlabs_retire(bool resize) {
assert(UseTLAB, "Only call with UseTLAB");
assert(!resize || ResizeTLAB, "Only call for resize when ResizeTLAB is enabled");
ThreadLocalAllocStats stats;
for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
ThreadLocalAllocBuffer& tlab = t->tlab();
tlab.retire(&stats);
if (resize) {
tlab.resize();
}
}
stats.publish();
#ifdef ASSERT
ShenandoahCheckCleanGCLABClosure cl;
for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
cl.do_thread(t);
}
workers()->threads_do(&cl);
#endif
}
void ShenandoahHeap::resize_tlabs() {
CollectedHeap::resize_all_tlabs();
void ShenandoahHeap::gclabs_retire(bool resize) {
assert(UseTLAB, "Only call with UseTLAB");
assert(!resize || ResizeTLAB, "Only call for resize when ResizeTLAB is enabled");
ShenandoahRetireGCLABClosure cl(resize);
for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
cl.do_thread(t);
}
workers()->threads_do(&cl);
}
class ShenandoahEvacuateUpdateRootsTask : public AbstractGangTask {
@ -1122,25 +1173,6 @@ size_t ShenandoahHeap::max_tlab_size() const {
return ShenandoahHeapRegion::max_tlab_size_words();
}
class ShenandoahRetireAndResetGCLABClosure : public ThreadClosure {
public:
void do_thread(Thread* thread) {
PLAB* gclab = ShenandoahThreadLocalData::gclab(thread);
gclab->retire();
if (ShenandoahThreadLocalData::gclab_size(thread) > 0) {
ShenandoahThreadLocalData::set_gclab_size(thread, 0);
}
}
};
void ShenandoahHeap::retire_and_reset_gclabs() {
ShenandoahRetireAndResetGCLABClosure cl;
for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
cl.do_thread(t);
}
workers()->threads_do(&cl);
}
void ShenandoahHeap::collect(GCCause::Cause cause) {
control_thread()->request_gc(cause);
}
@ -1173,8 +1205,8 @@ jlong ShenandoahHeap::millis_since_last_gc() {
}
void ShenandoahHeap::prepare_for_verify() {
if (SafepointSynchronize::is_at_safepoint() || ! UseTLAB) {
make_parsable(false);
if (SafepointSynchronize::is_at_safepoint() && UseTLAB) {
labs_make_parsable();
}
}
@ -1264,7 +1296,7 @@ public:
* This is public API, used in preparation of object_iterate().
* Since we don't do linear scan of heap in object_iterate() (see comment below), we don't
* need to make the heap parsable. For Shenandoah-internal linear heap scans that we can
* control, we call SH::make_tlabs_parsable().
* control, we call SH::tlabs_retire, SH::gclabs_retire.
*/
void ShenandoahHeap::ensure_parsability(bool retire_tlabs) {
// No-op.
@ -1420,10 +1452,13 @@ void ShenandoahHeap::op_init_mark() {
}
set_concurrent_mark_in_progress(true);
// We need to reset all TLABs because we'd lose marks on all objects allocated in them.
{
ShenandoahGCPhase phase(ShenandoahPhaseTimings::make_parsable);
make_parsable(true);
// We need to reset all TLABs because they might be below the TAMS, and we need to mark
// the objects in them. Do not let mutators allocate any new objects in their current TLABs.
// It is also a good place to resize the TLAB sizes for future allocations.
if (UseTLAB) {
ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_manage_tlabs);
tlabs_retire(ResizeTLAB);
}
{
@ -1437,11 +1472,6 @@ void ShenandoahHeap::op_init_mark() {
concurrent_mark()->mark_roots(ShenandoahPhaseTimings::scan_roots);
if (UseTLAB) {
ShenandoahGCPhase phase(ShenandoahPhaseTimings::resize_tlabs);
resize_tlabs();
}
if (ShenandoahPacing) {
pacer()->setup_for_mark();
}
@ -1539,9 +1569,9 @@ void ShenandoahHeap::op_final_mark() {
// which would be outside the collection set, so no cset writes would happen there.
// Weaker one: new allocations would happen past update watermark, and so less work would
// be needed for reference updates (would update the large filler instead).
{
ShenandoahGCPhase phase(ShenandoahPhaseTimings::retire_tlabs);
make_parsable(true);
if (UseTLAB) {
ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_manage_labs);
tlabs_retire(false);
}
{
@ -1917,10 +1947,6 @@ void ShenandoahHeap::op_full(GCCause::Cause cause) {
metrics.snap_before();
full_gc()->do_it(cause);
if (UseTLAB) {
ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_resize_tlabs);
resize_all_tlabs();
}
metrics.snap_after();
@ -2511,9 +2537,12 @@ void ShenandoahHeap::op_init_updaterefs() {
set_evacuation_in_progress(false);
{
ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_refs_retire_gclabs);
retire_and_reset_gclabs();
// Evacuation is over, no GCLABs are needed anymore. GCLABs are under URWM, so we need to
// make them parsable for update code to work correctly. Plus, we can compute new sizes
// for future GCLABs here.
if (UseTLAB) {
ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_refs_manage_gclabs);
gclabs_retire(true);
}
if (ShenandoahVerify) {

8
src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp
View File

@ -595,7 +595,6 @@ private:
inline HeapWord* allocate_from_gclab(Thread* thread, size_t size);
HeapWord* allocate_from_gclab_slow(Thread* thread, size_t size);
HeapWord* allocate_new_gclab(size_t min_size, size_t word_size, size_t* actual_size);
void retire_and_reset_gclabs();
public:
HeapWord* allocate_memory(ShenandoahAllocRequest& request);
@ -615,10 +614,11 @@ public:
size_t max_tlab_size() const;
size_t tlab_used(Thread* ignored) const;
void resize_tlabs();
void ensure_parsability(bool retire_labs);
void ensure_parsability(bool retire_tlabs);
void make_parsable(bool retire_tlabs);
void labs_make_parsable();
void tlabs_retire(bool resize);
void gclabs_retire(bool resize);
// ---------- Marking support
//

5
src/hotspot/share/gc/shenandoah/shenandoahMarkCompact.cpp
View File

@ -145,7 +145,10 @@ void ShenandoahMarkCompact::do_it(GCCause::Cause gc_cause) {
_preserved_marks->init(heap->workers()->active_workers());
}
heap->make_parsable(true);
if (UseTLAB) {
heap->gclabs_retire(true);
heap->tlabs_retire(true);
}
OrderAccess::fence();

8
src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp
View File

@ -60,11 +60,10 @@ class outputStream;
\
f(init_mark_gross, "Pause Init Mark (G)") \
f(init_mark, "Pause Init Mark (N)") \
f(make_parsable, " Make Parsable") \
f(init_manage_tlabs, " Manage TLABs") \
f(init_update_region_states, " Update Region States") \
f(scan_roots, " Scan Roots") \
SHENANDOAH_PAR_PHASE_DO(scan_, " S: ", f) \
f(resize_tlabs, " Resize TLABs") \
\
f(conc_mark, "Concurrent Marking") \
f(conc_mark_roots, " Roots ") \
@ -86,7 +85,7 @@ class outputStream;
SHENANDOAH_PAR_PHASE_DO(purge_weak_par_, " WR: ", f) \
f(purge_cldg, " CLDG") \
f(final_update_region_states, " Update Region States") \
f(retire_tlabs, " Retire TLABs") \
f(final_manage_labs, " Manage GC/TLABs") \
f(choose_cset, " Choose Collection Set") \
f(final_rebuild_freeset, " Rebuild Free Set") \
f(init_evac, " Initial Evacuation") \
@ -113,7 +112,7 @@ class outputStream;
\
f(init_update_refs_gross, "Pause Init Update Refs (G)") \
f(init_update_refs, "Pause Init Update Refs (N)") \
f(init_update_refs_retire_gclabs, " Retire GCLABs") \
f(init_update_refs_manage_gclabs, " Manage GCLABs") \
\
f(conc_update_refs, "Concurrent Update Refs") \
\
@ -166,7 +165,6 @@ class outputStream;
f(full_gc_copy_objects_humong, " Humongous Objects") \
f(full_gc_copy_objects_reset_complete, " Reset Complete Bitmap") \
f(full_gc_copy_objects_rebuild, " Rebuild Region Sets") \
f(full_gc_resize_tlabs, " Resize TLABs") \
f(full_gc_heapdump_post, " Post Heap Dump") \
\
f(conc_uncommit, "Concurrent Uncommit") \

5
src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp
View File

@ -690,7 +690,10 @@ void ShenandoahVerifier::verify_at_safepoint(const char *label,
}
OrderAccess::fence();
_heap->make_parsable(false);
if (UseTLAB) {
_heap->labs_make_parsable();
}
// Allocate temporary bitmap for storing marking wavefront:
_verification_bit_map->clear();