stan/jdk-24
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

8232575: Shenandoah: asynchronous object/region pinning

Browse Source 
Reviewed-by: rkennke
This commit is contained in:
Aleksey Shipilev 2019-10-21 15:11:43 +02:00
parent 76465eb1d9
commit c3d807276d
10 changed files with 125 additions and 40 deletions

3
src/hotspot/share/gc/shenandoah/heuristics/shenandoahTraversalAggressiveHeuristics.cpp

@ -75,6 +75,9 @@ void ShenandoahTraversalAggressiveHeuristics::choose_collection_set(ShenandoahCo
RegionData *data = get_region_data_cache(heap->num_regions());
size_t cnt = 0;
// About to choose the collection set, make sure we have pinned regions in correct state
heap->assert_pinned_region_status();
// Step 0. Prepare all regions
for (size_t i = 0; i < heap->num_regions(); i++) {
ShenandoahHeapRegion* r = heap->get_region(i);

3
src/hotspot/share/gc/shenandoah/heuristics/shenandoahTraversalHeuristics.cpp

@ -59,6 +59,9 @@ void ShenandoahTraversalHeuristics::choose_collection_set(ShenandoahCollectionSe
RegionData *data = get_region_data_cache(heap->num_regions());
size_t cnt = 0;
// About to choose the collection set, make sure we have pinned regions in correct state
heap->assert_pinned_region_status();
// Step 0. Prepare all regions
for (size_t i = 0; i < heap->num_regions(); i++) {

53
src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp

@ -1507,6 +1507,14 @@ void ShenandoahHeap::op_final_mark() {
make_parsable(true);
}
// We are about to select the collection set, make sure it knows about
// current pinning status. Also, this allows trashing more regions that
// now have their pinning status dropped.
{
ShenandoahGCPhase phase(ShenandoahPhaseTimings::sync_pinned);
sync_pinned_region_status();
}
// Trash the collection set left over from previous cycle, if any.
{
ShenandoahGCPhase phase(ShenandoahPhaseTimings::trash_cset);
@ -1784,6 +1792,7 @@ void ShenandoahHeap::op_degenerated(ShenandoahDegenPoint point) {
// it, we fail degeneration right away and slide into Full GC to recover.
{
sync_pinned_region_status();
collection_set()->clear_current_index();
ShenandoahHeapRegion* r;
@ -2148,16 +2157,45 @@ void ShenandoahHeap::unregister_nmethod(nmethod* nm) {
}
oop ShenandoahHeap::pin_object(JavaThread* thr, oop o) {
ShenandoahHeapLocker locker(lock());
heap_region_containing(o)->make_pinned();
heap_region_containing(o)->record_pin();
return o;
}
void ShenandoahHeap::unpin_object(JavaThread* thr, oop o) {
ShenandoahHeapLocker locker(lock());
heap_region_containing(o)->make_unpinned();
heap_region_containing(o)->record_unpin();
}
void ShenandoahHeap::sync_pinned_region_status() {
ShenandoahHeapLocker locker(lock());
for (size_t i = 0; i < num_regions(); i++) {
ShenandoahHeapRegion *r = get_region(i);
if (r->is_active()) {
if (r->is_pinned()) {
if (r->pin_count() == 0) {
r->make_unpinned();
}
} else {
if (r->pin_count() > 0) {
r->make_pinned();
}
}
}
}
assert_pinned_region_status();
}
#ifdef ASSERT
void ShenandoahHeap::assert_pinned_region_status() {
for (size_t i = 0; i < num_regions(); i++) {
ShenandoahHeapRegion* r = get_region(i);
assert((r->is_pinned() && r->pin_count() > 0) || (!r->is_pinned() && r->pin_count() == 0),
"Region " SIZE_FORMAT " pinning status is inconsistent", i);
}
}
#endif
GCTimer* ShenandoahHeap::gc_timer() const {
return _gc_timer;
}
@ -2316,6 +2354,13 @@ void ShenandoahHeap::op_final_updaterefs() {
verifier()->verify_roots_in_to_space();
}
// Drop unnecessary "pinned" state from regions that does not have CP marks
// anymore, as this would allow trashing them below.
{
ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_update_refs_sync_pinned);
sync_pinned_region_status();
}
{
ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_update_refs_trash_cset);
trash_cset_regions();

4
src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp

@ -43,6 +43,7 @@ class ShenandoahGCSession;
class ShenandoahGCStateResetter;
class ShenandoahHeuristics;
class ShenandoahMarkingContext;
class ShenandoahMarkCompact;
class ShenandoahMode;
class ShenandoahPhaseTimings;
class ShenandoahHeap;
@ -574,6 +575,9 @@ public:
oop pin_object(JavaThread* thread, oop obj);
void unpin_object(JavaThread* thread, oop obj);
void sync_pinned_region_status();
void assert_pinned_region_status() NOT_DEBUG_RETURN;
// ---------- Allocation support
//
private:

48
src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp

@ -59,7 +59,6 @@ ShenandoahHeapRegion::ShenandoahHeapRegion(ShenandoahHeap* heap, HeapWord* start
_reserved(MemRegion(start, size_words)),
_region_number(index),
_new_top(NULL),
_critical_pins(0),
_empty_time(os::elapsedTime()),
_state(committed ? _empty_committed : _empty_uncommitted),
_tlab_allocs(0),
@ -69,7 +68,8 @@ ShenandoahHeapRegion::ShenandoahHeapRegion(ShenandoahHeap* heap, HeapWord* start
_seqnum_first_alloc_gc(0),
_seqnum_last_alloc_mutator(0),
_seqnum_last_alloc_gc(0),
_live_data(0) {
_live_data(0),
_critical_pins(0) {
ContiguousSpace::initialize(_reserved, true, committed);
}
@ -187,25 +187,21 @@ void ShenandoahHeapRegion::make_humongous_cont_bypass() {
void ShenandoahHeapRegion::make_pinned() {
_heap->assert_heaplock_owned_by_current_thread();
assert(pin_count() > 0, "Should have pins: " SIZE_FORMAT, pin_count());
switch (_state) {
case _regular:
assert (_critical_pins == 0, "sanity");
set_state(_pinned);
case _pinned_cset:
case _pinned:
_critical_pins++;
return;
case _humongous_start:
assert (_critical_pins == 0, "sanity");
set_state(_pinned_humongous_start);
case _pinned_humongous_start:
_critical_pins++;
return;
case _cset:
guarantee(_heap->cancelled_gc(), "only valid when evac has been cancelled");
assert (_critical_pins == 0, "sanity");
_state = _pinned_cset;
_critical_pins++;
return;
default:
report_illegal_transition("pinning");
@ -214,32 +210,21 @@ void ShenandoahHeapRegion::make_pinned() {
void ShenandoahHeapRegion::make_unpinned() {
_heap->assert_heaplock_owned_by_current_thread();
assert(pin_count() == 0, "Should not have pins: " SIZE_FORMAT, pin_count());
switch (_state) {
case _pinned:
assert (_critical_pins > 0, "sanity");
_critical_pins--;
if (_critical_pins == 0) {
set_state(_regular);
}
set_state(_regular);
return;
case _regular:
case _humongous_start:
assert (_critical_pins == 0, "sanity");
return;
case _pinned_cset:
guarantee(_heap->cancelled_gc(), "only valid when evac has been cancelled");
assert (_critical_pins > 0, "sanity");
_critical_pins--;
if (_critical_pins == 0) {
set_state(_cset);
}
set_state(_cset);
return;
case _pinned_humongous_start:
assert (_critical_pins > 0, "sanity");
_critical_pins--;
if (_critical_pins == 0) {
set_state(_humongous_start);
}
set_state(_humongous_start);
return;
default:
report_illegal_transition("unpinning");
@ -434,7 +419,7 @@ void ShenandoahHeapRegion::print_on(outputStream* st) const {
st->print("|G " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_gclab_allocs()), proper_unit_for_byte_size(get_gclab_allocs()));
st->print("|S " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_shared_allocs()), proper_unit_for_byte_size(get_shared_allocs()));
st->print("|L " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_live_data_bytes()), proper_unit_for_byte_size(get_live_data_bytes()));
st->print("|CP " SIZE_FORMAT_W(3), _critical_pins);
st->print("|CP " SIZE_FORMAT_W(3), pin_count());
st->print("|SN " UINT64_FORMAT_X_W(12) ", " UINT64_FORMAT_X_W(8) ", " UINT64_FORMAT_X_W(8) ", " UINT64_FORMAT_X_W(8),
seqnum_first_alloc_mutator(), seqnum_last_alloc_mutator(),
seqnum_first_alloc_gc(), seqnum_last_alloc_gc());
@ -702,3 +687,16 @@ void ShenandoahHeapRegion::set_state(RegionState to) {
}
_state = to;
}
void ShenandoahHeapRegion::record_pin() {
Atomic::add((size_t)1, &_critical_pins);
}
void ShenandoahHeapRegion::record_unpin() {
assert(pin_count() > 0, "Region " SIZE_FORMAT " should have non-zero pins", region_number());
Atomic::sub((size_t)1, &_critical_pins);
}
size_t ShenandoahHeapRegion::pin_count() const {
return Atomic::load(&_critical_pins);
}

6
src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp

@ -203,6 +203,10 @@ public:
RegionState state() const { return _state; }
int state_ordinal() const { return region_state_to_ordinal(_state); }
void record_pin();
void record_unpin();
size_t pin_count() const;
private:
static size_t RegionCount;
static size_t RegionSizeBytes;
@ -238,7 +242,6 @@ private:
// Rarely updated fields
HeapWord* _new_top;
size_t _critical_pins;
double _empty_time;
// Seldom updated fields
@ -255,6 +258,7 @@ private:
uint64_t _seqnum_last_alloc_gc;
volatile size_t _live_data;
volatile size_t _critical_pins;
// Claim some space at the end to protect next region
DEFINE_PAD_MINUS_SIZE(0, DEFAULT_CACHE_LINE_SIZE, 0);

3
src/hotspot/share/gc/shenandoah/shenandoahHeuristics.cpp

@ -120,6 +120,9 @@ void ShenandoahHeuristics::choose_collection_set(ShenandoahCollectionSet* collec
ShenandoahHeap* heap = ShenandoahHeap::heap();
// Check all pinned regions have updated status before choosing the collection set.
heap->assert_pinned_region_status();
// Step 1. Build up the region candidates we care about, rejecting losers and accepting winners right away.
size_t num_regions = heap->num_regions();

7
src/hotspot/share/gc/shenandoah/shenandoahMarkCompact.cpp

@ -128,6 +128,9 @@ void ShenandoahMarkCompact::do_it(GCCause::Cause gc_cause) {
// e. Set back forwarded objects bit back, in case some steps above dropped it.
heap->set_has_forwarded_objects(has_forwarded_objects);
// f. Sync pinned region status from the CP marks
heap->sync_pinned_region_status();
// The rest of prologue:
BiasedLocking::preserve_marks();
_preserved_marks->init(heap->workers()->active_workers());
@ -505,6 +508,10 @@ void ShenandoahMarkCompact::phase2_calculate_target_addresses(ShenandoahHeapRegi
ShenandoahHeap* heap = ShenandoahHeap::heap();
// About to figure out which regions can be compacted, make sure pinning status
// had been updated in GC prologue.
heap->assert_pinned_region_status();
{
// Trash the immediately collectible regions before computing addresses
ShenandoahTrashImmediateGarbageClosure tigcl;

4
src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp

@ -101,6 +101,7 @@ class outputStream;
f(purge_cldg, " CLDG") \
f(complete_liveness, " Complete Liveness") \
f(retire_tlabs, " Retire TLABs") \
f(sync_pinned, " Sync Pinned") \
f(trash_cset, " Trash CSet") \
f(prepare_evac, " Prepare Evacuation") \
\
@ -161,6 +162,7 @@ class outputStream;
f(final_update_refs_string_dedup_queue_roots, " UR: Dedup Queue Roots") \
f(final_update_refs_finish_queues, " UR: Finish Queues") \
\
f(final_update_refs_sync_pinned, " Sync Pinned") \
f(final_update_refs_trash_cset, " Trash CSet") \
\
f(degen_gc_gross, "Pause Degenerated GC (G)") \
@ -193,6 +195,7 @@ class outputStream;
f(traversal_gc_prepare, " Prepare") \
f(traversal_gc_make_parsable, " Make Parsable") \
f(traversal_gc_resize_tlabs, " Resize TLABs") \
f(traversal_gc_prepare_sync_pinned, " Sync Pinned") \
\
/* Per-thread timer block, should have "roots" counters in consistent order */ \
f(init_traversal_gc_work, " Work") \
@ -264,6 +267,7 @@ class outputStream;
f(final_traversal_update_string_dedup_queue_roots, " TU: Dedup Queue Roots") \
f(final_traversal_update_finish_queues, " TU: Finish Queues") \
\
f(traversal_gc_sync_pinned, " Sync Pinned") \
f(traversal_gc_cleanup, " Cleanup") \
\
f(full_gc_gross, "Pause Full GC (G)") \

34
src/hotspot/share/gc/shenandoah/shenandoahTraversalGC.cpp

@ -340,9 +340,6 @@ void ShenandoahTraversalGC::prepare_regions() {
}
void ShenandoahTraversalGC::prepare() {
_heap->collection_set()->clear();
assert(_heap->collection_set()->count() == 0, "collection set not clear");
{
ShenandoahGCPhase phase(ShenandoahPhaseTimings::traversal_gc_make_parsable);
_heap->make_parsable(true);
@ -356,15 +353,26 @@ void ShenandoahTraversalGC::prepare() {
assert(_heap->marking_context()->is_bitmap_clear(), "need clean mark bitmap");
assert(!_heap->marking_context()->is_complete(), "should not be complete");
ShenandoahFreeSet* free_set = _heap->free_set();
// About to choose the collection set, make sure we know which regions are pinned.
{
ShenandoahGCPhase phase_cleanup(ShenandoahPhaseTimings::traversal_gc_prepare_sync_pinned);
_heap->sync_pinned_region_status();
}
ShenandoahCollectionSet* collection_set = _heap->collection_set();
{
ShenandoahHeapLocker lock(_heap->lock());
// Find collection set
_heap->heuristics()->choose_collection_set(collection_set);
prepare_regions();
collection_set->clear();
assert(collection_set->count() == 0, "collection set not clear");
// Rebuild free set
free_set->rebuild();
// Find collection set
_heap->heuristics()->choose_collection_set(collection_set);
prepare_regions();
// Rebuild free set
_heap->free_set()->rebuild();
}
log_info(gc, ergo)("Collectable Garbage: " SIZE_FORMAT "%s, " SIZE_FORMAT "%s CSet, " SIZE_FORMAT " CSet regions",
byte_size_in_proper_unit(collection_set->garbage()), proper_unit_for_byte_size(collection_set->garbage()),
@ -385,7 +393,6 @@ void ShenandoahTraversalGC::init_traversal_collection() {
{
ShenandoahGCPhase phase_prepare(ShenandoahPhaseTimings::traversal_gc_prepare);
ShenandoahHeapLocker lock(_heap->lock());
prepare();
}
@ -613,6 +620,13 @@ void ShenandoahTraversalGC::final_traversal_collection() {
// Resize metaspace
MetaspaceGC::compute_new_size();
// Need to see that pinned region status is updated: newly pinned regions must not
// be trashed. New unpinned regions should be trashed.
{
ShenandoahGCPhase phase_cleanup(ShenandoahPhaseTimings::traversal_gc_sync_pinned);
_heap->sync_pinned_region_status();
}
// Still good? We can now trash the cset, and make final verification
{
ShenandoahGCPhase phase_cleanup(ShenandoahPhaseTimings::traversal_gc_cleanup);