8327387: G1: Refactor region liveness processing after completion of concurrent marking

Reviewed-by: gli, tschatzl
This commit is contained in:
Albert Mingkun Yang 2024-03-11 09:41:35 +00:00
parent f2b5ffdb8e
commit 63dd6d1ac5
4 changed files with 151 additions and 230 deletions

View File

@ -29,6 +29,7 @@
#include "gc/g1/g1BatchedTask.hpp"
#include "gc/g1/g1CardSetMemory.hpp"
#include "gc/g1/g1CollectedHeap.inline.hpp"
#include "gc/g1/g1CollectionSetChooser.hpp"
#include "gc/g1/g1CollectorState.hpp"
#include "gc/g1/g1ConcurrentMark.inline.hpp"
#include "gc/g1/g1ConcurrentMarkThread.inline.hpp"
@ -1176,111 +1177,158 @@ void G1ConcurrentMark::verify_during_pause(G1HeapVerifier::G1VerifyType type,
}
}
class G1UpdateRemSetTrackingBeforeRebuildTask : public WorkerTask {
// Update per-region liveness info based on CM stats. Then, reclaim empty
// regions right away and select certain regions (e.g. sparse ones) for remset
// rebuild.
class G1UpdateRegionLivenessAndSelectForRebuildTask : public WorkerTask {
G1CollectedHeap* _g1h;
G1ConcurrentMark* _cm;
HeapRegionClaimer _hrclaimer;
uint volatile _total_selected_for_rebuild;
G1PrintRegionLivenessInfoClosure _cl;
// Reclaimed empty regions
FreeRegionList _cleanup_list;
class G1UpdateRemSetTrackingBeforeRebuild : public HeapRegionClosure {
struct G1OnRegionClosure : public HeapRegionClosure {
G1CollectedHeap* _g1h;
G1ConcurrentMark* _cm;
// The number of regions actually selected for rebuild.
uint _num_selected_for_rebuild;
G1PrintRegionLivenessInfoClosure* _cl;
size_t _freed_bytes;
uint _num_old_regions_removed;
uint _num_humongous_regions_removed;
FreeRegionList* _local_cleanup_list;
uint _num_regions_selected_for_rebuild; // The number of regions actually selected for rebuild.
G1OnRegionClosure(G1CollectedHeap* g1h,
G1ConcurrentMark* cm,
FreeRegionList* local_cleanup_list) :
_g1h(g1h),
_cm(cm),
_num_selected_for_rebuild(0),
_freed_bytes(0),
_num_old_regions_removed(0),
_num_humongous_regions_removed(0),
_local_cleanup_list(local_cleanup_list) {}
void update_remset_before_rebuild(HeapRegion* hr) {
G1RemSetTrackingPolicy* tracking_policy = _g1h->policy()->remset_tracker();
void reclaim_empty_humongous_region(HeapRegion* hr) {
assert(!hr->has_pinned_objects(), "precondition");
assert(hr->is_starts_humongous(), "precondition");
bool selected_for_rebuild;
if (hr->is_humongous()) {
bool const is_live = _cm->contains_live_object(hr->humongous_start_region()->hrm_index());
selected_for_rebuild = tracking_policy->update_humongous_before_rebuild(hr, is_live);
} else {
size_t const live_bytes = _cm->live_bytes(hr->hrm_index());
selected_for_rebuild = tracking_policy->update_before_rebuild(hr, live_bytes);
}
if (selected_for_rebuild) {
_num_regions_selected_for_rebuild++;
}
_cm->update_top_at_rebuild_start(hr);
}
auto on_humongous_region = [&] (HeapRegion* hr) {
assert(hr->used() > 0, "precondition");
assert(!hr->has_pinned_objects(), "precondition");
assert(hr->is_humongous(), "precondition");
// Distribute the given marked bytes across the humongous object starting
// with hr and note end of marking for these regions.
void distribute_marked_bytes(HeapRegion* hr, size_t marked_bytes) {
// Dead humongous objects (marked_bytes == 0) may have already been unloaded.
assert(marked_bytes == 0 || cast_to_oop(hr->bottom())->size() * HeapWordSize == marked_bytes,
"Marked bytes should either be 0 or the same as humongous object (%zu) but is %zu",
cast_to_oop(hr->bottom())->size() * HeapWordSize, marked_bytes);
auto distribute_bytes = [&] (HeapRegion* r) {
size_t const bytes_to_add = MIN2(HeapRegion::GrainBytes, marked_bytes);
log_trace(gc, marking)("Adding %zu bytes to humongous region %u (%s)",
bytes_to_add, r->hrm_index(), r->get_type_str());
add_marked_bytes_and_note_end(r, bytes_to_add);
marked_bytes -= bytes_to_add;
_num_humongous_regions_removed++;
_freed_bytes += hr->used();
hr->set_containing_set(nullptr);
hr->clear_cardtable();
_g1h->concurrent_mark()->clear_statistics(hr);
_g1h->free_humongous_region(hr, _local_cleanup_list);
};
_g1h->humongous_obj_regions_iterate(hr, distribute_bytes);
_g1h->humongous_obj_regions_iterate(hr, on_humongous_region);
}
void update_marked_bytes(HeapRegion* hr) {
uint const region_idx = hr->hrm_index();
size_t const marked_bytes = _cm->live_bytes(region_idx);
// The marking attributes the object's size completely to the humongous starts
// region. We need to distribute this value across the entire set of regions a
// humongous object spans.
if (hr->is_humongous()) {
assert(hr->is_starts_humongous() || marked_bytes == 0,
"Should not have live bytes %zu in continues humongous region %u (%s)",
marked_bytes, region_idx, hr->get_type_str());
if (hr->is_starts_humongous()) {
distribute_marked_bytes(hr, marked_bytes);
void reclaim_empty_old_region(HeapRegion* hr) {
assert(hr->used() > 0, "precondition");
assert(!hr->has_pinned_objects(), "precondition");
assert(hr->is_old(), "precondition");
_num_old_regions_removed++;
_freed_bytes += hr->used();
hr->set_containing_set(nullptr);
hr->clear_cardtable();
_g1h->concurrent_mark()->clear_statistics(hr);
_g1h->free_region(hr, _local_cleanup_list);
}
bool do_heap_region(HeapRegion* hr) override {
G1RemSetTrackingPolicy* tracker = _g1h->policy()->remset_tracker();
if (hr->is_starts_humongous()) {
// The liveness of this humongous obj decided by either its allocation
// time (allocated after conc-mark-start, i.e. live) or conc-marking.
const bool is_live = hr->top_at_mark_start() == hr->bottom()
|| _cm->contains_live_object(hr->hrm_index());
if (is_live) {
const bool selected_for_rebuild = tracker->update_humongous_before_rebuild(hr);
auto on_humongous_region = [&] (HeapRegion* hr) {
if (selected_for_rebuild) {
_num_selected_for_rebuild++;
}
_cm->update_top_at_rebuild_start(hr);
};
_g1h->humongous_obj_regions_iterate(hr, on_humongous_region);
} else {
reclaim_empty_humongous_region(hr);
}
} else if (hr->is_old()) {
hr->note_end_of_marking(_cm->live_bytes(hr->hrm_index()));
if (hr->live_bytes() != 0) {
if (tracker->update_old_before_rebuild(hr)) {
_num_selected_for_rebuild++;
}
_cm->update_top_at_rebuild_start(hr);
} else {
reclaim_empty_old_region(hr);
}
} else {
log_trace(gc, marking)("Adding %zu bytes to region %u (%s)", marked_bytes, region_idx, hr->get_type_str());
add_marked_bytes_and_note_end(hr, marked_bytes);
}
}
void add_marked_bytes_and_note_end(HeapRegion* hr, size_t marked_bytes) {
hr->note_end_of_marking(marked_bytes);
_cl->do_heap_region(hr);
}
public:
G1UpdateRemSetTrackingBeforeRebuild(G1CollectedHeap* g1h, G1ConcurrentMark* cm, G1PrintRegionLivenessInfoClosure* cl) :
_g1h(g1h), _cm(cm), _cl(cl), _num_regions_selected_for_rebuild(0) { }
virtual bool do_heap_region(HeapRegion* r) {
update_remset_before_rebuild(r);
update_marked_bytes(r);
return false;
}
uint num_selected_for_rebuild() const { return _num_regions_selected_for_rebuild; }
};
public:
G1UpdateRemSetTrackingBeforeRebuildTask(G1CollectedHeap* g1h, G1ConcurrentMark* cm, uint num_workers) :
WorkerTask("G1 Update RemSet Tracking Before Rebuild"),
_g1h(g1h), _cm(cm), _hrclaimer(num_workers), _total_selected_for_rebuild(0), _cl("Post-Marking") { }
G1UpdateRegionLivenessAndSelectForRebuildTask(G1CollectedHeap* g1h,
G1ConcurrentMark* cm,
uint num_workers) :
WorkerTask("G1 Update Region Liveness and Select For Rebuild"),
_g1h(g1h),
_cm(cm),
_hrclaimer(num_workers),
_total_selected_for_rebuild(0),
_cleanup_list("Empty Regions After Mark List") {}
virtual void work(uint worker_id) {
G1UpdateRemSetTrackingBeforeRebuild update_cl(_g1h, _cm, &_cl);
_g1h->heap_region_par_iterate_from_worker_offset(&update_cl, &_hrclaimer, worker_id);
Atomic::add(&_total_selected_for_rebuild, update_cl.num_selected_for_rebuild());
~G1UpdateRegionLivenessAndSelectForRebuildTask() {
if (!_cleanup_list.is_empty()) {
log_debug(gc)("Reclaimed %u empty regions", _cleanup_list.length());
// Now print the empty regions list.
_g1h->hr_printer()->cleanup(&_cleanup_list);
// And actually make them available.
_g1h->prepend_to_freelist(&_cleanup_list);
}
}
void work(uint worker_id) override {
FreeRegionList local_cleanup_list("Local Cleanup List");
G1OnRegionClosure on_region_cl(_g1h, _cm, &local_cleanup_list);
_g1h->heap_region_par_iterate_from_worker_offset(&on_region_cl, &_hrclaimer, worker_id);
Atomic::add(&_total_selected_for_rebuild, on_region_cl._num_selected_for_rebuild);
// Update the old/humongous region sets
_g1h->remove_from_old_gen_sets(on_region_cl._num_old_regions_removed,
on_region_cl._num_humongous_regions_removed);
{
MutexLocker x(G1RareEvent_lock, Mutex::_no_safepoint_check_flag);
_g1h->decrement_summary_bytes(on_region_cl._freed_bytes);
_cleanup_list.add_ordered(&local_cleanup_list);
assert(local_cleanup_list.is_empty(), "post-condition");
}
}
uint total_selected_for_rebuild() const { return _total_selected_for_rebuild; }
// Number of regions for which roughly one thread should be spawned for this work.
static const uint RegionsPerThread = 384;
static uint desired_num_workers(uint num_regions) {
const uint num_regions_per_worker = 384;
return (num_regions + num_regions_per_worker - 1) / num_regions_per_worker;
}
};
class G1UpdateRegionsAfterRebuild : public HeapRegionClosure {
@ -1359,24 +1407,22 @@ void G1ConcurrentMark::remark() {
_g1h->verifier()->verify_bitmap_clear(true /* above_tams_only */);
{
GCTraceTime(Debug, gc, phases) debug("Update Remembered Set Tracking Before Rebuild", _gc_timer_cm);
GCTraceTime(Debug, gc, phases) debug("Select For Rebuild and Reclaim Empty Regions", _gc_timer_cm);
uint const workers_by_capacity = (_g1h->num_regions() + G1UpdateRemSetTrackingBeforeRebuildTask::RegionsPerThread - 1) /
G1UpdateRemSetTrackingBeforeRebuildTask::RegionsPerThread;
uint const num_workers = MIN2(_g1h->workers()->active_workers(), workers_by_capacity);
G1UpdateRemSetTrackingBeforeRebuildTask cl(_g1h, this, num_workers);
G1UpdateRegionLivenessAndSelectForRebuildTask cl(_g1h, this, _g1h->workers()->active_workers());
uint const num_workers = MIN2(G1UpdateRegionLivenessAndSelectForRebuildTask::desired_num_workers(_g1h->num_regions()),
_g1h->workers()->active_workers());
log_debug(gc,ergo)("Running %s using %u workers for %u regions in heap", cl.name(), num_workers, _g1h->num_regions());
_g1h->workers()->run_task(&cl, num_workers);
log_debug(gc, remset, tracking)("Remembered Set Tracking update regions total %u, selected %u",
_g1h->num_regions(), cl.total_selected_for_rebuild());
_g1h->num_regions(), cl.total_selected_for_rebuild());
_needs_remembered_set_rebuild = (cl.total_selected_for_rebuild() > 0);
}
{
GCTraceTime(Debug, gc, phases) debug("Reclaim Empty Regions", _gc_timer_cm);
reclaim_empty_regions();
if (log_is_enabled(Trace, gc, liveness)) {
G1PrintRegionLivenessInfoClosure cl("Post-Marking");
_g1h->heap_region_iterate(&cl);
}
// Potentially, some empty-regions have been reclaimed; make this a
@ -1424,98 +1470,6 @@ void G1ConcurrentMark::remark() {
policy->record_concurrent_mark_remark_end();
}
class G1ReclaimEmptyRegionsTask : public WorkerTask {
class G1ReclaimEmptyRegionsClosure : public HeapRegionClosure {
G1CollectedHeap* _g1h;
size_t _freed_bytes;
FreeRegionList* _local_cleanup_list;
uint _old_regions_removed;
uint _humongous_regions_removed;
public:
G1ReclaimEmptyRegionsClosure(G1CollectedHeap* g1h,
FreeRegionList* local_cleanup_list) :
_g1h(g1h),
_freed_bytes(0),
_local_cleanup_list(local_cleanup_list),
_old_regions_removed(0),
_humongous_regions_removed(0) { }
size_t freed_bytes() { return _freed_bytes; }
uint old_regions_removed() { return _old_regions_removed; }
uint humongous_regions_removed() { return _humongous_regions_removed; }
bool do_heap_region(HeapRegion *hr) {
bool can_reclaim = hr->used() > 0 && hr->live_bytes() == 0 &&
!hr->is_young() && !hr->has_pinned_objects();
if (can_reclaim) {
log_trace(gc, marking)("Reclaimed empty old gen region %u (%s) bot " PTR_FORMAT,
hr->hrm_index(), hr->get_short_type_str(), p2i(hr->bottom()));
_freed_bytes += hr->used();
hr->set_containing_set(nullptr);
if (hr->is_humongous()) {
_humongous_regions_removed++;
_g1h->free_humongous_region(hr, _local_cleanup_list);
} else {
_old_regions_removed++;
_g1h->free_region(hr, _local_cleanup_list);
}
hr->clear_cardtable();
_g1h->concurrent_mark()->clear_statistics(hr);
}
return false;
}
};
G1CollectedHeap* _g1h;
FreeRegionList* _cleanup_list;
HeapRegionClaimer _hrclaimer;
public:
G1ReclaimEmptyRegionsTask(G1CollectedHeap* g1h, FreeRegionList* cleanup_list, uint n_workers) :
WorkerTask("G1 Cleanup"),
_g1h(g1h),
_cleanup_list(cleanup_list),
_hrclaimer(n_workers) {
}
void work(uint worker_id) {
FreeRegionList local_cleanup_list("Local Cleanup List");
G1ReclaimEmptyRegionsClosure cl(_g1h, &local_cleanup_list);
_g1h->heap_region_par_iterate_from_worker_offset(&cl, &_hrclaimer, worker_id);
assert(cl.is_complete(), "Shouldn't have aborted!");
// Now update the old/humongous region sets
_g1h->remove_from_old_gen_sets(cl.old_regions_removed(),
cl.humongous_regions_removed());
{
MutexLocker x(G1RareEvent_lock, Mutex::_no_safepoint_check_flag);
_g1h->decrement_summary_bytes(cl.freed_bytes());
_cleanup_list->add_ordered(&local_cleanup_list);
assert(local_cleanup_list.is_empty(), "post-condition");
}
}
};
void G1ConcurrentMark::reclaim_empty_regions() {
WorkerThreads* workers = _g1h->workers();
FreeRegionList empty_regions_list("Empty Regions After Mark List");
G1ReclaimEmptyRegionsTask cl(_g1h, &empty_regions_list, workers->active_workers());
workers->run_task(&cl);
if (!empty_regions_list.is_empty()) {
log_debug(gc)("Reclaimed %u empty regions", empty_regions_list.length());
// Now print the empty regions list.
_g1h->hr_printer()->cleanup(&empty_regions_list);
// And actually make them available.
_g1h->prepend_to_freelist(&empty_regions_list);
}
}
void G1ConcurrentMark::compute_new_sizes() {
MetaspaceGC::compute_new_size();

View File

@ -463,8 +463,6 @@ class G1ConcurrentMark : public CHeapObj<mtGC> {
void weak_refs_work();
void reclaim_empty_regions();
// After reclaiming empty regions, update heap sizes.
void compute_new_sizes();

View File

@ -54,72 +54,41 @@ void G1RemSetTrackingPolicy::update_at_free(HeapRegion* r) {
/* nothing to do */
}
static void print_before_rebuild(HeapRegion* r, bool selected_for_rebuild, size_t total_live_bytes, size_t live_bytes) {
log_trace(gc, remset, tracking)("Before rebuild region %u "
"(tams: " PTR_FORMAT ") "
"total_live_bytes %zu "
"selected %s "
"(live_bytes %zu "
"type %s)",
r->hrm_index(),
p2i(r->top_at_mark_start()),
total_live_bytes,
BOOL_TO_STR(selected_for_rebuild),
live_bytes,
r->get_type_str());
}
bool G1RemSetTrackingPolicy::update_humongous_before_rebuild(HeapRegion* r, bool is_live) {
bool G1RemSetTrackingPolicy::update_humongous_before_rebuild(HeapRegion* r) {
assert(SafepointSynchronize::is_at_safepoint(), "should be at safepoint");
assert(r->is_humongous(), "Region %u should be humongous", r->hrm_index());
assert(r->is_starts_humongous(), "Region %u should be Humongous", r->hrm_index());
assert(!r->rem_set()->is_updating(), "Remembered set of region %u is updating before rebuild", r->hrm_index());
bool selected_for_rebuild = false;
// For humongous regions, to be of interest for rebuilding the remembered set the following must apply:
// - We always try to update the remembered sets of humongous regions containing
// type arrays as they might have been reset after full gc.
if (is_live && cast_to_oop(r->humongous_start_region()->bottom())->is_typeArray() && !r->rem_set()->is_tracked()) {
r->rem_set()->set_state_updating();
// Humongous regions containing type-array objs are remset-tracked to
// support eager-reclaim. However, their remset state can be reset after
// Full-GC. Try to re-enable remset-tracking for them if possible.
if (cast_to_oop(r->bottom())->is_typeArray() && !r->rem_set()->is_tracked()) {
auto on_humongous_region = [] (HeapRegion* r) {
r->rem_set()->set_state_updating();
};
G1CollectedHeap::heap()->humongous_obj_regions_iterate(r, on_humongous_region);
selected_for_rebuild = true;
}
size_t const live_bytes = is_live ? HeapRegion::GrainBytes : 0;
print_before_rebuild(r, selected_for_rebuild, live_bytes, live_bytes);
return selected_for_rebuild;
}
bool G1RemSetTrackingPolicy::update_before_rebuild(HeapRegion* r, size_t live_bytes_below_tams) {
bool G1RemSetTrackingPolicy::update_old_before_rebuild(HeapRegion* r) {
assert(SafepointSynchronize::is_at_safepoint(), "should be at safepoint");
assert(!r->is_humongous(), "Region %u is humongous", r->hrm_index());
// Only consider updating the remembered set for old gen regions.
if (!r->is_old()) {
return false;
}
assert(r->is_old(), "Region %u should be Old", r->hrm_index());
assert(!r->rem_set()->is_updating(), "Remembered set of region %u is updating before rebuild", r->hrm_index());
size_t live_bytes_above_tams = pointer_delta(r->top(), r->top_at_mark_start()) * HeapWordSize;
size_t total_live_bytes = live_bytes_below_tams + live_bytes_above_tams;
bool selected_for_rebuild = false;
// For old regions, to be of interest for rebuilding the remembered set the following must apply:
// - They must contain some live data in them.
// - Only need to rebuild non-complete remembered sets.
// - Otherwise only add those old gen regions which occupancy is low enough that there
// is a chance that we will ever evacuate them in the mixed gcs.
if ((total_live_bytes > 0) &&
G1CollectionSetChooser::region_occupancy_low_enough_for_evac(total_live_bytes) &&
!r->rem_set()->is_tracked()) {
if (G1CollectionSetChooser::region_occupancy_low_enough_for_evac(r->live_bytes()) &&
!r->rem_set()->is_tracked()) {
r->rem_set()->set_state_updating();
selected_for_rebuild = true;
}
print_before_rebuild(r, selected_for_rebuild, total_live_bytes, live_bytes_below_tams);
return selected_for_rebuild;
}

View File

@ -43,10 +43,10 @@ public:
void update_at_allocate(HeapRegion* r);
// Update remembered set tracking state for humongous regions before we are going to
// rebuild remembered sets. Called at safepoint in the remark pause.
bool update_humongous_before_rebuild(HeapRegion* r, bool is_live);
// Update remembered set tracking state before we are going to rebuild remembered
// sets. Called at safepoint in the remark pause.
bool update_before_rebuild(HeapRegion* r, size_t live_bytes_below_tams);
bool update_humongous_before_rebuild(HeapRegion* r);
// Update remembered set tracking state for old regions before we are going
// to rebuild remembered sets. Called at safepoint in the remark pause.
bool update_old_before_rebuild(HeapRegion* r);
// Update remembered set tracking state after rebuild is complete, i.e. the cleanup
// pause. Called at safepoint.
void update_after_rebuild(HeapRegion* r);