8214237: Join parallel phases post evacuation

Reviewed-by: iwalulya, sjohanss
This commit is contained in:
Thomas Schatzl 2021-05-07 11:38:29 +00:00
parent 2798b0d98a
commit 14f0afe811
26 changed files with 1081 additions and 801 deletions

@ -57,6 +57,9 @@ protected:
void record_work_item(uint worker_id, uint index, size_t count);
public:
// Worker cost for "almost no work" to be done.
static constexpr double AlmostNoWork = 0.01;
G1AbstractSubTask(G1GCPhaseTimes::GCParPhases tag) : _tag(tag) { }
virtual ~G1AbstractSubTask() { }

@ -32,7 +32,6 @@
#include "gc/g1/g1Allocator.inline.hpp"
#include "gc/g1/g1Arguments.hpp"
#include "gc/g1/g1BarrierSet.hpp"
#include "gc/g1/g1CardTableEntryClosure.hpp"
#include "gc/g1/g1CollectedHeap.inline.hpp"
#include "gc/g1/g1CollectionSet.hpp"
#include "gc/g1/g1CollectorState.hpp"
@ -68,6 +67,7 @@
#include "gc/g1/g1ServiceThread.hpp"
#include "gc/g1/g1UncommitRegionTask.hpp"
#include "gc/g1/g1VMOperations.hpp"
#include "gc/g1/g1YoungGCPostEvacuateTasks.hpp"
#include "gc/g1/heapRegion.inline.hpp"
#include "gc/g1/heapRegionRemSet.hpp"
#include "gc/g1/heapRegionSet.inline.hpp"
@ -124,40 +124,6 @@ size_t G1CollectedHeap::_humongous_object_threshold_in_words = 0;
// apply to TLAB allocation, which is not part of this interface: it
// is done by clients of this interface.)
class RedirtyLoggedCardTableEntryClosure : public G1CardTableEntryClosure {
private:
size_t _num_dirtied;
G1CollectedHeap* _g1h;
G1CardTable* _g1_ct;
HeapRegion* region_for_card(CardValue* card_ptr) const {
return _g1h->heap_region_containing(_g1_ct->addr_for(card_ptr));
}
bool will_become_free(HeapRegion* hr) const {
// A region will be freed by free_collection_set if the region is in the
// collection set and has not had an evacuation failure.
return _g1h->is_in_cset(hr) && !hr->evacuation_failed();
}
public:
RedirtyLoggedCardTableEntryClosure(G1CollectedHeap* g1h) : G1CardTableEntryClosure(),
_num_dirtied(0), _g1h(g1h), _g1_ct(g1h->card_table()) { }
void do_card_ptr(CardValue* card_ptr, uint worker_id) {
HeapRegion* hr = region_for_card(card_ptr);
// Should only dirty cards in regions that won't be freed.
if (!will_become_free(hr)) {
*card_ptr = G1CardTable::dirty_card_val();
_num_dirtied++;
}
}
size_t num_dirtied() const { return _num_dirtied; }
};
void G1RegionMappingChangedListener::reset_from_card_cache(uint start_idx, size_t num_regions) {
HeapRegionRemSet::invalidate_from_card_cache(start_idx, num_regions);
}
@ -174,6 +140,12 @@ Tickspan G1CollectedHeap::run_task_timed(AbstractGangTask* task) {
return Ticks::now() - start;
}
void G1CollectedHeap::run_batch_task(G1BatchedGangTask* cl) {
uint num_workers = MAX2(1u, MIN2(cl->num_workers_estimate(), workers()->active_workers()));
cl->set_max_workers(num_workers);
workers()->run_task(cl, num_workers);
}
HeapRegion* G1CollectedHeap::new_heap_region(uint hrs_index,
MemRegion mr) {
return new HeapRegion(hrs_index, bot(), mr);
@ -1426,7 +1398,8 @@ G1CollectedHeap::G1CollectedHeap() :
_expand_heap_after_alloc_failure(true),
_g1mm(NULL),
_humongous_reclaim_candidates(),
_has_humongous_reclaim_candidates(false),
_num_humongous_objects(0),
_num_humongous_reclaim_candidates(0),
_hr_printer(),
_collector_state(),
_old_marking_cycles_started(0),
@ -3086,28 +3059,6 @@ void G1CollectedHeap::do_collection_pause_at_safepoint_helper(double target_paus
}
}
void G1CollectedHeap::remove_self_forwarding_pointers(G1RedirtyCardsQueueSet* rdcqs) {
uint num_workers = MIN2(workers()->active_workers(), num_regions_failed_evacuation());
G1ParRemoveSelfForwardPtrsTask cl(rdcqs);
log_debug(gc, ergo)("Running %s using %u workers for %u failed regions",
cl.name(), num_workers, num_regions_failed_evacuation());
workers()->run_task(&cl, num_workers);
assert(cl.num_failed_regions() == num_regions_failed_evacuation(),
"Removed regions %u inconsistent with expected %u",
cl.num_failed_regions(), num_regions_failed_evacuation());
}
void G1CollectedHeap::restore_after_evac_failure(G1RedirtyCardsQueueSet* rdcqs) {
double remove_self_forwards_start = os::elapsedTime();
remove_self_forwarding_pointers(rdcqs);
_preserved_marks_set.restore(workers());
phase_times()->record_evac_fail_remove_self_forwards((os::elapsedTime() - remove_self_forwards_start) * 1000.0);
}
void G1CollectedHeap::preserve_mark_during_evac_failure(uint worker_id, oop obj, markWord m) {
_evacuation_failed_info_array[worker_id].register_copy_failure(obj->size());
_preserved_marks_set.get(worker_id)->push_if_necessary(obj, m);
@ -3187,53 +3138,6 @@ void G1CollectedHeap::string_dedup_cleaning(BoolObjectClosure* is_alive,
workers()->run_task(&cl);
}
class G1RedirtyLoggedCardsTask : public AbstractGangTask {
private:
G1RedirtyCardsQueueSet* _qset;
G1CollectedHeap* _g1h;
BufferNode* volatile _nodes;
void par_apply(RedirtyLoggedCardTableEntryClosure* cl, uint worker_id) {
size_t buffer_size = _qset->buffer_size();
BufferNode* next = Atomic::load(&_nodes);
while (next != NULL) {
BufferNode* node = next;
next = Atomic::cmpxchg(&_nodes, node, node->next());
if (next == node) {
cl->apply_to_buffer(node, buffer_size, worker_id);
next = node->next();
}
}
}
public:
G1RedirtyLoggedCardsTask(G1RedirtyCardsQueueSet* qset, G1CollectedHeap* g1h) :
AbstractGangTask("Redirty Cards"),
_qset(qset), _g1h(g1h), _nodes(qset->all_completed_buffers()) { }
virtual void work(uint worker_id) {
G1GCPhaseTimes* p = _g1h->phase_times();
G1GCParPhaseTimesTracker x(p, G1GCPhaseTimes::RedirtyCards, worker_id);
RedirtyLoggedCardTableEntryClosure cl(_g1h);
par_apply(&cl, worker_id);
p->record_thread_work_item(G1GCPhaseTimes::RedirtyCards, worker_id, cl.num_dirtied());
}
};
void G1CollectedHeap::redirty_logged_cards(G1RedirtyCardsQueueSet* rdcqs) {
double redirty_logged_cards_start = os::elapsedTime();
G1RedirtyLoggedCardsTask redirty_task(rdcqs, this);
workers()->run_task(&redirty_task);
G1DirtyCardQueueSet& dcq = G1BarrierSet::dirty_card_queue_set();
dcq.merge_bufferlists(rdcqs);
phase_times()->record_redirty_logged_cards_time_ms((os::elapsedTime() - redirty_logged_cards_start) * 1000.0);
}
// Weak Reference Processing support
bool G1STWIsAliveClosure::do_object_b(oop p) {
@ -3513,18 +3417,24 @@ void G1CollectedHeap::make_pending_list_reachable() {
}
}
void G1CollectedHeap::merge_per_thread_state_info(G1ParScanThreadStateSet* per_thread_states) {
Ticks start = Ticks::now();
per_thread_states->flush();
phase_times()->record_or_add_time_secs(G1GCPhaseTimes::MergePSS, 0 /* worker_id */, (Ticks::now() - start).seconds());
static bool do_humongous_object_logging() {
return log_is_enabled(Debug, gc, humongous);
}
bool G1CollectedHeap::should_do_eager_reclaim() const {
// As eager reclaim logging also gives information about humongous objects in
// the heap in general, always do the eager reclaim pass even without known
// candidates.
return (G1EagerReclaimHumongousObjects &&
(has_humongous_reclaim_candidates() || do_humongous_object_logging()));
}
class G1PrepareEvacuationTask : public AbstractGangTask {
class G1PrepareRegionsClosure : public HeapRegionClosure {
G1CollectedHeap* _g1h;
G1PrepareEvacuationTask* _parent_task;
size_t _worker_humongous_total;
size_t _worker_humongous_candidates;
uint _worker_humongous_total;
uint _worker_humongous_candidates;
bool humongous_region_is_candidate(HeapRegion* region) const {
assert(region->is_starts_humongous(), "Must start a humongous object");
@ -3622,8 +3532,8 @@ class G1PrepareEvacuationTask : public AbstractGangTask {
G1CollectedHeap* _g1h;
HeapRegionClaimer _claimer;
volatile size_t _humongous_total;
volatile size_t _humongous_candidates;
volatile uint _humongous_total;
volatile uint _humongous_candidates;
public:
G1PrepareEvacuationTask(G1CollectedHeap* g1h) :
AbstractGangTask("Prepare Evacuation"),
@ -3632,28 +3542,24 @@ public:
_humongous_total(0),
_humongous_candidates(0) { }
~G1PrepareEvacuationTask() {
_g1h->set_has_humongous_reclaim_candidate(_humongous_candidates > 0);
}
void work(uint worker_id) {
G1PrepareRegionsClosure cl(_g1h, this);
_g1h->heap_region_par_iterate_from_worker_offset(&cl, &_claimer, worker_id);
}
void add_humongous_candidates(size_t candidates) {
void add_humongous_candidates(uint candidates) {
Atomic::add(&_humongous_candidates, candidates);
}
void add_humongous_total(size_t total) {
void add_humongous_total(uint total) {
Atomic::add(&_humongous_total, total);
}
size_t humongous_candidates() {
uint humongous_candidates() {
return _humongous_candidates;
}
size_t humongous_total() {
uint humongous_total() {
return _humongous_total;
}
};
@ -3681,9 +3587,9 @@ void G1CollectedHeap::pre_evacuate_collection_set(G1EvacuationInfo& evacuation_i
G1PrepareEvacuationTask g1_prep_task(this);
Tickspan task_time = run_task_timed(&g1_prep_task);
phase_times()->record_register_regions(task_time.seconds() * 1000.0,
g1_prep_task.humongous_total(),
g1_prep_task.humongous_candidates());
phase_times()->record_register_regions(task_time.seconds() * 1000.0);
_num_humongous_objects = g1_prep_task.humongous_total();
_num_humongous_reclaim_candidates = g1_prep_task.humongous_candidates();
}
assert(_verifier->check_region_attr_table(), "Inconsistency in the region attributes table.");
@ -3927,8 +3833,6 @@ void G1CollectedHeap::post_evacuate_collection_set(G1EvacuationInfo& evacuation_
G1ParScanThreadStateSet* per_thread_states) {
G1GCPhaseTimes* p = phase_times();
rem_set()->cleanup_after_scan_heap_roots();
// Process any discovered reference objects - we have
// to do this _before_ we retire the GC alloc regions
// as we may have to copy some 'reachable' referent
@ -3952,47 +3856,11 @@ void G1CollectedHeap::post_evacuate_collection_set(G1EvacuationInfo& evacuation_
_allocator->release_gc_alloc_regions(evacuation_info);
if (evacuation_failed()) {
restore_after_evac_failure(rdcqs);
post_evacuate_cleanup_1(per_thread_states, rdcqs);
// Reset the G1EvacuationFailureALot counters and flags
NOT_PRODUCT(reset_evacuation_should_fail();)
post_evacuate_cleanup_2(&_preserved_marks_set, rdcqs, &evacuation_info, per_thread_states->surviving_young_words());
double recalculate_used_start = os::elapsedTime();
set_used(recalculate_used());
p->record_evac_fail_recalc_used_time((os::elapsedTime() - recalculate_used_start) * 1000.0);
if (_archive_allocator != NULL) {
_archive_allocator->clear_used();
}
for (uint i = 0; i < ParallelGCThreads; i++) {
if (_evacuation_failed_info_array[i].has_failed()) {
_gc_tracer_stw->report_evacuation_failed(_evacuation_failed_info_array[i]);
}
}
} else {
// The "used" of the the collection set have already been subtracted
// when they were freed. Add in the bytes used.
increase_used(_bytes_used_during_gc);
}
_preserved_marks_set.assert_empty();
merge_per_thread_state_info(per_thread_states);
// Reset and re-enable the hot card cache.
// Note the counts for the cards in the regions in the
// collection set are reset when the collection set is freed.
_hot_card_cache->reset_hot_cache();
_hot_card_cache->set_use_cache(true);
purge_code_root_memory();
redirty_logged_cards(rdcqs);
free_collection_set(&_collection_set, evacuation_info, per_thread_states->surviving_young_words());
eagerly_reclaim_humongous_regions();
assert_used_and_recalculate_used_equal(this);
rebuild_free_region_list();
@ -4001,11 +3869,6 @@ void G1CollectedHeap::post_evacuate_collection_set(G1EvacuationInfo& evacuation_
evacuation_info.set_collectionset_used_before(collection_set()->bytes_used_before());
evacuation_info.set_bytes_used(_bytes_used_during_gc);
#if COMPILER2_OR_JVMCI
double start = os::elapsedTime();
DerivedPointerTable::update_pointers();
phase_times()->record_derived_pointer_table_update_time((os::elapsedTime() - start) * 1000.0);
#endif
policy()->print_age_table();
}
@ -4074,415 +3937,34 @@ void G1CollectedHeap::decrement_summary_bytes(size_t bytes) {
decrease_used(bytes);
}
class G1FreeCollectionSetTask : public AbstractGangTask {
// Helper class to keep statistics for the collection set freeing
class FreeCSetStats {
size_t _before_used_bytes; // Usage in regions successfully evacutate
size_t _after_used_bytes; // Usage in regions failing evacuation
size_t _bytes_allocated_in_old_since_last_gc; // Size of young regions turned into old
size_t _failure_used_words; // Live size in failed regions
size_t _failure_waste_words; // Wasted size in failed regions
size_t _rs_length; // Remembered set size
uint _regions_freed; // Number of regions freed
public:
FreeCSetStats() :
_before_used_bytes(0),
_after_used_bytes(0),
_bytes_allocated_in_old_since_last_gc(0),
_failure_used_words(0),
_failure_waste_words(0),
_rs_length(0),
_regions_freed(0) { }
void merge_stats(FreeCSetStats* other) {
assert(other != NULL, "invariant");
_before_used_bytes += other->_before_used_bytes;
_after_used_bytes += other->_after_used_bytes;
_bytes_allocated_in_old_since_last_gc += other->_bytes_allocated_in_old_since_last_gc;
_failure_used_words += other->_failure_used_words;
_failure_waste_words += other->_failure_waste_words;
_rs_length += other->_rs_length;
_regions_freed += other->_regions_freed;
}
void report(G1CollectedHeap* g1h, G1EvacuationInfo* evacuation_info) {
evacuation_info->set_regions_freed(_regions_freed);
evacuation_info->increment_collectionset_used_after(_after_used_bytes);
g1h->decrement_summary_bytes(_before_used_bytes);
g1h->alloc_buffer_stats(G1HeapRegionAttr::Old)->add_failure_used_and_waste(_failure_used_words, _failure_waste_words);
G1Policy *policy = g1h->policy();
policy->old_gen_alloc_tracker()->add_allocated_bytes_since_last_gc(_bytes_allocated_in_old_since_last_gc);
policy->record_rs_length(_rs_length);
policy->cset_regions_freed();
}
void account_failed_region(HeapRegion* r) {
size_t used_words = r->marked_bytes() / HeapWordSize;
_failure_used_words += used_words;
_failure_waste_words += HeapRegion::GrainWords - used_words;
_after_used_bytes += r->used();
// When moving a young gen region to old gen, we "allocate" that whole
// region there. This is in addition to any already evacuated objects.
// Notify the policy about that. Old gen regions do not cause an
// additional allocation: both the objects still in the region and the
// ones already moved are accounted for elsewhere.
if (r->is_young()) {
_bytes_allocated_in_old_since_last_gc += HeapRegion::GrainBytes;
}
}
void account_evacuated_region(HeapRegion* r) {
_before_used_bytes += r->used();
_regions_freed += 1;
}
void account_rs_length(HeapRegion* r) {
_rs_length += r->rem_set()->occupied();
}
};
// Closure applied to all regions in the collection set.
class FreeCSetClosure : public HeapRegionClosure {
// Helper to send JFR events for regions.
class JFREventForRegion {
EventGCPhaseParallel _event;
public:
JFREventForRegion(HeapRegion* region, uint worker_id) : _event() {
_event.set_gcId(GCId::current());
_event.set_gcWorkerId(worker_id);
if (region->is_young()) {
_event.set_name(G1GCPhaseTimes::phase_name(G1GCPhaseTimes::YoungFreeCSet));
} else {
_event.set_name(G1GCPhaseTimes::phase_name(G1GCPhaseTimes::NonYoungFreeCSet));
}
}
~JFREventForRegion() {
_event.commit();
}
};
// Helper to do timing for region work.
class TimerForRegion {
Tickspan& _time;
Ticks _start_time;
public:
TimerForRegion(Tickspan& time) : _time(time), _start_time(Ticks::now()) { }
~TimerForRegion() {
_time += Ticks::now() - _start_time;
}
};
// FreeCSetClosure members
G1CollectedHeap* _g1h;
const size_t* _surviving_young_words;
uint _worker_id;
Tickspan _young_time;
Tickspan _non_young_time;
FreeCSetStats* _stats;
void assert_in_cset(HeapRegion* r) {
assert(r->young_index_in_cset() != 0 &&
(uint)r->young_index_in_cset() <= _g1h->collection_set()->young_region_length(),
"Young index %u is wrong for region %u of type %s with %u young regions",
r->young_index_in_cset(), r->hrm_index(), r->get_type_str(), _g1h->collection_set()->young_region_length());
}
void handle_evacuated_region(HeapRegion* r) {
assert(!r->is_empty(), "Region %u is an empty region in the collection set.", r->hrm_index());
stats()->account_evacuated_region(r);
// Free the region and and its remembered set.
_g1h->free_region(r, NULL);
_g1h->hr_printer()->cleanup(r);
}
void handle_failed_region(HeapRegion* r) {
// Do some allocation statistics accounting. Regions that failed evacuation
// are always made old, so there is no need to update anything in the young
// gen statistics, but we need to update old gen statistics.
stats()->account_failed_region(r);
// Update the region state due to the failed evacuation.
r->handle_evacuation_failure();
// Add region to old set, need to hold lock.
MutexLocker x(OldSets_lock, Mutex::_no_safepoint_check_flag);
_g1h->old_set_add(r);
}
Tickspan& timer_for_region(HeapRegion* r) {
return r->is_young() ? _young_time : _non_young_time;
}
FreeCSetStats* stats() {
return _stats;
}
public:
FreeCSetClosure(const size_t* surviving_young_words,
uint worker_id,
FreeCSetStats* stats) :
HeapRegionClosure(),
_g1h(G1CollectedHeap::heap()),
_surviving_young_words(surviving_young_words),
_worker_id(worker_id),
_young_time(),
_non_young_time(),
_stats(stats) { }
virtual bool do_heap_region(HeapRegion* r) {
assert(r->in_collection_set(), "Invariant: %u missing from CSet", r->hrm_index());
JFREventForRegion event(r, _worker_id);
TimerForRegion timer(timer_for_region(r));
_g1h->clear_region_attr(r);
stats()->account_rs_length(r);
if (r->is_young()) {
assert_in_cset(r);
r->record_surv_words_in_group(_surviving_young_words[r->young_index_in_cset()]);
}
if (r->evacuation_failed()) {
handle_failed_region(r);
} else {
handle_evacuated_region(r);
}
assert(!_g1h->is_on_master_free_list(r), "sanity");
return false;
}
void report_timing(Tickspan parallel_time) {
G1GCPhaseTimes* pt = _g1h->phase_times();
pt->record_time_secs(G1GCPhaseTimes::ParFreeCSet, _worker_id, parallel_time.seconds());
if (_young_time.value() > 0) {
pt->record_time_secs(G1GCPhaseTimes::YoungFreeCSet, _worker_id, _young_time.seconds());
}
if (_non_young_time.value() > 0) {
pt->record_time_secs(G1GCPhaseTimes::NonYoungFreeCSet, _worker_id, _non_young_time.seconds());
}
}
};
// G1FreeCollectionSetTask members
G1CollectedHeap* _g1h;
G1EvacuationInfo* _evacuation_info;
FreeCSetStats* _worker_stats;
HeapRegionClaimer _claimer;
const size_t* _surviving_young_words;
uint _active_workers;
FreeCSetStats* worker_stats(uint worker) {
return &_worker_stats[worker];
}
void report_statistics() {
// Merge the accounting
FreeCSetStats total_stats;
for (uint worker = 0; worker < _active_workers; worker++) {
total_stats.merge_stats(worker_stats(worker));
}
total_stats.report(_g1h, _evacuation_info);
}
public:
G1FreeCollectionSetTask(G1EvacuationInfo* evacuation_info, const size_t* surviving_young_words, uint active_workers) :
AbstractGangTask("G1 Free Collection Set"),
_g1h(G1CollectedHeap::heap()),
_evacuation_info(evacuation_info),
_worker_stats(NEW_C_HEAP_ARRAY(FreeCSetStats, active_workers, mtGC)),
_claimer(active_workers),
_surviving_young_words(surviving_young_words),
_active_workers(active_workers) {
for (uint worker = 0; worker < active_workers; worker++) {
::new (&_worker_stats[worker]) FreeCSetStats();
}
}
~G1FreeCollectionSetTask() {
Ticks serial_time = Ticks::now();
report_statistics();
for (uint worker = 0; worker < _active_workers; worker++) {
_worker_stats[worker].~FreeCSetStats();
}
FREE_C_HEAP_ARRAY(FreeCSetStats, _worker_stats);
_g1h->phase_times()->record_serial_free_cset_time_ms((Ticks::now() - serial_time).seconds() * 1000.0);
}
virtual void work(uint worker_id) {
EventGCPhaseParallel event;
Ticks start = Ticks::now();
FreeCSetClosure cl(_surviving_young_words, worker_id, worker_stats(worker_id));
_g1h->collection_set_par_iterate_all(&cl, &_claimer, worker_id);
// Report the total parallel time along with some more detailed metrics.
cl.report_timing(Ticks::now() - start);
event.commit(GCId::current(), worker_id, G1GCPhaseTimes::phase_name(G1GCPhaseTimes::ParFreeCSet));
}
};
void G1CollectedHeap::free_collection_set(G1CollectionSet* collection_set, G1EvacuationInfo& evacuation_info, const size_t* surviving_young_words) {
_eden.clear();
// The free collections set is split up in two tasks, the first
// frees the collection set and records what regions are free,
// and the second one rebuilds the free list. This proved to be
// more efficient than adding a sorted list to another.
Ticks free_cset_start_time = Ticks::now();
void G1CollectedHeap::post_evacuate_cleanup_1(G1ParScanThreadStateSet* per_thread_states,
G1RedirtyCardsQueueSet* rdcqs) {
Ticks start = Ticks::now();
{
uint const num_cs_regions = _collection_set.region_length();
uint const num_workers = clamp(num_cs_regions, 1u, workers()->active_workers());
G1FreeCollectionSetTask cl(&evacuation_info, surviving_young_words, num_workers);
log_debug(gc, ergo)("Running %s using %u workers for collection set length %u (%u)",
cl.name(), num_workers, num_cs_regions, num_regions());
workers()->run_task(&cl, num_workers);
G1PostEvacuateCollectionSetCleanupTask1 cl(per_thread_states, rdcqs);
run_batch_task(&cl);
}
Ticks free_cset_end_time = Ticks::now();
phase_times()->record_total_free_cset_time_ms((free_cset_end_time - free_cset_start_time).seconds() * 1000.0);
collection_set->clear();
phase_times()->record_post_evacuate_cleanup_task_1_time((Ticks::now() - start).seconds() * 1000.0);
}
class G1FreeHumongousRegionClosure : public HeapRegionClosure {
uint _humongous_objects_reclaimed;
uint _humongous_regions_reclaimed;
size_t _freed_bytes;
public:
G1FreeHumongousRegionClosure() :
_humongous_objects_reclaimed(0), _humongous_regions_reclaimed(0), _freed_bytes(0) {
void G1CollectedHeap::post_evacuate_cleanup_2(PreservedMarksSet* preserved_marks,
G1RedirtyCardsQueueSet* rdcqs,
G1EvacuationInfo* evacuation_info,
const size_t* surviving_young_words) {
Ticks start = Ticks::now();
{
G1PostEvacuateCollectionSetCleanupTask2 cl(preserved_marks, rdcqs, evacuation_info, surviving_young_words);
run_batch_task(&cl);
}
phase_times()->record_post_evacuate_cleanup_task_2_time((Ticks::now() - start).seconds() * 1000.0);
}
virtual bool do_heap_region(HeapRegion* r) {
if (!r->is_starts_humongous()) {
return false;
}
void G1CollectedHeap::clear_eden() {
_eden.clear();
}
G1CollectedHeap* g1h = G1CollectedHeap::heap();
oop obj = cast_to_oop(r->bottom());
G1CMBitMap* next_bitmap = g1h->concurrent_mark()->next_mark_bitmap();
// The following checks whether the humongous object is live are sufficient.
// The main additional check (in addition to having a reference from the roots
// or the young gen) is whether the humongous object has a remembered set entry.
//
// A humongous object cannot be live if there is no remembered set for it
// because:
// - there can be no references from within humongous starts regions referencing
// the object because we never allocate other objects into them.
// (I.e. there are no intra-region references that may be missed by the
// remembered set)
// - as soon there is a remembered set entry to the humongous starts region
// (i.e. it has "escaped" to an old object) this remembered set entry will stay
// until the end of a concurrent mark.
//
// It is not required to check whether the object has been found dead by marking
// or not, in fact it would prevent reclamation within a concurrent cycle, as
// all objects allocated during that time are considered live.
// SATB marking is even more conservative than the remembered set.
// So if at this point in the collection there is no remembered set entry,
// nobody has a reference to it.
// At the start of collection we flush all refinement logs, and remembered sets
// are completely up-to-date wrt to references to the humongous object.
//
// Other implementation considerations:
// - never consider object arrays at this time because they would pose
// considerable effort for cleaning up the the remembered sets. This is
// required because stale remembered sets might reference locations that
// are currently allocated into.
uint region_idx = r->hrm_index();
if (!g1h->is_humongous_reclaim_candidate(region_idx) ||
!r->rem_set()->is_empty()) {
log_debug(gc, humongous)("Live humongous region %u object size " SIZE_FORMAT " start " PTR_FORMAT " with remset " SIZE_FORMAT " code roots " SIZE_FORMAT " is marked %d reclaim candidate %d type array %d",
region_idx,
(size_t)obj->size() * HeapWordSize,
p2i(r->bottom()),
r->rem_set()->occupied(),
r->rem_set()->strong_code_roots_list_length(),
next_bitmap->is_marked(r->bottom()),
g1h->is_humongous_reclaim_candidate(region_idx),
obj->is_typeArray()
);
return false;
}
guarantee(obj->is_typeArray(),
"Only eagerly reclaiming type arrays is supported, but the object "
PTR_FORMAT " is not.", p2i(r->bottom()));
log_debug(gc, humongous)("Dead humongous region %u object size " SIZE_FORMAT " start " PTR_FORMAT " with remset " SIZE_FORMAT " code roots " SIZE_FORMAT " is marked %d reclaim candidate %d type array %d",
region_idx,
(size_t)obj->size() * HeapWordSize,
p2i(r->bottom()),
r->rem_set()->occupied(),
r->rem_set()->strong_code_roots_list_length(),
next_bitmap->is_marked(r->bottom()),
g1h->is_humongous_reclaim_candidate(region_idx),
obj->is_typeArray()
);
G1ConcurrentMark* const cm = g1h->concurrent_mark();
cm->humongous_object_eagerly_reclaimed(r);
assert(!cm->is_marked_in_prev_bitmap(obj) && !cm->is_marked_in_next_bitmap(obj),
"Eagerly reclaimed humongous region %u should not be marked at all but is in prev %s next %s",
region_idx,
BOOL_TO_STR(cm->is_marked_in_prev_bitmap(obj)),
BOOL_TO_STR(cm->is_marked_in_next_bitmap(obj)));
_humongous_objects_reclaimed++;
do {
HeapRegion* next = g1h->next_region_in_humongous(r);
_freed_bytes += r->used();
r->set_containing_set(NULL);
_humongous_regions_reclaimed++;
g1h->free_humongous_region(r, NULL);
g1h->hr_printer()->cleanup(r);
r = next;
} while (r != NULL);
return false;
}
uint humongous_objects_reclaimed() {
return _humongous_objects_reclaimed;
}
uint humongous_regions_reclaimed() {
return _humongous_regions_reclaimed;
}
size_t bytes_freed() const {
return _freed_bytes;
}
};
void G1CollectedHeap::eagerly_reclaim_humongous_regions() {
assert_at_safepoint_on_vm_thread();
if (!G1EagerReclaimHumongousObjects ||
(!_has_humongous_reclaim_candidates && !log_is_enabled(Debug, gc, humongous))) {
phase_times()->record_fast_reclaim_humongous_time_ms(0.0, 0);
return;
}
double start_time = os::elapsedTime();
G1FreeHumongousRegionClosure cl;
heap_region_iterate(&cl);
remove_from_old_gen_sets(0, 0, cl.humongous_regions_reclaimed());
decrement_summary_bytes(cl.bytes_freed());
phase_times()->record_fast_reclaim_humongous_time_ms((os::elapsedTime() - start_time) * 1000.0,
cl.humongous_objects_reclaimed());
void G1CollectedHeap::clear_collection_set() {
collection_set()->clear();
}
void G1CollectedHeap::rebuild_free_region_list() {
@ -4848,11 +4330,35 @@ void G1CollectedHeap::unregister_nmethod(nmethod* nm) {
nm->oops_do(&reg_cl, true);
}
void G1CollectedHeap::update_used_after_gc() {
if (evacuation_failed()) {
// Reset the G1EvacuationFailureALot counters and flags
NOT_PRODUCT(reset_evacuation_should_fail();)
set_used(recalculate_used());
if (_archive_allocator != NULL) {
_archive_allocator->clear_used();
}
for (uint i = 0; i < ParallelGCThreads; i++) {
if (_evacuation_failed_info_array[i].has_failed()) {
_gc_tracer_stw->report_evacuation_failed(_evacuation_failed_info_array[i]);
}
}
} else {
// The "used" of the the collection set have already been subtracted
// when they were freed. Add in the bytes used.
increase_used(_bytes_used_during_gc);
}
}
void G1CollectedHeap::reset_hot_card_cache() {
_hot_card_cache->reset_hot_cache();
_hot_card_cache->set_use_cache(true);
}
void G1CollectedHeap::purge_code_root_memory() {
double purge_start = os::elapsedTime();
G1CodeRootSet::purge();
double purge_time_ms = (os::elapsedTime() - purge_start) * 1000.0;
phase_times()->record_strong_code_root_purge_time(purge_time_ms);
}
class RebuildStrongCodeRootClosure: public CodeBlobClosure {

@ -74,6 +74,7 @@ class ObjectClosure;
class SpaceClosure;
class CompactibleSpaceClosure;
class Space;
class G1BatchedGangTask;
class G1CardTableEntryClosure;
class G1CollectionSet;
class G1Policy;
@ -171,8 +172,6 @@ private:
HeapRegionSet _archive_set;
HeapRegionSet _humongous_set;
void eagerly_reclaim_humongous_regions();
void rebuild_free_region_list();
// Start a new incremental collection set for the next pause.
void start_new_collection_set();
@ -245,9 +244,9 @@ private:
// candidates removed from the set as they are found reachable from
// roots or the young generation.
class HumongousReclaimCandidates : public G1BiasedMappedArray<bool> {
protected:
protected:
bool default_value() const { return false; }
public:
public:
void clear() { G1BiasedMappedArray<bool>::clear(); }
void set_candidate(uint region, bool value) {
set_by_index(region, value);
@ -258,9 +257,16 @@ private:
};
HumongousReclaimCandidates _humongous_reclaim_candidates;
// Stores whether during humongous object registration we found candidate regions.
// If not, we can skip a few steps.
bool _has_humongous_reclaim_candidates;
uint _num_humongous_objects; // Current amount of (all) humongous objects found in the heap.
uint _num_humongous_reclaim_candidates; // Number of humongous object eager reclaim candidates.
public:
uint num_humongous_objects() const { return _num_humongous_objects; }
uint num_humongous_reclaim_candidates() const { return _num_humongous_reclaim_candidates; }
bool has_humongous_reclaim_candidates() const { return _num_humongous_reclaim_candidates > 0; }
bool should_do_eager_reclaim() const;
private:
G1HRPrinter _hr_printer;
@ -546,10 +552,6 @@ private:
// to discover.
void make_pending_list_reachable();
// Merges the information gathered on a per-thread basis for all worker threads
// during GC into global variables.
void merge_per_thread_state_info(G1ParScanThreadStateSet* per_thread_states);
void verify_numa_regions(const char* desc);
public:
@ -560,6 +562,8 @@ public:
// Runs the given AbstractGangTask with the current active workers,
// returning the total time taken.
Tickspan run_task_timed(AbstractGangTask* task);
// Run the given batch task using the work gang.
void run_batch_task(G1BatchedGangTask* cl);
G1Allocator* allocator() {
return _allocator;
@ -608,7 +612,6 @@ public:
// These are only valid for starts_humongous regions.
inline void set_humongous_reclaim_candidate(uint region, bool value);
inline bool is_humongous_reclaim_candidate(uint region);
inline void set_has_humongous_reclaim_candidate(bool value);
// Remove from the reclaim candidate set. Also remove from the
// collection set so that later encounters avoid the slow path.
@ -833,9 +836,16 @@ public:
// The g1 remembered set of the heap.
G1RemSet* _rem_set;
// After a collection pause, convert the regions in the collection set into free
// regions.
void free_collection_set(G1CollectionSet* collection_set, G1EvacuationInfo& evacuation_info, const size_t* surviving_young_words);
void post_evacuate_cleanup_1(G1ParScanThreadStateSet* per_thread_states,
G1RedirtyCardsQueueSet* rdcqs);
void post_evacuate_cleanup_2(PreservedMarksSet* preserved_marks,
G1RedirtyCardsQueueSet* rdcqs,
G1EvacuationInfo* evacuation_info,
const size_t* surviving_young_words);
// After a collection pause, reset eden and the collection set.
void clear_eden();
void clear_collection_set();
// Abandon the current collection set without recording policy
// statistics or updating free lists.
@ -856,14 +866,6 @@ public:
EvacuationFailedInfo* _evacuation_failed_info_array;
// Failed evacuations cause some logical from-space objects to have
// forwarding pointers to themselves. Reset them.
void remove_self_forwarding_pointers(G1RedirtyCardsQueueSet* rdcqs);
// Restore the objects in the regions in the collection set after an
// evacuation failure.
void restore_after_evac_failure(G1RedirtyCardsQueueSet* rdcqs);
PreservedMarksSet _preserved_marks_set;
// Preserve the mark of "obj", if necessary, in preparation for its mark
@ -1385,6 +1387,13 @@ public:
// No nmethod verification implemented.
virtual void verify_nmethod(nmethod* nm) {}
// Recalculate amount of used memory after GC. Must be called after all allocation
// has finished.
void update_used_after_gc();
// Reset and re-enable the hot card cache.
// Note the counts for the cards in the regions in the
// collection set are reset when the collection set is freed.
void reset_hot_card_cache();
// Free up superfluous code root memory.
void purge_code_root_memory();
@ -1400,9 +1409,6 @@ public:
// Performs cleaning of data structures after class unloading.
void complete_cleaning(BoolObjectClosure* is_alive, bool class_unloading_occurred);
// Redirty logged cards in the refinement queue.
void redirty_logged_cards(G1RedirtyCardsQueueSet* rdcqs);
// Verification
// Deduplicate the string

@ -312,10 +312,6 @@ inline bool G1CollectedHeap::is_humongous_reclaim_candidate(uint region) {
return _humongous_reclaim_candidates.is_candidate(region);
}
inline void G1CollectedHeap::set_has_humongous_reclaim_candidate(bool value) {
_has_humongous_reclaim_candidates = value;
}
inline void G1CollectedHeap::set_humongous_is_live(oop obj) {
uint region = addr_to_region(cast_from_oop<HeapWord*>(obj));
// Clear the flag in the humongous_reclaim_candidates table. Also

@ -502,7 +502,7 @@ static void clear_mark_if_set(G1CMBitMap* bitmap, HeapWord* addr) {
}
void G1ConcurrentMark::humongous_object_eagerly_reclaimed(HeapRegion* r) {
assert_at_safepoint_on_vm_thread();
assert_at_safepoint();
// Need to clear all mark bits of the humongous object.
clear_mark_if_set(_prev_mark_bitmap, r->bottom());

@ -108,6 +108,17 @@ G1GCPhaseTimes::G1GCPhaseTimes(STWGCTimer* gc_timer, uint max_gc_threads) :
_gc_par_phases[GCWorkerTotal] = new WorkerDataArray<double>("GCWorkerTotal", "GC Worker Total (ms):", max_gc_threads);
_gc_par_phases[GCWorkerEnd] = new WorkerDataArray<double>("GCWorkerEnd", "GC Worker End (ms):", max_gc_threads);
_gc_par_phases[Other] = new WorkerDataArray<double>("Other", "GC Worker Other (ms):", max_gc_threads);
_gc_par_phases[MergePSS] = new WorkerDataArray<double>("MergePSS", "Merge Per-Thread State (ms):", max_gc_threads);
_gc_par_phases[RemoveSelfForwardingPtr] = new WorkerDataArray<double>("RemoveSelfForwardingPtr", "Remove Self Forwards (ms):", max_gc_threads);
_gc_par_phases[ClearCardTable] = new WorkerDataArray<double>("ClearLoggedCards", "Clear Logged Cards (ms):", max_gc_threads);
_gc_par_phases[RecalculateUsed] = new WorkerDataArray<double>("RecalculateUsed", "Recalculate Used Memory (ms):", max_gc_threads);
_gc_par_phases[ResetHotCardCache] = new WorkerDataArray<double>("ResetHotCardCache", "Reset Hot Card Cache (ms):", max_gc_threads);
_gc_par_phases[PurgeCodeRoots] = new WorkerDataArray<double>("PurgeCodeRoots", "Purge Code Roots (ms):", max_gc_threads);
#if COMPILER2_OR_JVMCI
_gc_par_phases[UpdateDerivedPointers] = new WorkerDataArray<double>("UpdateDerivedPointers", "Update Derived Pointers (ms):", max_gc_threads);
#endif
_gc_par_phases[EagerlyReclaimHumongousObjects] = new WorkerDataArray<double>("EagerlyReclaimHumongousObjects", "Eagerly Reclaim Humongous Objects (ms):", max_gc_threads);
_gc_par_phases[RestorePreservedMarks] = new WorkerDataArray<double>("RestorePreservedMarks", "Restore Preserved Marks (ms):", max_gc_threads);
_gc_par_phases[ScanHR]->create_thread_work_items("Scanned Cards:", ScanHRScannedCards);
_gc_par_phases[ScanHR]->create_thread_work_items("Scanned Blocks:", ScanHRScannedBlocks);
@ -122,11 +133,13 @@ G1GCPhaseTimes::G1GCPhaseTimes(STWGCTimer* gc_timer, uint max_gc_threads) :
_gc_par_phases[MergeLB]->create_thread_work_items("Dirty Cards:", MergeLBDirtyCards);
_gc_par_phases[MergeLB]->create_thread_work_items("Skipped Cards:", MergeLBSkippedCards);
_gc_par_phases[MergePSS] = new WorkerDataArray<double>("MergePSS", "Merge Per-Thread State", 1 /* length */, true /* is_serial */);
_gc_par_phases[MergePSS]->create_thread_work_items("Copied Bytes", MergePSSCopiedBytes);
_gc_par_phases[MergePSS]->create_thread_work_items("LAB Waste", MergePSSLABWasteBytes);
_gc_par_phases[MergePSS]->create_thread_work_items("LAB Undo Waste", MergePSSLABUndoWasteBytes);
_gc_par_phases[MergePSS]->create_thread_work_items("Copied Bytes", MergePSSCopiedBytes, max_gc_threads);
_gc_par_phases[MergePSS]->create_thread_work_items("LAB Waste", MergePSSLABWasteBytes, max_gc_threads);
_gc_par_phases[MergePSS]->create_thread_work_items("LAB Undo Waste", MergePSSLABUndoWasteBytes, max_gc_threads);
_gc_par_phases[EagerlyReclaimHumongousObjects]->create_thread_work_items("Humongous Total", EagerlyReclaimNumTotal);
_gc_par_phases[EagerlyReclaimHumongousObjects]->create_thread_work_items("Humongous Candidates", EagerlyReclaimNumCandidates);
_gc_par_phases[EagerlyReclaimHumongousObjects]->create_thread_work_items("Humongous Reclaimed", EagerlyReclaimNumReclaimed);
_gc_par_phases[Termination]->create_thread_work_items("Termination Attempts:");
@ -140,10 +153,10 @@ G1GCPhaseTimes::G1GCPhaseTimes(STWGCTimer* gc_timer, uint max_gc_threads) :
_gc_par_phases[StringDedupTableFixup] = NULL;
}
_gc_par_phases[RedirtyCards] = new WorkerDataArray<double>("RedirtyCards", "Parallel Redirty (ms):", max_gc_threads);
_gc_par_phases[RedirtyCards] = new WorkerDataArray<double>("RedirtyCards", "Redirty Logged Cards (ms):", max_gc_threads);
_gc_par_phases[RedirtyCards]->create_thread_work_items("Redirtied Cards:");
_gc_par_phases[ParFreeCSet] = new WorkerDataArray<double>("ParFreeCSet", "Parallel Free Collection Set (ms):", max_gc_threads);
_gc_par_phases[FreeCollectionSet] = new WorkerDataArray<double>("FreeCSet", "Free Collection Set (ms):", max_gc_threads);
_gc_par_phases[YoungFreeCSet] = new WorkerDataArray<double>("YoungFreeCSet", "Young Free Collection Set (ms):", max_gc_threads);
_gc_par_phases[NonYoungFreeCSet] = new WorkerDataArray<double>("NonYoungFreeCSet", "Non-Young Free Collection Set (ms):", max_gc_threads);
_gc_par_phases[RebuildFreeList] = new WorkerDataArray<double>("RebuildFreeList", "Parallel Rebuild Free List (ms):", max_gc_threads);
@ -155,19 +168,15 @@ void G1GCPhaseTimes::reset() {
_cur_collection_initial_evac_time_ms = 0.0;
_cur_optional_evac_time_ms = 0.0;
_cur_collection_code_root_fixup_time_ms = 0.0;
_cur_strong_code_root_purge_time_ms = 0.0;
_cur_merge_heap_roots_time_ms = 0.0;
_cur_optional_merge_heap_roots_time_ms = 0.0;
_cur_prepare_merge_heap_roots_time_ms = 0.0;
_cur_optional_prepare_merge_heap_roots_time_ms = 0.0;
_cur_evac_fail_recalc_used = 0.0;
_cur_evac_fail_remove_self_forwards = 0.0;
_cur_string_deduplication_time_ms = 0.0;
_cur_prepare_tlab_time_ms = 0.0;
_cur_resize_tlab_time_ms = 0.0;
_cur_concatenate_dirty_card_logs_time_ms = 0.0;
_cur_derived_pointer_table_update_time_ms = 0.0;
_cur_clear_ct_time_ms = 0.0;
_cur_post_evacuate_cleanup_1_time_ms = 0.0;
_cur_post_evacuate_cleanup_2_time_ms = 0.0;
_cur_expand_heap_time_ms = 0.0;
_cur_ref_proc_time_ms = 0.0;
_cur_collection_start_sec = 0.0;
@ -177,18 +186,12 @@ void G1GCPhaseTimes::reset() {
_recorded_clear_claimed_marks_time_ms = 0.0;
_recorded_young_cset_choice_time_ms = 0.0;
_recorded_non_young_cset_choice_time_ms = 0.0;
_recorded_redirty_logged_cards_time_ms = 0.0;
_recorded_preserve_cm_referents_time_ms = 0.0;
_recorded_start_new_cset_time_ms = 0.0;
_recorded_total_free_cset_time_ms = 0.0;
_recorded_serial_free_cset_time_ms = 0.0;
_recorded_total_rebuild_freelist_time_ms = 0.0;
_recorded_serial_rebuild_freelist_time_ms = 0.0;
_cur_fast_reclaim_humongous_time_ms = 0.0;
_cur_region_register_time = 0.0;
_cur_fast_reclaim_humongous_total = 0;
_cur_fast_reclaim_humongous_candidates = 0;
_cur_fast_reclaim_humongous_reclaimed = 0;
_cur_verify_before_time_ms = 0.0;
_cur_verify_after_time_ms = 0.0;
@ -403,10 +406,6 @@ double G1GCPhaseTimes::print_pre_evacuate_collection_set() const {
debug_time("Concatenate Dirty Card Logs", _cur_concatenate_dirty_card_logs_time_ms);
debug_time("Choose Collection Set", (_recorded_young_cset_choice_time_ms + _recorded_non_young_cset_choice_time_ms));
debug_time("Region Register", _cur_region_register_time);
if (G1EagerReclaimHumongousObjects) {
trace_count("Humongous Total", _cur_fast_reclaim_humongous_total);
trace_count("Humongous Candidate", _cur_fast_reclaim_humongous_candidates);
}
debug_time("Prepare Heap Roots", _recorded_prepare_heap_roots_time_ms);
if (_recorded_clear_claimed_marks_time_ms > 0.0) {
@ -462,31 +461,21 @@ double G1GCPhaseTimes::print_evacuate_initial_collection_set() const {
}
double G1GCPhaseTimes::print_post_evacuate_collection_set() const {
const double evac_fail_handling = _cur_evac_fail_recalc_used +
_cur_evac_fail_remove_self_forwards;
assert(_gc_par_phases[MergePSS]->get(0) != WorkerDataArray<double>::uninitialized(), "must be set");
const double merge_pss = _gc_par_phases[MergePSS]->get(0) * MILLIUNITS;
const double sum_ms = evac_fail_handling +
_cur_collection_code_root_fixup_time_ms +
const double sum_ms = _cur_collection_code_root_fixup_time_ms +
_recorded_preserve_cm_referents_time_ms +
_cur_ref_proc_time_ms +
(_weak_phase_times.total_time_sec() * MILLIUNITS) +
_cur_clear_ct_time_ms +
merge_pss +
_cur_strong_code_root_purge_time_ms +
_recorded_redirty_logged_cards_time_ms +
_recorded_total_free_cset_time_ms +
_cur_string_deduplication_time_ms +
_cur_post_evacuate_cleanup_1_time_ms +
_cur_post_evacuate_cleanup_2_time_ms +
_recorded_total_rebuild_freelist_time_ms +
_cur_fast_reclaim_humongous_time_ms +
_cur_expand_heap_time_ms +
_cur_string_deduplication_time_ms;
_recorded_start_new_cset_time_ms +
_cur_expand_heap_time_ms;
info_time("Post Evacuate Collection Set", sum_ms);
debug_time("Code Roots Fixup", _cur_collection_code_root_fixup_time_ms);
debug_time("Clear Card Table", _cur_clear_ct_time_ms);
debug_time_for_reference("Reference Processing", _cur_ref_proc_time_ms);
_ref_phase_times.print_all_references(2, false);
_weak_phase_times.log_total(2);
@ -498,42 +487,44 @@ double G1GCPhaseTimes::print_post_evacuate_collection_set() const {
debug_phase(_gc_par_phases[StringDedupTableFixup], 1);
}
debug_time("Post Evacuate Cleanup 1", _cur_post_evacuate_cleanup_1_time_ms);
debug_phase(_gc_par_phases[MergePSS], 1);
debug_phase(_gc_par_phases[ClearCardTable], 1);
debug_phase(_gc_par_phases[RecalculateUsed], 1);
if (G1CollectedHeap::heap()->evacuation_failed()) {
debug_time("Evacuation Failure", evac_fail_handling);
trace_time("Recalculate Used", _cur_evac_fail_recalc_used);
trace_time("Remove Self Forwards",_cur_evac_fail_remove_self_forwards);
debug_phase(_gc_par_phases[RemoveSelfForwardingPtr], 1);
}
debug_phase(_gc_par_phases[MergePSS], 0);
debug_time("Code Roots Purge", _cur_strong_code_root_purge_time_ms);
debug_time("Redirty Cards", _recorded_redirty_logged_cards_time_ms);
trace_phase(_gc_par_phases[RedirtyCards]);
debug_time("Post Evacuate Cleanup 2", _cur_post_evacuate_cleanup_2_time_ms);
if (G1CollectedHeap::heap()->evacuation_failed()) {
debug_phase(_gc_par_phases[RecalculateUsed], 1);
debug_phase(_gc_par_phases[RestorePreservedMarks], 1);
}
debug_phase(_gc_par_phases[ResetHotCardCache], 1);
debug_phase(_gc_par_phases[PurgeCodeRoots], 1);
#if COMPILER2_OR_JVMCI
debug_time("DerivedPointerTable Update", _cur_derived_pointer_table_update_time_ms);
debug_phase(_gc_par_phases[UpdateDerivedPointers], 1);
#endif
if (G1CollectedHeap::heap()->should_do_eager_reclaim()) {
debug_phase(_gc_par_phases[EagerlyReclaimHumongousObjects], 1);
}
debug_phase(_gc_par_phases[RedirtyCards], 1);
debug_phase(_gc_par_phases[FreeCollectionSet], 1);
trace_phase(_gc_par_phases[YoungFreeCSet], true, 2);
trace_phase(_gc_par_phases[NonYoungFreeCSet], true, 2);
debug_time("Free Collection Set", _recorded_total_free_cset_time_ms);
trace_time("Serial Free Collection Set", _recorded_serial_free_cset_time_ms);
trace_phase(_gc_par_phases[ParFreeCSet]);
trace_phase(_gc_par_phases[YoungFreeCSet], true, 1);
trace_phase(_gc_par_phases[NonYoungFreeCSet], true, 1);
debug_time("Rebuild Free List", _recorded_total_rebuild_freelist_time_ms);
trace_time("Serial Rebuild Free List ", _recorded_serial_rebuild_freelist_time_ms);
trace_phase(_gc_par_phases[RebuildFreeList]);
if (G1EagerReclaimHumongousObjects) {
debug_time("Humongous Reclaim", _cur_fast_reclaim_humongous_time_ms);
trace_count("Humongous Reclaimed", _cur_fast_reclaim_humongous_reclaimed);
}
debug_time("Start New Collection Set", _recorded_start_new_cset_time_ms);
if (UseTLAB && ResizeTLAB) {
debug_time("Resize TLABs", _cur_resize_tlab_time_ms);
}
debug_time("Expand Heap After Collection", _cur_expand_heap_time_ms);
return sum_ms;
}

@ -73,11 +73,21 @@ class G1GCPhaseTimes : public CHeapObj<mtGC> {
StringDedupQueueFixup,
StringDedupTableFixup,
RedirtyCards,
ParFreeCSet,
FreeCollectionSet,
YoungFreeCSet,
NonYoungFreeCSet,
RebuildFreeList,
MergePSS,
RemoveSelfForwardingPtr,
ClearCardTable,
RecalculateUsed,
ResetHotCardCache,
PurgeCodeRoots,
#if COMPILER2_OR_JVMCI
UpdateDerivedPointers,
#endif
EagerlyReclaimHumongousObjects,
RestorePreservedMarks,
GCParPhasesSentinel
};
@ -120,6 +130,12 @@ class G1GCPhaseTimes : public CHeapObj<mtGC> {
MergePSSLABUndoWasteBytes
};
enum GCEagerlyReclaimHumongousObjectsItems {
EagerlyReclaimNumTotal,
EagerlyReclaimNumCandidates,
EagerlyReclaimNumReclaimed
};
private:
// Markers for grouping the phases in the GCPhases enum above
static const int GCMainParPhasesLast = GCWorkerEnd;
@ -129,10 +145,6 @@ class G1GCPhaseTimes : public CHeapObj<mtGC> {
double _cur_collection_initial_evac_time_ms;
double _cur_optional_evac_time_ms;
double _cur_collection_code_root_fixup_time_ms;
double _cur_strong_code_root_purge_time_ms;
double _cur_evac_fail_recalc_used;
double _cur_evac_fail_remove_self_forwards;
double _cur_string_deduplication_time_ms;
@ -147,9 +159,9 @@ class G1GCPhaseTimes : public CHeapObj<mtGC> {
double _cur_concatenate_dirty_card_logs_time_ms;
double _cur_derived_pointer_table_update_time_ms;
double _cur_post_evacuate_cleanup_1_time_ms;
double _cur_post_evacuate_cleanup_2_time_ms;
double _cur_clear_ct_time_ms;
double _cur_expand_heap_time_ms;
double _cur_ref_proc_time_ms;
@ -165,16 +177,10 @@ class G1GCPhaseTimes : public CHeapObj<mtGC> {
double _recorded_young_cset_choice_time_ms;
double _recorded_non_young_cset_choice_time_ms;
double _recorded_redirty_logged_cards_time_ms;
double _recorded_preserve_cm_referents_time_ms;
double _recorded_merge_pss_time_ms;
double _recorded_start_new_cset_time_ms;
double _recorded_total_free_cset_time_ms;
double _recorded_serial_free_cset_time_ms;
double _recorded_total_rebuild_freelist_time_ms;
@ -183,11 +189,6 @@ class G1GCPhaseTimes : public CHeapObj<mtGC> {
double _cur_region_register_time;
double _cur_fast_reclaim_humongous_time_ms;
size_t _cur_fast_reclaim_humongous_total;
size_t _cur_fast_reclaim_humongous_candidates;
size_t _cur_fast_reclaim_humongous_reclaimed;
double _cur_verify_before_time_ms;
double _cur_verify_after_time_ms;
@ -260,14 +261,6 @@ class G1GCPhaseTimes : public CHeapObj<mtGC> {
_cur_concatenate_dirty_card_logs_time_ms = ms;
}
void record_derived_pointer_table_update_time(double ms) {
_cur_derived_pointer_table_update_time_ms = ms;
}
void record_clear_ct_time(double ms) {
_cur_clear_ct_time_ms = ms;
}
void record_expand_heap_time(double ms) {
_cur_expand_heap_time_ms = ms;
}
@ -284,10 +277,6 @@ class G1GCPhaseTimes : public CHeapObj<mtGC> {
_cur_collection_code_root_fixup_time_ms += ms;
}
void record_strong_code_root_purge_time(double ms) {
_cur_strong_code_root_purge_time_ms = ms;
}
void record_merge_heap_roots_time(double ms) {
_cur_merge_heap_roots_time_ms += ms;
}
@ -304,14 +293,6 @@ class G1GCPhaseTimes : public CHeapObj<mtGC> {
_cur_optional_prepare_merge_heap_roots_time_ms += ms;
}
void record_evac_fail_recalc_used_time(double ms) {
_cur_evac_fail_recalc_used = ms;
}
void record_evac_fail_remove_self_forwards(double ms) {
_cur_evac_fail_remove_self_forwards = ms;
}
void record_string_deduplication_time(double ms) {
_cur_string_deduplication_time_ms = ms;
}
@ -324,10 +305,6 @@ class G1GCPhaseTimes : public CHeapObj<mtGC> {
_root_region_scan_wait_time_ms = time_ms;
}
void record_total_free_cset_time_ms(double time_ms) {
_recorded_total_free_cset_time_ms = time_ms;
}
void record_serial_free_cset_time_ms(double time_ms) {
_recorded_serial_free_cset_time_ms = time_ms;
}
@ -340,15 +317,16 @@ class G1GCPhaseTimes : public CHeapObj<mtGC> {
_recorded_serial_rebuild_freelist_time_ms = time_ms;
}
void record_register_regions(double time_ms, size_t total, size_t candidates) {
void record_register_regions(double time_ms) {
_cur_region_register_time = time_ms;
_cur_fast_reclaim_humongous_total = total;
_cur_fast_reclaim_humongous_candidates = candidates;
}
void record_fast_reclaim_humongous_time_ms(double value, size_t reclaimed) {
_cur_fast_reclaim_humongous_time_ms = value;
_cur_fast_reclaim_humongous_reclaimed = reclaimed;
void record_post_evacuate_cleanup_task_1_time(double time_ms) {
_cur_post_evacuate_cleanup_1_time_ms = time_ms;
}
void record_post_evacuate_cleanup_task_2_time(double time_ms) {
_cur_post_evacuate_cleanup_2_time_ms = time_ms;
}
void record_young_cset_choice_time_ms(double time_ms) {
@ -359,10 +337,6 @@ class G1GCPhaseTimes : public CHeapObj<mtGC> {
_recorded_non_young_cset_choice_time_ms = time_ms;
}
void record_redirty_logged_cards_time_ms(double time_ms) {
_recorded_redirty_logged_cards_time_ms = time_ms;
}
void record_preserve_cm_referents_time_ms(double time_ms) {
_recorded_preserve_cm_referents_time_ms = time_ms;
}
@ -403,10 +377,6 @@ class G1GCPhaseTimes : public CHeapObj<mtGC> {
return _cur_collection_initial_evac_time_ms + _cur_optional_evac_time_ms;
}
double cur_clear_ct_time_ms() {
return _cur_clear_ct_time_ms;
}
double cur_expand_heap_time_ms() {
return _cur_expand_heap_time_ms;
}
@ -419,10 +389,6 @@ class G1GCPhaseTimes : public CHeapObj<mtGC> {
return _recorded_young_cset_choice_time_ms;
}
double total_free_cset_time_ms() {
return _recorded_total_free_cset_time_ms;
}
double total_rebuild_freelist_time_ms() {
return _recorded_total_rebuild_freelist_time_ms;
}
@ -431,14 +397,6 @@ class G1GCPhaseTimes : public CHeapObj<mtGC> {
return _recorded_non_young_cset_choice_time_ms;
}
double fast_reclaim_humongous_time_ms() {
return _cur_fast_reclaim_humongous_time_ms;
}
size_t fast_reclaim_humongous_candidates() const {
return _cur_fast_reclaim_humongous_candidates;
}
ReferenceProcessorPhaseTimes* ref_phase_times() { return &_ref_phase_times; }
WeakProcessorTimes* weak_phase_times() { return &_weak_phase_times; }

@ -128,7 +128,6 @@ private:
// Resets the hot card cache and discards the entries.
void reset_hot_cache() {
assert(SafepointSynchronize::is_at_safepoint(), "Should be at a safepoint");
assert(Thread::current()->is_VM_thread(), "Current thread should be the VMthread");
if (default_use_cache()) {
reset_hot_cache_internal();
}

@ -575,7 +575,7 @@ double G1Policy::other_time_ms(double pause_time_ms) const {
}
double G1Policy::constant_other_time_ms(double pause_time_ms) const {
return other_time_ms(pause_time_ms) - phase_times()->total_free_cset_time_ms() - phase_times()->total_rebuild_freelist_time_ms();
return other_time_ms(pause_time_ms) - phase_times()->total_rebuild_freelist_time_ms();
}
bool G1Policy::about_to_start_mixed_phase() const {

@ -24,6 +24,7 @@
#include "precompiled.hpp"
#include "gc/g1/g1BarrierSet.hpp"
#include "gc/g1/g1BatchedGangTask.hpp"
#include "gc/g1/g1BlockOffsetTable.inline.hpp"
#include "gc/g1/g1CardTable.inline.hpp"
#include "gc/g1/g1CardTableEntryClosure.hpp"
@ -218,7 +219,7 @@ private:
// entries from free or archive regions.
HeapWord** _scan_top;
class G1ClearCardTableTask : public AbstractGangTask {
class G1ClearCardTableTask : public G1AbstractSubTask {
G1CollectedHeap* _g1h;
G1DirtyRegions* _regions;
uint _chunk_length;
@ -232,7 +233,7 @@ private:
G1DirtyRegions* regions,
uint chunk_length,
G1RemSetScanState* scan_state) :
AbstractGangTask("G1 Clear Card Table Task"),
G1AbstractSubTask(G1GCPhaseTimes::ClearCardTable),
_g1h(g1h),
_regions(regions),
_chunk_length(chunk_length),
@ -242,9 +243,27 @@ private:
assert(chunk_length > 0, "must be");
}
double worker_cost() const override {
uint num_regions = _regions->size();
if (num_regions == 0) {
// There is no card table clean work, only some cleanup of memory.
return AlmostNoWork;
}
return ((double)align_up((size_t)num_regions << HeapRegion::LogCardsPerRegion, chunk_size()) / chunk_size());
}
virtual ~G1ClearCardTableTask() {
_scan_state->cleanup();
#ifndef PRODUCT
G1CollectedHeap::heap()->verifier()->verify_card_table_cleanup();
#endif
}
static uint chunk_size() { return M; }
void work(uint worker_id) {
void do_work(uint worker_id) override {
while (_cur_dirty_regions < _regions->size()) {
uint next = Atomic::fetch_and_add(&_cur_dirty_regions, _chunk_length);
uint max = MIN2(next + _chunk_length, _regions->size());
@ -259,31 +278,6 @@ private:
}
};
// Clear the card table of "dirty" regions.
void clear_card_table(WorkGang* workers) {
uint num_regions = _all_dirty_regions->size();
if (num_regions == 0) {
return;
}
uint const num_chunks = (uint)(align_up((size_t)num_regions << HeapRegion::LogCardsPerRegion, G1ClearCardTableTask::chunk_size()) / G1ClearCardTableTask::chunk_size());
uint const num_workers = MIN2(num_chunks, workers->active_workers());
uint const chunk_length = G1ClearCardTableTask::chunk_size() / (uint)HeapRegion::CardsPerRegion;
// Iterate over the dirty cards region list.
G1ClearCardTableTask cl(G1CollectedHeap::heap(), _all_dirty_regions, chunk_length, this);
log_debug(gc, ergo)("Running %s using %u workers for %u "
"units of work for %u regions.",
cl.name(), num_workers, num_chunks, num_regions);
workers->run_task(&cl, num_workers);
#ifndef PRODUCT
G1CollectedHeap::heap()->verifier()->verify_card_table_cleanup();
#endif
}
public:
G1RemSetScanState() :
_max_reserved_regions(0),
@ -391,9 +385,13 @@ public:
}
}
void cleanup(WorkGang* workers) {
clear_card_table(workers);
G1AbstractSubTask* create_cleanup_after_scan_heap_roots_task() {
uint const chunk_length = G1ClearCardTableTask::chunk_size() / (uint)HeapRegion::CardsPerRegion;
return new G1ClearCardTableTask(G1CollectedHeap::heap(), _all_dirty_regions, chunk_length, this);
}
void cleanup() {
delete _all_dirty_regions;
_all_dirty_regions = NULL;
@ -1309,7 +1307,7 @@ public:
// We schedule flushing the remembered sets of humongous fast reclaim candidates
// onto the card table first to allow the remaining parallelized tasks hide it.
if (_initial_evacuation &&
p->fast_reclaim_humongous_candidates() > 0 &&
g1h->has_humongous_reclaim_candidates() &&
!_fast_reclaim_handled &&
!Atomic::cmpxchg(&_fast_reclaim_handled, false, true)) {
@ -1420,13 +1418,8 @@ void G1RemSet::exclude_region_from_scan(uint region_idx) {
_scan_state->clear_scan_top(region_idx);
}
void G1RemSet::cleanup_after_scan_heap_roots() {
G1GCPhaseTimes* phase_times = _g1h->phase_times();
// Set all cards back to clean.
double start = os::elapsedTime();
_scan_state->cleanup(_g1h->workers());
phase_times->record_clear_ct_time((os::elapsedTime() - start) * 1000.0);
G1AbstractSubTask* G1RemSet::create_cleanup_after_scan_heap_roots_task() {
return _scan_state->create_cleanup_after_scan_heap_roots_task();
}
inline void check_card_ptr(CardTable::CardValue* card_ptr, G1CardTable* ct) {

@ -39,8 +39,8 @@
class BitMap;
class CardTableBarrierSet;
class G1BlockOffsetTable;
class CodeBlobClosure;
class G1AbstractSubTask;
class G1CollectedHeap;
class G1CMBitMap;
class G1HotCardCache;
@ -107,8 +107,9 @@ public:
// Prepare for and cleanup after scanning the heap roots. Must be called
// once before and after in sequential code.
void prepare_for_scan_heap_roots();
// Cleans the card table from temporary duplicate detection information.
void cleanup_after_scan_heap_roots();
// Creates a gang task for cleaining up temporary data structures and the
// card table, removing temporary duplicate detection information.
G1AbstractSubTask* create_cleanup_after_scan_heap_roots_task();
// Excludes the given region from heap root scanning.
void exclude_region_from_scan(uint region_idx);
// Creates a snapshot of the current _top values at the start of collection to

@ -0,0 +1,606 @@
/*
* Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*
*/
#include "precompiled.hpp"
#include "compiler/oopMap.hpp"
#include "gc/g1/g1CardTableEntryClosure.hpp"
#include "gc/g1/g1CollectedHeap.inline.hpp"
#include "gc/g1/g1ConcurrentMark.inline.hpp"
#include "gc/g1/g1EvacStats.inline.hpp"
#include "gc/g1/g1ParScanThreadState.hpp"
#include "gc/g1/g1RemSet.hpp"
#include "gc/g1/g1YoungGCPostEvacuateTasks.hpp"
#include "jfr/jfrEvents.hpp"
#include "utilities/ticks.hpp"
G1PostEvacuateCollectionSetCleanupTask1::G1PostEvacuateCollectionSetCleanupTask1(G1ParScanThreadStateSet* per_thread_states,
G1RedirtyCardsQueueSet* rdcqs) :
G1BatchedGangTask("Post Evacuate Cleanup 1", G1CollectedHeap::heap()->phase_times())
{
add_serial_task(new MergePssTask(per_thread_states));
add_serial_task(new RecalculateUsedTask());
if (RemoveSelfForwardPtrsTask::should_execute()) {
add_parallel_task(new RemoveSelfForwardPtrsTask(rdcqs));
}
add_parallel_task(G1CollectedHeap::heap()->rem_set()->create_cleanup_after_scan_heap_roots_task());
}
G1PostEvacuateCollectionSetCleanupTask1::MergePssTask::MergePssTask(G1ParScanThreadStateSet* per_thread_states) :
G1AbstractSubTask(G1GCPhaseTimes::MergePSS),
_per_thread_states(per_thread_states) { }
void G1PostEvacuateCollectionSetCleanupTask1::MergePssTask::do_work(uint worker_id) {
_per_thread_states->flush();
}
double G1PostEvacuateCollectionSetCleanupTask1::RecalculateUsedTask::worker_cost() const {
// If there is no evacuation failure, the work to perform is minimal.
return G1CollectedHeap::heap()->evacuation_failed() ? 1.0 : AlmostNoWork;
}
void G1PostEvacuateCollectionSetCleanupTask1::RecalculateUsedTask::do_work(uint worker_id) {
G1CollectedHeap::heap()->update_used_after_gc();
}
bool G1PostEvacuateCollectionSetCleanupTask1::RemoveSelfForwardPtrsTask::should_execute() {
return G1CollectedHeap::heap()->evacuation_failed();
}
G1PostEvacuateCollectionSetCleanupTask1::RemoveSelfForwardPtrsTask::RemoveSelfForwardPtrsTask(G1RedirtyCardsQueueSet* rdcqs) :
G1AbstractSubTask(G1GCPhaseTimes::RemoveSelfForwardingPtr), _task(rdcqs) { }
G1PostEvacuateCollectionSetCleanupTask1::RemoveSelfForwardPtrsTask::~RemoveSelfForwardPtrsTask() {
G1CollectedHeap* g1h = G1CollectedHeap::heap();
assert(_task.num_failed_regions() == g1h->num_regions_failed_evacuation(),
"Removed regions %u inconsistent with expected %u",
_task.num_failed_regions(), g1h->num_regions_failed_evacuation());
}
double G1PostEvacuateCollectionSetCleanupTask1::RemoveSelfForwardPtrsTask::worker_cost() const {
assert(should_execute(), "Should not call this if not executed");
return G1CollectedHeap::heap()->num_regions_failed_evacuation();
}
void G1PostEvacuateCollectionSetCleanupTask1::RemoveSelfForwardPtrsTask::do_work(uint worker_id) {
_task.work(worker_id);
}
class G1FreeHumongousRegionClosure : public HeapRegionClosure {
uint _humongous_objects_reclaimed;
uint _humongous_regions_reclaimed;
size_t _freed_bytes;
public:
G1FreeHumongousRegionClosure() :
_humongous_objects_reclaimed(0),
_humongous_regions_reclaimed(0),
_freed_bytes(0) {
}
virtual bool do_heap_region(HeapRegion* r) {
if (!r->is_starts_humongous()) {
return false;
}
G1CollectedHeap* g1h = G1CollectedHeap::heap();
oop obj = cast_to_oop(r->bottom());
G1CMBitMap* next_bitmap = g1h->concurrent_mark()->next_mark_bitmap();
// The following checks whether the humongous object is live are sufficient.
// The main additional check (in addition to having a reference from the roots
// or the young gen) is whether the humongous object has a remembered set entry.
//
// A humongous object cannot be live if there is no remembered set for it
// because:
// - there can be no references from within humongous starts regions referencing
// the object because we never allocate other objects into them.
// (I.e. there are no intra-region references that may be missed by the
// remembered set)
// - as soon there is a remembered set entry to the humongous starts region
// (i.e. it has "escaped" to an old object) this remembered set entry will stay
// until the end of a concurrent mark.
//
// It is not required to check whether the object has been found dead by marking
// or not, in fact it would prevent reclamation within a concurrent cycle, as
// all objects allocated during that time are considered live.
// SATB marking is even more conservative than the remembered set.
// So if at this point in the collection there is no remembered set entry,
// nobody has a reference to it.
// At the start of collection we flush all refinement logs, and remembered sets
// are completely up-to-date wrt to references to the humongous object.
//
// Other implementation considerations:
// - never consider object arrays at this time because they would pose
// considerable effort for cleaning up the the remembered sets. This is
// required because stale remembered sets might reference locations that
// are currently allocated into.
uint region_idx = r->hrm_index();
if (!g1h->is_humongous_reclaim_candidate(region_idx) ||
!r->rem_set()->is_empty()) {
log_debug(gc, humongous)("Live humongous region %u object size " SIZE_FORMAT " start " PTR_FORMAT " with remset " SIZE_FORMAT " code roots " SIZE_FORMAT " is marked %d reclaim candidate %d type array %d",
region_idx,
(size_t)obj->size() * HeapWordSize,
p2i(r->bottom()),
r->rem_set()->occupied(),
r->rem_set()->strong_code_roots_list_length(),
next_bitmap->is_marked(r->bottom()),
g1h->is_humongous_reclaim_candidate(region_idx),
obj->is_typeArray()
);
return false;
}
guarantee(obj->is_typeArray(),
"Only eagerly reclaiming type arrays is supported, but the object "
PTR_FORMAT " is not.", p2i(r->bottom()));
log_debug(gc, humongous)("Dead humongous region %u object size " SIZE_FORMAT " start " PTR_FORMAT " with remset " SIZE_FORMAT " code roots " SIZE_FORMAT " is marked %d reclaim candidate %d type array %d",
region_idx,
(size_t)obj->size() * HeapWordSize,
p2i(r->bottom()),
r->rem_set()->occupied(),
r->rem_set()->strong_code_roots_list_length(),
next_bitmap->is_marked(r->bottom()),
g1h->is_humongous_reclaim_candidate(region_idx),
obj->is_typeArray()
);
G1ConcurrentMark* const cm = g1h->concurrent_mark();
cm->humongous_object_eagerly_reclaimed(r);
assert(!cm->is_marked_in_prev_bitmap(obj) && !cm->is_marked_in_next_bitmap(obj),
"Eagerly reclaimed humongous region %u should not be marked at all but is in prev %s next %s",
region_idx,
BOOL_TO_STR(cm->is_marked_in_prev_bitmap(obj)),
BOOL_TO_STR(cm->is_marked_in_next_bitmap(obj)));
_humongous_objects_reclaimed++;
do {
HeapRegion* next = g1h->next_region_in_humongous(r);
_freed_bytes += r->used();
r->set_containing_set(nullptr);
_humongous_regions_reclaimed++;
g1h->free_humongous_region(r, nullptr);
g1h->hr_printer()->cleanup(r);
r = next;
} while (r != nullptr);
return false;
}
uint humongous_objects_reclaimed() {
return _humongous_objects_reclaimed;
}
uint humongous_regions_reclaimed() {
return _humongous_regions_reclaimed;
}
size_t bytes_freed() const {
return _freed_bytes;
}
};
void G1PostEvacuateCollectionSetCleanupTask2::ResetHotCardCacheTask::do_work(uint worker_id) {
G1CollectedHeap::heap()->reset_hot_card_cache();
}
void G1PostEvacuateCollectionSetCleanupTask2::PurgeCodeRootsTask::do_work(uint worker_id) {
G1CollectedHeap::heap()->purge_code_root_memory();
}
#if COMPILER2_OR_JVMCI
void G1PostEvacuateCollectionSetCleanupTask2::UpdateDerivedPointersTask::do_work(uint worker_id) {
DerivedPointerTable::update_pointers();
}
#endif
bool G1PostEvacuateCollectionSetCleanupTask2::EagerlyReclaimHumongousObjectsTask::should_execute() {
return G1CollectedHeap::heap()->should_do_eager_reclaim();
}
G1PostEvacuateCollectionSetCleanupTask2::EagerlyReclaimHumongousObjectsTask::EagerlyReclaimHumongousObjectsTask() :
G1AbstractSubTask(G1GCPhaseTimes::EagerlyReclaimHumongousObjects),
_humongous_regions_reclaimed(0),
_bytes_freed(0) { }
void G1PostEvacuateCollectionSetCleanupTask2::EagerlyReclaimHumongousObjectsTask::do_work(uint worker_id) {
G1CollectedHeap* g1h = G1CollectedHeap::heap();
G1FreeHumongousRegionClosure cl;
g1h->heap_region_iterate(&cl);
record_work_item(worker_id, G1GCPhaseTimes::EagerlyReclaimNumTotal, g1h->num_humongous_objects());
record_work_item(worker_id, G1GCPhaseTimes::EagerlyReclaimNumCandidates, g1h->num_humongous_reclaim_candidates());
record_work_item(worker_id, G1GCPhaseTimes::EagerlyReclaimNumReclaimed, cl.humongous_objects_reclaimed());
_humongous_regions_reclaimed = cl.humongous_regions_reclaimed();
_bytes_freed = cl.bytes_freed();
}
G1PostEvacuateCollectionSetCleanupTask2::EagerlyReclaimHumongousObjectsTask::~EagerlyReclaimHumongousObjectsTask() {
G1CollectedHeap* g1h = G1CollectedHeap::heap();
g1h->remove_from_old_gen_sets(0, 0, _humongous_regions_reclaimed);
g1h->decrement_summary_bytes(_bytes_freed);
}
bool G1PostEvacuateCollectionSetCleanupTask2::RestorePreservedMarksTask::should_execute() {
return G1CollectedHeap::heap()->evacuation_failed();
}
G1PostEvacuateCollectionSetCleanupTask2::RestorePreservedMarksTask::RestorePreservedMarksTask(PreservedMarksSet* preserved_marks) :
G1AbstractSubTask(G1GCPhaseTimes::RestorePreservedMarks), _preserved_marks(preserved_marks), _task(preserved_marks->create_task()) { }
G1PostEvacuateCollectionSetCleanupTask2::RestorePreservedMarksTask::~RestorePreservedMarksTask() {
delete _task;
}
double G1PostEvacuateCollectionSetCleanupTask2::RestorePreservedMarksTask::worker_cost() const {
assert(should_execute(), "Should not call this if not executed");
return _preserved_marks->num();
}
void G1PostEvacuateCollectionSetCleanupTask2::RestorePreservedMarksTask::do_work(uint worker_id) {
_task->work(worker_id);
}
class RedirtyLoggedCardTableEntryClosure : public G1CardTableEntryClosure {
private:
size_t _num_dirtied;
G1CollectedHeap* _g1h;
G1CardTable* _g1_ct;
HeapRegion* region_for_card(CardValue* card_ptr) const {
return _g1h->heap_region_containing(_g1_ct->addr_for(card_ptr));
}
bool will_become_free(HeapRegion* hr) const {
// A region will be freed by free_collection_set if the region is in the
// collection set and has not had an evacuation failure.
return _g1h->is_in_cset(hr) && !hr->evacuation_failed();
}
public:
RedirtyLoggedCardTableEntryClosure(G1CollectedHeap* g1h) : G1CardTableEntryClosure(),
_num_dirtied(0), _g1h(g1h), _g1_ct(g1h->card_table()) { }
void do_card_ptr(CardValue* card_ptr, uint worker_id) {
HeapRegion* hr = region_for_card(card_ptr);
// Should only dirty cards in regions that won't be freed.
if (!will_become_free(hr)) {
*card_ptr = G1CardTable::dirty_card_val();
_num_dirtied++;
}
}
size_t num_dirtied() const { return _num_dirtied; }
};
G1PostEvacuateCollectionSetCleanupTask2::RedirtyLoggedCardsTask::RedirtyLoggedCardsTask(G1RedirtyCardsQueueSet* rdcqs) :
G1AbstractSubTask(G1GCPhaseTimes::RedirtyCards),
_rdcqs(rdcqs),
_nodes(rdcqs->all_completed_buffers()) { }
G1PostEvacuateCollectionSetCleanupTask2::RedirtyLoggedCardsTask::~RedirtyLoggedCardsTask() {
G1DirtyCardQueueSet& dcq = G1BarrierSet::dirty_card_queue_set();
dcq.merge_bufferlists(_rdcqs);
_rdcqs->verify_empty();
}
double G1PostEvacuateCollectionSetCleanupTask2::RedirtyLoggedCardsTask::worker_cost() const {
// Needs more investigation.
return G1CollectedHeap::heap()->workers()->active_workers();
}
void G1PostEvacuateCollectionSetCleanupTask2::RedirtyLoggedCardsTask::do_work(uint worker_id) {
RedirtyLoggedCardTableEntryClosure cl(G1CollectedHeap::heap());
const size_t buffer_size = _rdcqs->buffer_size();
BufferNode* next = Atomic::load(&_nodes);
while (next != nullptr) {
BufferNode* node = next;
next = Atomic::cmpxchg(&_nodes, node, node->next());
if (next == node) {
cl.apply_to_buffer(node, buffer_size, worker_id);
next = node->next();
}
}
record_work_item(worker_id, 0, cl.num_dirtied());
}
// Helper class to keep statistics for the collection set freeing
class FreeCSetStats {
size_t _before_used_bytes; // Usage in regions successfully evacutate
size_t _after_used_bytes; // Usage in regions failing evacuation
size_t _bytes_allocated_in_old_since_last_gc; // Size of young regions turned into old
size_t _failure_used_words; // Live size in failed regions
size_t _failure_waste_words; // Wasted size in failed regions
size_t _rs_length; // Remembered set size
uint _regions_freed; // Number of regions freed
public:
FreeCSetStats() :
_before_used_bytes(0),
_after_used_bytes(0),
_bytes_allocated_in_old_since_last_gc(0),
_failure_used_words(0),
_failure_waste_words(0),
_rs_length(0),
_regions_freed(0) { }
void merge_stats(FreeCSetStats* other) {
assert(other != nullptr, "invariant");
_before_used_bytes += other->_before_used_bytes;
_after_used_bytes += other->_after_used_bytes;
_bytes_allocated_in_old_since_last_gc += other->_bytes_allocated_in_old_since_last_gc;
_failure_used_words += other->_failure_used_words;
_failure_waste_words += other->_failure_waste_words;
_rs_length += other->_rs_length;
_regions_freed += other->_regions_freed;
}
void report(G1CollectedHeap* g1h, G1EvacuationInfo* evacuation_info) {
evacuation_info->set_regions_freed(_regions_freed);
evacuation_info->increment_collectionset_used_after(_after_used_bytes);
g1h->decrement_summary_bytes(_before_used_bytes);
g1h->alloc_buffer_stats(G1HeapRegionAttr::Old)->add_failure_used_and_waste(_failure_used_words, _failure_waste_words);
G1Policy *policy = g1h->policy();
policy->old_gen_alloc_tracker()->add_allocated_bytes_since_last_gc(_bytes_allocated_in_old_since_last_gc);
policy->record_rs_length(_rs_length);
policy->cset_regions_freed();
}
void account_failed_region(HeapRegion* r) {
size_t used_words = r->marked_bytes() / HeapWordSize;
_failure_used_words += used_words;
_failure_waste_words += HeapRegion::GrainWords - used_words;
_after_used_bytes += r->used();
// When moving a young gen region to old gen, we "allocate" that whole
// region there. This is in addition to any already evacuated objects.
// Notify the policy about that. Old gen regions do not cause an
// additional allocation: both the objects still in the region and the
// ones already moved are accounted for elsewhere.
if (r->is_young()) {
_bytes_allocated_in_old_since_last_gc += HeapRegion::GrainBytes;
}
}
void account_evacuated_region(HeapRegion* r) {
size_t used = r->used();
assert(used > 0, "region %u %s zero used", r->hrm_index(), r->get_short_type_str());
_before_used_bytes += used;
_regions_freed += 1;
}
void account_rs_length(HeapRegion* r) {
_rs_length += r->rem_set()->occupied();
}
};
// Closure applied to all regions in the collection set.
class FreeCSetClosure : public HeapRegionClosure {
// Helper to send JFR events for regions.
class JFREventForRegion {
EventGCPhaseParallel _event;
public:
JFREventForRegion(HeapRegion* region, uint worker_id) : _event() {
_event.set_gcId(GCId::current());
_event.set_gcWorkerId(worker_id);
if (region->is_young()) {
_event.set_name(G1GCPhaseTimes::phase_name(G1GCPhaseTimes::YoungFreeCSet));
} else {
_event.set_name(G1GCPhaseTimes::phase_name(G1GCPhaseTimes::NonYoungFreeCSet));
}
}
~JFREventForRegion() {
_event.commit();
}
};
// Helper to do timing for region work.
class TimerForRegion {
Tickspan& _time;
Ticks _start_time;
public:
TimerForRegion(Tickspan& time) : _time(time), _start_time(Ticks::now()) { }
~TimerForRegion() {
_time += Ticks::now() - _start_time;
}
};
// FreeCSetClosure members
G1CollectedHeap* _g1h;
const size_t* _surviving_young_words;
uint _worker_id;
Tickspan _young_time;
Tickspan _non_young_time;
FreeCSetStats* _stats;
void assert_in_cset(HeapRegion* r) {
assert(r->young_index_in_cset() != 0 &&
(uint)r->young_index_in_cset() <= _g1h->collection_set()->young_region_length(),
"Young index %u is wrong for region %u of type %s with %u young regions",
r->young_index_in_cset(), r->hrm_index(), r->get_type_str(), _g1h->collection_set()->young_region_length());
}
void handle_evacuated_region(HeapRegion* r) {
assert(!r->is_empty(), "Region %u is an empty region in the collection set.", r->hrm_index());
stats()->account_evacuated_region(r);
// Free the region and and its remembered set.
_g1h->free_region(r, nullptr);
_g1h->hr_printer()->cleanup(r);
}
void handle_failed_region(HeapRegion* r) {
// Do some allocation statistics accounting. Regions that failed evacuation
// are always made old, so there is no need to update anything in the young
// gen statistics, but we need to update old gen statistics.
stats()->account_failed_region(r);
// Update the region state due to the failed evacuation.
r->handle_evacuation_failure();
// Add region to old set, need to hold lock.
MutexLocker x(OldSets_lock, Mutex::_no_safepoint_check_flag);
_g1h->old_set_add(r);
}
Tickspan& timer_for_region(HeapRegion* r) {
return r->is_young() ? _young_time : _non_young_time;
}
FreeCSetStats* stats() {
return _stats;
}
public:
FreeCSetClosure(const size_t* surviving_young_words,
uint worker_id,
FreeCSetStats* stats) :
HeapRegionClosure(),
_g1h(G1CollectedHeap::heap()),
_surviving_young_words(surviving_young_words),
_worker_id(worker_id),
_young_time(),
_non_young_time(),
_stats(stats) { }
virtual bool do_heap_region(HeapRegion* r) {
assert(r->in_collection_set(), "Invariant: %u missing from CSet", r->hrm_index());
JFREventForRegion event(r, _worker_id);
TimerForRegion timer(timer_for_region(r));
_g1h->clear_region_attr(r);
stats()->account_rs_length(r);
if (r->is_young()) {
assert_in_cset(r);
r->record_surv_words_in_group(_surviving_young_words[r->young_index_in_cset()]);
}
if (r->evacuation_failed()) {
handle_failed_region(r);
} else {
handle_evacuated_region(r);
}
assert(!_g1h->is_on_master_free_list(r), "sanity");
return false;
}
void report_timing() {
G1GCPhaseTimes* pt = _g1h->phase_times();
if (_young_time.value() > 0) {
pt->record_time_secs(G1GCPhaseTimes::YoungFreeCSet, _worker_id, _young_time.seconds());
}
if (_non_young_time.value() > 0) {
pt->record_time_secs(G1GCPhaseTimes::NonYoungFreeCSet, _worker_id, _non_young_time.seconds());
}
}
};
FreeCSetStats* G1PostEvacuateCollectionSetCleanupTask2::FreeCollectionSetTask::worker_stats(uint worker) {
return &_worker_stats[worker];
}
void G1PostEvacuateCollectionSetCleanupTask2::FreeCollectionSetTask::report_statistics() {
// Merge the accounting
FreeCSetStats total_stats;
for (uint worker = 0; worker < _active_workers; worker++) {
total_stats.merge_stats(worker_stats(worker));
}
total_stats.report(_g1h, _evacuation_info);
}
G1PostEvacuateCollectionSetCleanupTask2::FreeCollectionSetTask::FreeCollectionSetTask(G1EvacuationInfo* evacuation_info,
const size_t* surviving_young_words) :
G1AbstractSubTask(G1GCPhaseTimes::FreeCollectionSet),
_g1h(G1CollectedHeap::heap()),
_evacuation_info(evacuation_info),
_worker_stats(nullptr),
_claimer(0),
_surviving_young_words(surviving_young_words),
_active_workers(0) {
_g1h->clear_eden();
}
G1PostEvacuateCollectionSetCleanupTask2::FreeCollectionSetTask::~FreeCollectionSetTask() {
Ticks serial_time = Ticks::now();
report_statistics();
for (uint worker = 0; worker < _active_workers; worker++) {
_worker_stats[worker].~FreeCSetStats();
}
FREE_C_HEAP_ARRAY(FreeCSetStats, _worker_stats);
_g1h->phase_times()->record_serial_free_cset_time_ms((Ticks::now() - serial_time).seconds() * 1000.0);
_g1h->clear_collection_set();
}
double G1PostEvacuateCollectionSetCleanupTask2::FreeCollectionSetTask::worker_cost() const {
return G1CollectedHeap::heap()->collection_set()->region_length();
}
void G1PostEvacuateCollectionSetCleanupTask2::FreeCollectionSetTask::set_max_workers(uint max_workers) {
_active_workers = max_workers;
_worker_stats = NEW_C_HEAP_ARRAY(FreeCSetStats, max_workers, mtGC);
for (uint worker = 0; worker < _active_workers; worker++) {
::new (&_worker_stats[worker]) FreeCSetStats();
}
_claimer.set_n_workers(_active_workers);
}
void G1PostEvacuateCollectionSetCleanupTask2::FreeCollectionSetTask::do_work(uint worker_id) {
FreeCSetClosure cl(_surviving_young_words, worker_id, worker_stats(worker_id));
_g1h->collection_set_par_iterate_all(&cl, &_claimer, worker_id);
// Report per-region type timings.
cl.report_timing();
}
G1PostEvacuateCollectionSetCleanupTask2::G1PostEvacuateCollectionSetCleanupTask2(PreservedMarksSet* preserved_marks_set,
G1RedirtyCardsQueueSet* rdcqs,
G1EvacuationInfo* evacuation_info,
const size_t* surviving_young_words) :
G1BatchedGangTask("Post Evacuate Cleanup 2", G1CollectedHeap::heap()->phase_times())
{
add_serial_task(new ResetHotCardCacheTask());
add_serial_task(new PurgeCodeRootsTask());
#if COMPILER2_OR_JVMCI
add_serial_task(new UpdateDerivedPointersTask());
#endif
if (EagerlyReclaimHumongousObjectsTask::should_execute()) {
add_serial_task(new EagerlyReclaimHumongousObjectsTask());
}
if (RestorePreservedMarksTask::should_execute()) {
add_parallel_task(new RestorePreservedMarksTask(preserved_marks_set));
}
add_parallel_task(new RedirtyLoggedCardsTask(rdcqs));
add_parallel_task(new FreeCollectionSetTask(evacuation_info, surviving_young_words));
}

@ -0,0 +1,199 @@
/*
* Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*
*/
#ifndef SHARE_GC_G1_G1YOUNGGCPOSTEVACUATETASKS_HPP
#define SHARE_GC_G1_G1YOUNGGCPOSTEVACUATETASKS_HPP
#include "gc/g1/g1BatchedGangTask.hpp"
#include "gc/g1/g1EvacFailure.hpp"
class FreeCSetStats;
class G1CollectedHeap;
class G1EvacuationInfo;
class G1ParScanThreadStateSet;
class G1RedirtyCardsQueueSet;
// First set of post evacuate collection set tasks containing ("s" means serial):
// - Merge PSS (s)
// - Recalculate Used (s)
// - Remove Self Forwards (on evacuation failure)
// - Clear Card Table
class G1PostEvacuateCollectionSetCleanupTask1 : public G1BatchedGangTask {
class MergePssTask;
class RecalculateUsedTask;
class RemoveSelfForwardPtrsTask;
public:
G1PostEvacuateCollectionSetCleanupTask1(G1ParScanThreadStateSet* per_thread_states,
G1RedirtyCardsQueueSet* rdcqs);
};
class G1PostEvacuateCollectionSetCleanupTask1::MergePssTask : public G1AbstractSubTask {
G1ParScanThreadStateSet* _per_thread_states;
public:
MergePssTask(G1ParScanThreadStateSet* per_thread_states);
double worker_cost() const override { return 1.0; }
void do_work(uint worker_id) override;
};
class G1PostEvacuateCollectionSetCleanupTask1::RecalculateUsedTask : public G1AbstractSubTask {
public:
RecalculateUsedTask() : G1AbstractSubTask(G1GCPhaseTimes::RecalculateUsed) { }
double worker_cost() const override;
void do_work(uint worker_id) override;
};
class G1PostEvacuateCollectionSetCleanupTask1::RemoveSelfForwardPtrsTask : public G1AbstractSubTask {
G1ParRemoveSelfForwardPtrsTask _task;
public:
RemoveSelfForwardPtrsTask(G1RedirtyCardsQueueSet* rdcqs);
~RemoveSelfForwardPtrsTask();
static bool should_execute();
double worker_cost() const override;
void do_work(uint worker_id) override;
};
// Second set of post evacuate collection set tasks containing (s means serial):
// - Eagerly Reclaim Humongous Objects (s)
// - Purge Code Roots (s)
// - Reset Hot Card Cache (s)
// - Update Derived Pointers (s)
// - Redirty Logged Cards
// - Restore Preserved Marks (on evacuation failure)
// - Free Collection Set
class G1PostEvacuateCollectionSetCleanupTask2 : public G1BatchedGangTask {
class EagerlyReclaimHumongousObjectsTask;
class PurgeCodeRootsTask;
class ResetHotCardCacheTask;
#if COMPILER2_OR_JVMCI
class UpdateDerivedPointersTask;
#endif
class RedirtyLoggedCardsTask;
class RestorePreservedMarksTask;
class FreeCollectionSetTask;
public:
G1PostEvacuateCollectionSetCleanupTask2(PreservedMarksSet* preserved_marks_set,
G1RedirtyCardsQueueSet* rdcqs,
G1EvacuationInfo* evacuation_info,
const size_t* surviving_young_words);
};
class G1PostEvacuateCollectionSetCleanupTask2::ResetHotCardCacheTask : public G1AbstractSubTask {
public:
ResetHotCardCacheTask() : G1AbstractSubTask(G1GCPhaseTimes::ResetHotCardCache) { }
double worker_cost() const override { return 0.5; }
void do_work(uint worker_id) override;
};
class G1PostEvacuateCollectionSetCleanupTask2::PurgeCodeRootsTask : public G1AbstractSubTask {
public:
PurgeCodeRootsTask() : G1AbstractSubTask(G1GCPhaseTimes::PurgeCodeRoots) { }
double worker_cost() const override { return 1.0; }
void do_work(uint worker_id) override;
};
#if COMPILER2_OR_JVMCI
class G1PostEvacuateCollectionSetCleanupTask2::UpdateDerivedPointersTask : public G1AbstractSubTask {
public:
UpdateDerivedPointersTask() : G1AbstractSubTask(G1GCPhaseTimes::UpdateDerivedPointers) { }
double worker_cost() const override { return 1.0; }
void do_work(uint worker_id) override;
};
#endif
class G1PostEvacuateCollectionSetCleanupTask2::EagerlyReclaimHumongousObjectsTask : public G1AbstractSubTask {
uint _humongous_regions_reclaimed;
size_t _bytes_freed;
public:
EagerlyReclaimHumongousObjectsTask();
virtual ~EagerlyReclaimHumongousObjectsTask();
static bool should_execute();
double worker_cost() const override { return 1.0; }
void do_work(uint worker_id) override;
};
class G1PostEvacuateCollectionSetCleanupTask2::RestorePreservedMarksTask : public G1AbstractSubTask {
PreservedMarksSet* _preserved_marks;
AbstractGangTask* _task;
public:
RestorePreservedMarksTask(PreservedMarksSet* preserved_marks);
virtual ~RestorePreservedMarksTask();
static bool should_execute();
double worker_cost() const override;
void do_work(uint worker_id) override;
};
class G1PostEvacuateCollectionSetCleanupTask2::RedirtyLoggedCardsTask : public G1AbstractSubTask {
G1RedirtyCardsQueueSet* _rdcqs;
BufferNode* volatile _nodes;
public:
RedirtyLoggedCardsTask(G1RedirtyCardsQueueSet* rdcqs);
virtual ~RedirtyLoggedCardsTask();
double worker_cost() const override;
void do_work(uint worker_id) override;
};
class G1PostEvacuateCollectionSetCleanupTask2::FreeCollectionSetTask : public G1AbstractSubTask {
G1CollectedHeap* _g1h;
G1EvacuationInfo* _evacuation_info;
FreeCSetStats* _worker_stats;
HeapRegionClaimer _claimer;
const size_t* _surviving_young_words;
uint _active_workers;
FreeCSetStats* worker_stats(uint worker);
void report_statistics();
public:
FreeCollectionSetTask(G1EvacuationInfo* evacuation_info, const size_t* surviving_young_words);
virtual ~FreeCollectionSetTask();
double worker_cost() const override;
void set_max_workers(uint max_workers) override;
void do_work(uint worker_id) override;
};
#endif // SHARE_GC_G1_G1YOUNGGCPOSTEVACUATETASKS_HPP

@ -734,7 +734,6 @@ void HeapRegionManager::verify_optional() {
HeapRegionClaimer::HeapRegionClaimer(uint n_workers) :
_n_workers(n_workers), _n_regions(G1CollectedHeap::heap()->_hrm._allocated_heapregions_length), _claims(NULL) {
assert(n_workers > 0, "Need at least one worker.");
uint* new_claims = NEW_C_HEAP_ARRAY(uint, _n_regions, mtGC);
memset(new_claims, Unclaimed, sizeof(*_claims) * _n_regions);
_claims = new_claims;
@ -745,6 +744,7 @@ HeapRegionClaimer::~HeapRegionClaimer() {
}
uint HeapRegionClaimer::offset_for_worker(uint worker_id) const {
assert(_n_workers > 0, "must be set");
assert(worker_id < _n_workers, "Invalid worker_id.");
return _n_regions * worker_id / _n_workers;
}

@ -315,6 +315,11 @@ class HeapRegionClaimer : public StackObj {
return _n_regions;
}
void set_n_workers(uint n_workers) {
assert(_n_workers == 0, "already set");
assert(n_workers > 0, "must be");
_n_workers = n_workers;
}
// Return a start offset given a worker id.
uint offset_for_worker(uint worker_id) const;

@ -142,6 +142,10 @@ void PreservedMarksSet::restore(WorkGang* workers) {
assert_empty();
}
AbstractGangTask* PreservedMarksSet::create_task() {
return new RestorePreservedMarksTask(this);
}
void PreservedMarksSet::reclaim() {
assert_empty();

@ -30,6 +30,7 @@
#include "oops/oop.hpp"
#include "utilities/stack.hpp"
class AbstractGangTask;
class PreservedMarksSet;
class WorkGang;
@ -113,6 +114,8 @@ public:
// is NULL, perform the work serially in the current thread.
void restore(WorkGang* workers);
AbstractGangTask* create_task();
// Reclaim stack array.
void reclaim();

@ -64,7 +64,7 @@ void WorkerDataArray<size_t>::WDAPrinter::summary(outputStream* out, size_t min,
template <>
void WorkerDataArray<double>::WDAPrinter::details(const WorkerDataArray<double>* phase, outputStream* out) {
out->print("%-25s", "");
out->print("%-30s", "");
for (uint i = 0; i < phase->_length; ++i) {
double value = phase->get(i);
if (value != phase->uninitialized()) {
@ -78,7 +78,7 @@ void WorkerDataArray<double>::WDAPrinter::details(const WorkerDataArray<double>*
template <>
void WorkerDataArray<size_t>::WDAPrinter::details(const WorkerDataArray<size_t>* phase, outputStream* out) {
out->print("%-25s", "");
out->print("%-30s", "");
for (uint i = 0; i < phase->_length; ++i) {
size_t value = phase->get(i);
if (value != phase->uninitialized()) {

@ -150,7 +150,7 @@ void WorkerDataArray<T>::print_summary_on(outputStream* out, bool print_sum) con
if (_is_serial) {
out->print("%s:", title());
} else {
out->print("%-25s", title());
out->print("%-30s", title());
}
uint start = 0;

@ -54,7 +54,7 @@ class GangTaskDispatcher;
// An abstract task to be worked on by a gang.
// You subclass this to supply your own work() method
class AbstractGangTask {
class AbstractGangTask : public CHeapObj<mtInternal> {
const char* _name;
const uint _gc_id;

@ -76,7 +76,7 @@ class WorkerDataArrayTest : public ::testing::Test {
static const char* prepend_with(const char* str, const char* orig) {
stringStream out;
out.print("%-25s", str);
out.print("%-30s", str);
out.print("%s", orig);
return out.as_string();
}

@ -47,6 +47,12 @@ public class TestEagerReclaimHumongousRegionsLog {
private static final String LogSeparator = ": ";
static final String SumSeparator = "Sum: ";
private static String getSumValue(String s) {
return s.substring(s.indexOf(SumSeparator) + SumSeparator.length(), s.indexOf(", Workers"));
}
public static void runTest() throws Exception {
ProcessBuilder pb = ProcessTools.createJavaProcessBuilder(
"-Xbootclasspath/a:.",
@ -68,10 +74,10 @@ public class TestEagerReclaimHumongousRegionsLog {
// This gives an array of lines containing eager reclaim of humongous regions
// log messages contents after the ":" in the following order for every GC:
// Region Register: a.ams
// Humongous Total: b
// Humongous Candidate: c
// Humongous Reclaim: d.dms
// Humongous Reclaimed: e
// Eagerly Reclaim Humonguous Objects b.cms
// Humongous Total: Min: 1, Avg: 1.0, Max: 1, Diff: 0, Sum: c, Workers: 1
// Humongous Candidate: Min: 1, Avg: 1.0, Max: 1, Diff: 0, Sum: d, Workers: 1
// Humongous Reclaimed: Min: 1, Avg: 1.0, Max: 1, Diff: 0, Sum: e, Workers: 1
// Humongous Regions: f->g
String[] lines = Arrays.stream(output.getStdout().split("\\R"))
@ -81,9 +87,9 @@ public class TestEagerReclaimHumongousRegionsLog {
Asserts.assertTrue(lines.length % 6 == 0, "There seems to be an unexpected amount of log messages (total: " + lines.length + ") per GC");
for (int i = 0; i < lines.length; i += 6) {
int total = Integer.parseInt(lines[i + 1]);
int candidate = Integer.parseInt(lines[i + 2]);
int reclaimed = Integer.parseInt(lines[i + 4]);
int total = Integer.parseInt(getSumValue(lines[i + 2]));
int candidate = Integer.parseInt(getSumValue(lines[i + 3]));
int reclaimed = Integer.parseInt(getSumValue(lines[i + 4]));
int before = Integer.parseInt(lines[i + 5].substring(0, 1));
int after = Integer.parseInt(lines[i + 5].substring(3, 4));

@ -127,11 +127,10 @@ public class TestGCLogMessages {
new LogMessageWithLevel("JNI Global Roots", Level.TRACE),
new LogMessageWithLevel("VM Global Roots", Level.TRACE),
// Redirty Cards
new LogMessageWithLevel("Redirty Cards", Level.DEBUG),
new LogMessageWithLevel("Parallel Redirty", Level.TRACE),
new LogMessageWithLevel("Redirtied Cards", Level.TRACE),
new LogMessageWithLevel("Redirty Logged Cards", Level.DEBUG),
new LogMessageWithLevel("Redirtied Cards", Level.DEBUG),
// Misc Top-level
new LogMessageWithLevel("Code Roots Purge", Level.DEBUG),
new LogMessageWithLevel("Purge Code Roots", Level.DEBUG),
new LogMessageWithLevel("String Deduplication", Level.DEBUG),
new LogMessageWithLevel("Queue Fixup", Level.DEBUG),
new LogMessageWithLevel("Table Fixup", Level.DEBUG),
@ -142,7 +141,6 @@ public class TestGCLogMessages {
// Free CSet
new LogMessageWithLevel("Free Collection Set", Level.DEBUG),
new LogMessageWithLevel("Serial Free Collection Set", Level.TRACE),
new LogMessageWithLevel("Parallel Free Collection Set", Level.TRACE),
new LogMessageWithLevel("Young Free Collection Set", Level.TRACE),
new LogMessageWithLevel("Non-Young Free Collection Set", Level.TRACE),
// Rebuild Free List
@ -150,8 +148,6 @@ public class TestGCLogMessages {
new LogMessageWithLevel("Serial Rebuild Free List", Level.TRACE),
new LogMessageWithLevel("Parallel Rebuild Free List", Level.TRACE),
// Humongous Eager Reclaim
new LogMessageWithLevel("Humongous Reclaim", Level.DEBUG),
// Merge PSS
new LogMessageWithLevel("Merge Per-Thread State", Level.DEBUG),
// TLAB handling
@ -166,7 +162,7 @@ public class TestGCLogMessages {
new LogMessageWithLevel("ResolvedMethodTable Weak", Level.DEBUG),
new LogMessageWithLevel("VM Weak", Level.DEBUG),
new LogMessageWithLevelC2OrJVMCIOnly("DerivedPointerTable Update", Level.DEBUG),
new LogMessageWithLevelC2OrJVMCIOnly("Update Derived Pointers", Level.DEBUG),
new LogMessageWithLevel("Start New Collection Set", Level.DEBUG),
};
@ -238,9 +234,9 @@ public class TestGCLogMessages {
}
LogMessageWithLevel exhFailureMessages[] = new LogMessageWithLevel[] {
new LogMessageWithLevel("Evacuation Failure", Level.DEBUG),
new LogMessageWithLevel("Recalculate Used", Level.TRACE),
new LogMessageWithLevel("Remove Self Forwards", Level.TRACE),
new LogMessageWithLevel("Recalculate Used Memory", Level.DEBUG),
new LogMessageWithLevel("Restore Preserved Marks", Level.DEBUG),
new LogMessageWithLevel("Remove Self Forwards", Level.DEBUG),
};
private void testWithEvacuationFailureLogs() throws Exception {

@ -63,7 +63,7 @@ public class TestIHOPStatic {
"-XX:-G1UseAdaptiveIHOP",
"-XX:NewSize=" + YOUNG_SIZE,
"-XX:MaxNewSize=" + YOUNG_SIZE,
"-Xlog:gc+ihop+ergo=debug,gc*=debug"
"-Xlog:gc+ihop+ergo=debug"
};
public static void main(String[] args) throws Throwable {

@ -68,7 +68,7 @@ public class TestPLABEvacuationFailure {
"failure wasted"));
private static final String[] COMMON_OPTIONS = {
"-Xlog:gc=debug,gc+plab=debug,gc+phases=trace",
"-Xlog:gc=debug,gc+phases=trace",
"-XX:+UseG1GC",
"-XX:InitiatingHeapOccupancyPercent=100",
"-XX:-G1UseAdaptiveIHOP",
@ -115,6 +115,7 @@ public class TestPLABEvacuationFailure {
System.out.println(appPlabEvacFailureOutput);
throw new RuntimeException("Expect exit code 0.");
}
// Get list of GC ID on evacuation failure
evacuationFailureIDs = getGcIdPlabEvacFailures(out);
logParser = new LogParser(appPlabEvacFailureOutput);
@ -195,7 +196,7 @@ public class TestPLABEvacuationFailure {
private static List<Long> getGcIdPlabEvacFailures(OutputAnalyzer out) {
return out.asLines().stream()
.filter(line -> line.contains("Evacuation Failure"))
.filter(line -> line.contains("space exhausted"))
.map(line -> LogParser.getGcIdFromLine(line, GC_ID_PATTERN))
.collect(Collectors.toList());
}

@ -107,7 +107,14 @@ public class TestG1ParallelPhases {
"StringDedupQueueFixup",
"StringDedupTableFixup",
"RedirtyCards",
"ParFreeCSet",
"RecalculateUsed",
"ResetHotCardCache",
"FreeCSet",
"PurgeCodeRoots",
"UpdateDerivedPointers",
"EagerlyReclaimHumongousObjects",
"ClearLoggedCards",
"MergePSS",
"NonYoungFreeCSet",
"YoungFreeCSet",
"RebuildFreeList"