stan/jdk-24
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

8245022: ParallelGC abuses StarTask to also include partial objarray scan tasks

Browse Source 
Change ParallelGC young collections to use ScannerTasks.

Reviewed-by: tschatzl, sjohanss
This commit is contained in:
Kim Barrett 2020-05-20 17:21:16 -04:00
parent 275079ce7f
commit 5b3c33ac67
10 changed files with 79 additions and 120 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
src/hotspot/share/gc/g1/g1CollectedHeap.cpp
View File

@ -1542,12 +1542,12 @@ G1CollectedHeap::G1CollectedHeap() :
_filler_array_max_size = _humongous_object_threshold_in_words;
uint n_queues = ParallelGCThreads;
_task_queues = new ScannerTasksQueueSet(n_queues);
_task_queues = new G1ScannerTasksQueueSet(n_queues);
_evacuation_failed_info_array = NEW_C_HEAP_ARRAY(EvacuationFailedInfo, n_queues, mtGC);
for (uint i = 0; i < n_queues; i++) {
ScannerTasksQueue* q = new ScannerTasksQueue();
G1ScannerTasksQueue* q = new G1ScannerTasksQueue();
q->initialize();
_task_queues->register_queue(i, q);
::new (&_evacuation_failed_info_array[i]) EvacuationFailedInfo();
@ -3436,14 +3436,14 @@ class G1STWRefProcTaskExecutor: public AbstractRefProcTaskExecutor {
private:
G1CollectedHeap* _g1h;
G1ParScanThreadStateSet* _pss;
ScannerTasksQueueSet* _queues;
G1ScannerTasksQueueSet* _queues;
WorkGang* _workers;
public:
G1STWRefProcTaskExecutor(G1CollectedHeap* g1h,
G1ParScanThreadStateSet* per_thread_states,
WorkGang* workers,
ScannerTasksQueueSet *task_queues) :
G1ScannerTasksQueueSet *task_queues) :
_g1h(g1h),
_pss(per_thread_states),
_queues(task_queues),
@ -3463,14 +3463,14 @@ class G1STWRefProcTaskProxy: public AbstractGangTask {
ProcessTask& _proc_task;
G1CollectedHeap* _g1h;
G1ParScanThreadStateSet* _pss;
ScannerTasksQueueSet* _task_queues;
G1ScannerTasksQueueSet* _task_queues;
TaskTerminator* _terminator;
public:
G1STWRefProcTaskProxy(ProcessTask& proc_task,
G1CollectedHeap* g1h,
G1ParScanThreadStateSet* per_thread_states,
ScannerTasksQueueSet *task_queues,
G1ScannerTasksQueueSet *task_queues,
TaskTerminator* terminator) :
AbstractGangTask("Process reference objects in parallel"),
_proc_task(proc_task),
@ -3801,7 +3801,7 @@ class G1EvacuateRegionsBaseTask : public AbstractGangTask {
protected:
G1CollectedHeap* _g1h;
G1ParScanThreadStateSet* _per_thread_states;
ScannerTasksQueueSet* _task_queues;
G1ScannerTasksQueueSet* _task_queues;
TaskTerminator _terminator;
uint _num_workers;
@ -3841,7 +3841,7 @@ protected:
public:
G1EvacuateRegionsBaseTask(const char* name,
G1ParScanThreadStateSet* per_thread_states,
ScannerTasksQueueSet* task_queues,
G1ScannerTasksQueueSet* task_queues,
uint num_workers) :
AbstractGangTask(name),
_g1h(G1CollectedHeap::heap()),
@ -3893,7 +3893,7 @@ class G1EvacuateRegionsTask : public G1EvacuateRegionsBaseTask {
public:
G1EvacuateRegionsTask(G1CollectedHeap* g1h,
G1ParScanThreadStateSet* per_thread_states,
ScannerTasksQueueSet* task_queues,
G1ScannerTasksQueueSet* task_queues,
G1RootProcessor* root_processor,
uint num_workers) :
G1EvacuateRegionsBaseTask("G1 Evacuate Regions", per_thread_states, task_queues, num_workers),
@ -3941,7 +3941,7 @@ class G1EvacuateOptionalRegionsTask : public G1EvacuateRegionsBaseTask {
public:
G1EvacuateOptionalRegionsTask(G1ParScanThreadStateSet* per_thread_states,
ScannerTasksQueueSet* queues,
G1ScannerTasksQueueSet* queues,
uint num_workers) :
G1EvacuateRegionsBaseTask("G1 Evacuate Optional Regions", per_thread_states, queues, num_workers) {
}

14
src/hotspot/share/gc/g1/g1CollectedHeap.hpp
View File

@ -98,8 +98,8 @@ class G1HeapSizingPolicy;
class G1HeapSummary;
class G1EvacSummary;
typedef OverflowTaskQueue<ScannerTask, mtGC> ScannerTasksQueue;
typedef GenericTaskQueueSet<ScannerTasksQueue, mtGC> ScannerTasksQueueSet;
typedef OverflowTaskQueue<ScannerTask, mtGC> G1ScannerTasksQueue;
typedef GenericTaskQueueSet<G1ScannerTasksQueue, mtGC> G1ScannerTasksQueueSet;
typedef int RegionIdx_t; // needs to hold [ 0..max_regions() )
typedef int CardIdx_t; // needs to hold [ 0..CardsPerRegion )
@ -815,7 +815,7 @@ public:
G1ConcurrentRefine* _cr;
// The parallel task queues
ScannerTasksQueueSet *_task_queues;
G1ScannerTasksQueueSet *_task_queues;
// True iff a evacuation has failed in the current collection.
bool _evacuation_failed;
@ -952,7 +952,7 @@ public:
G1CMSubjectToDiscoveryClosure _is_subject_to_discovery_cm;
public:
ScannerTasksQueue* task_queue(uint i) const;
G1ScannerTasksQueue* task_queue(uint i) const;
uint num_task_queues() const;
@ -1479,18 +1479,18 @@ private:
protected:
G1CollectedHeap* _g1h;
G1ParScanThreadState* _par_scan_state;
ScannerTasksQueueSet* _queues;
G1ScannerTasksQueueSet* _queues;
TaskTerminator* _terminator;
G1GCPhaseTimes::GCParPhases _phase;
G1ParScanThreadState* par_scan_state() { return _par_scan_state; }
ScannerTasksQueueSet* queues() { return _queues; }
G1ScannerTasksQueueSet* queues() { return _queues; }
TaskTerminator* terminator() { return _terminator; }
public:
G1ParEvacuateFollowersClosure(G1CollectedHeap* g1h,
G1ParScanThreadState* par_scan_state,
ScannerTasksQueueSet* queues,
G1ScannerTasksQueueSet* queues,
TaskTerminator* terminator,
G1GCPhaseTimes::GCParPhases phase)
: _start_term(0.0), _term_time(0.0), _term_attempts(0),

2
src/hotspot/share/gc/g1/g1CollectedHeap.inline.hpp
View File

@ -139,7 +139,7 @@ G1CollectedHeap::dirty_young_block(HeapWord* start, size_t word_size) {
card_table()->g1_mark_as_young(mr);
}
inline ScannerTasksQueue* G1CollectedHeap::task_queue(uint i) const {
inline G1ScannerTasksQueue* G1CollectedHeap::task_queue(uint i) const {
return _task_queues->queue(i);
}

4
src/hotspot/share/gc/g1/g1ParScanThreadState.hpp
View File

@ -46,7 +46,7 @@ class outputStream;
class G1ParScanThreadState : public CHeapObj<mtGC> {
G1CollectedHeap* _g1h;
ScannerTasksQueue* _task_queue;
G1ScannerTasksQueue* _task_queue;
G1RedirtyCardsQueue _rdcq;
G1CardTable* _ct;
G1EvacuationRootClosures* _closures;
@ -202,7 +202,7 @@ public:
Tickspan trim_ticks() const;
void reset_trim_ticks();
inline void steal_and_trim_queue(ScannerTasksQueueSet *task_queues);
inline void steal_and_trim_queue(G1ScannerTasksQueueSet *task_queues);
// An attempt to evacuate "obj" has failed; take necessary steps.
oop handle_evacuation_failure_par(oop obj, markWord m);

2
src/hotspot/share/gc/g1/g1ParScanThreadState.inline.hpp
View File

@ -136,7 +136,7 @@ inline void G1ParScanThreadState::dispatch_task(ScannerTask task) {
}
}
void G1ParScanThreadState::steal_and_trim_queue(ScannerTasksQueueSet *task_queues) {
void G1ParScanThreadState::steal_and_trim_queue(G1ScannerTasksQueueSet *task_queues) {
ScannerTask stolen_task;
while (task_queues->steal(_worker_id, stolen_task)) {
dispatch_task(stolen_task);

4
src/hotspot/share/gc/parallel/psClosure.inline.hpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2018, 2020, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -40,7 +40,7 @@ private:
template <class T> void do_oop_work(T *p) {
if (PSScavenge::should_scavenge(p)) {
// We never card mark roots, maybe call a func without test?
_promotion_manager->copy_and_push_safe_barrier<T, promote_immediately>(p);
_promotion_manager->copy_and_push_safe_barrier<promote_immediately>(p);
}
}
public:

35
src/hotspot/share/gc/parallel/psPromotionManager.cpp
View File

@ -43,7 +43,7 @@
#include "oops/compressedOops.inline.hpp"
PaddedEnd<PSPromotionManager>* PSPromotionManager::_manager_array = NULL;
PSPromotionManager::OopStarTaskQueueSet* PSPromotionManager::_stack_array_depth = NULL;
PSPromotionManager::PSScannerTasksQueueSet* PSPromotionManager::_stack_array_depth = NULL;
PreservedMarksSet* PSPromotionManager::_preserved_marks_set = NULL;
PSOldGen* PSPromotionManager::_old_gen = NULL;
MutableSpace* PSPromotionManager::_young_space = NULL;
@ -61,7 +61,7 @@ void PSPromotionManager::initialize() {
assert(_manager_array == NULL, "Attempt to initialize twice");
_manager_array = PaddedArray<PSPromotionManager, mtGC>::create_unfreeable(promotion_manager_num);
_stack_array_depth = new OopStarTaskQueueSet(ParallelGCThreads);
_stack_array_depth = new PSScannerTasksQueueSet(ParallelGCThreads);
// Create and register the PSPromotionManager(s) for the worker threads.
for(uint i=0; i<ParallelGCThreads; i++) {
@ -134,13 +134,14 @@ bool PSPromotionManager::post_scavenge(YoungGCTracer& gc_tracer) {
void
PSPromotionManager::print_local_stats(outputStream* const out, uint i) const {
#define FMT " " SIZE_FORMAT_W(10)
out->print_cr("%3u" FMT FMT FMT FMT, i, _masked_pushes, _masked_steals,
out->print_cr("%3u" FMT FMT FMT FMT,
i, _array_chunk_pushes, _array_chunk_steals,
_arrays_chunked, _array_chunks_processed);
#undef FMT
}
static const char* const pm_stats_hdr[] = {
" --------masked------- arrays array",
" ----partial array---- arrays array",
"thr push steal chunked chunks",
"--- ---------- ---------- ---------- ----------"
};
@ -177,7 +178,7 @@ PSPromotionManager::print_taskqueue_stats() {
void
PSPromotionManager::reset_stats() {
claimed_stack_depth()->stats.reset();
_masked_pushes = _masked_steals = 0;
_array_chunk_pushes = _array_chunk_steals = 0;
_arrays_chunked = _array_chunks_processed = 0;
}
#endif // TASKQUEUE_STATS
@ -249,23 +250,23 @@ void PSPromotionManager::drain_stacks_depth(bool totally_drain) {
MutableSpace* old_space = heap->old_gen()->object_space();
#endif /* ASSERT */
OopStarTaskQueue* const tq = claimed_stack_depth();
PSScannerTasksQueue* const tq = claimed_stack_depth();
do {
StarTask p;
ScannerTask task;
// Drain overflow stack first, so other threads can steal from
// claimed stack while we work.
while (tq->pop_overflow(p)) {
process_popped_location_depth(p);
while (tq->pop_overflow(task)) {
process_popped_location_depth(task);
}
if (totally_drain) {
while (tq->pop_local(p)) {
process_popped_location_depth(p);
while (tq->pop_local(task)) {
process_popped_location_depth(task);
}
} else {
while (tq->size() > _target_stack_size && tq->pop_local(p)) {
process_popped_location_depth(p);
while (tq->size() > _target_stack_size && tq->pop_local(task)) {
process_popped_location_depth(task);
}
}
} while ((totally_drain && !tq->taskqueue_empty()) || !tq->overflow_empty());
@ -309,8 +310,10 @@ template <class T> void PSPromotionManager::process_array_chunk_work(
}
}
void PSPromotionManager::process_array_chunk(oop old) {
void PSPromotionManager::process_array_chunk(PartialArrayScanTask task) {
assert(PSChunkLargeArrays, "invariant");
oop old = task.to_source_array();
assert(old->is_objArray(), "invariant");
assert(old->is_forwarded(), "invariant");
@ -325,8 +328,8 @@ void PSPromotionManager::process_array_chunk(oop old) {
start = end - _array_chunk_size;
assert(start > 0, "invariant");
arrayOop(old)->set_length(start);
push_depth(mask_chunked_array_oop(old));
TASKQUEUE_STATS_ONLY(++_masked_pushes);
push_depth(ScannerTask(PartialArrayScanTask(old)));
TASKQUEUE_STATS_ONLY(++_array_chunk_pushes);
} else {
// this is the final chunk for this array
start = 0;

71
src/hotspot/share/gc/parallel/psPromotionManager.hpp
View File

@ -56,18 +56,18 @@ class PSPromotionManager {
friend class PSRefProcTask;
private:
typedef OverflowTaskQueue<StarTask, mtGC> OopStarTaskQueue;
typedef GenericTaskQueueSet<OopStarTaskQueue, mtGC> OopStarTaskQueueSet;
typedef OverflowTaskQueue<ScannerTask, mtGC> PSScannerTasksQueue;
typedef GenericTaskQueueSet<PSScannerTasksQueue, mtGC> PSScannerTasksQueueSet;
static PaddedEnd<PSPromotionManager>* _manager_array;
static OopStarTaskQueueSet* _stack_array_depth;
static PSScannerTasksQueueSet* _stack_array_depth;
static PreservedMarksSet* _preserved_marks_set;
static PSOldGen* _old_gen;
static MutableSpace* _young_space;
#if TASKQUEUE_STATS
size_t _masked_pushes;
size_t _masked_steals;
size_t _array_chunk_pushes;
size_t _array_chunk_steals;
size_t _arrays_chunked;
size_t _array_chunks_processed;
@ -82,7 +82,7 @@ class PSPromotionManager {
bool _young_gen_is_full;
bool _old_gen_is_full;
OopStarTaskQueue _claimed_stack_depth;
PSScannerTasksQueue _claimed_stack_depth;
OverflowTaskQueue<oop, mtGC> _claimed_stack_breadth;
bool _totally_drain;
@ -101,59 +101,18 @@ class PSPromotionManager {
inline static PSPromotionManager* manager_array(uint index);
template <class T> inline void claim_or_forward_internal_depth(T* p);
// On the task queues we push reference locations as well as
// partially-scanned arrays (in the latter case, we push an oop to
// the from-space image of the array and the length on the
// from-space image indicates how many entries on the array we still
// need to scan. To be able to distinguish between reference
// locations and partially-scanned array oops we simply mask the
// latter oops with 0x01. The next three methods do the masking,
// unmasking, and checking whether the oop is masked or not. Notice
// that the signature of the mask and unmask methods looks a bit
// strange, as they accept and return different types (oop and
// oop*). This is because of the difference in types between what
// the task queue holds (oop*) and oops to partially-scanned arrays
// (oop). We do all the necessary casting in the mask / unmask
// methods to avoid sprinkling the rest of the code with more casts.
// These are added to the taskqueue so PS_CHUNKED_ARRAY_OOP_MASK (or any
// future masks) can't conflict with COMPRESSED_OOP_MASK
#define PS_CHUNKED_ARRAY_OOP_MASK 0x2
bool is_oop_masked(StarTask p) {
// If something is marked chunked it's always treated like wide oop*
return (((intptr_t)(oop*)p) & PS_CHUNKED_ARRAY_OOP_MASK) ==
PS_CHUNKED_ARRAY_OOP_MASK;
}
oop* mask_chunked_array_oop(oop obj) {
assert(!is_oop_masked(cast_from_oop<oop*>(obj)), "invariant");
oop* ret = (oop*) (cast_from_oop<uintptr_t>(obj) | PS_CHUNKED_ARRAY_OOP_MASK);
assert(is_oop_masked(ret), "invariant");
return ret;
}
oop unmask_chunked_array_oop(StarTask p) {
assert(is_oop_masked(p), "invariant");
assert(!p.is_narrow(), "chunked array oops cannot be narrow");
oop *chunk = (oop*)p; // cast p to oop (uses conversion operator)
oop ret = oop((oop*)((uintptr_t)chunk & ~PS_CHUNKED_ARRAY_OOP_MASK));
assert(!is_oop_masked(cast_from_oop<oop*>(ret)), "invariant");
return ret;
}
template <class T> void process_array_chunk_work(oop obj,
int start, int end);
void process_array_chunk(oop old);
void process_array_chunk(PartialArrayScanTask task);
template <class T> void push_depth(T* p);
void push_depth(ScannerTask task);
inline void promotion_trace_event(oop new_obj, oop old_obj, size_t obj_size,
uint age, bool tenured,
const PSPromotionLAB* lab);
protected:
static OopStarTaskQueueSet* stack_array_depth() { return _stack_array_depth; }
static PSScannerTasksQueueSet* stack_array_depth() { return _stack_array_depth; }
public:
// Static
static void initialize();
@ -164,12 +123,12 @@ class PSPromotionManager {
static PSPromotionManager* gc_thread_promotion_manager(uint index);
static PSPromotionManager* vm_thread_promotion_manager();
static bool steal_depth(int queue_num, StarTask& t);
static bool steal_depth(int queue_num, ScannerTask& t);
PSPromotionManager();
// Accessors
OopStarTaskQueue* claimed_stack_depth() {
PSScannerTasksQueue* claimed_stack_depth() {
return &_claimed_stack_depth;
}
@ -202,17 +161,17 @@ class PSPromotionManager {
return claimed_stack_depth()->is_empty();
}
inline void process_popped_location_depth(StarTask p);
inline void process_popped_location_depth(ScannerTask task);
static bool should_scavenge(oop* p, bool check_to_space = false);
static bool should_scavenge(narrowOop* p, bool check_to_space = false);
template <class T, bool promote_immediately>
template <bool promote_immediately, class T>
void copy_and_push_safe_barrier(T* p);
template <class T> inline void claim_or_forward_depth(T* p);
TASKQUEUE_STATS_ONLY(inline void record_steal(StarTask& p);)
TASKQUEUE_STATS_ONLY(inline void record_steal(ScannerTask task);)
void push_contents(oop obj);
};

37
src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2002, 2019, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2002, 2020, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -43,9 +43,8 @@ inline PSPromotionManager* PSPromotionManager::manager_array(uint index) {
return &_manager_array[index];
}
template <class T>
inline void PSPromotionManager::push_depth(T* p) {
claimed_stack_depth()->push(p);
inline void PSPromotionManager::push_depth(ScannerTask task) {
claimed_stack_depth()->push(task);
}
template <class T>
@ -60,7 +59,7 @@ inline void PSPromotionManager::claim_or_forward_internal_depth(T* p) {
}
RawAccess<IS_NOT_NULL>::oop_store(p, o);
} else {
push_depth(p);
push_depth(ScannerTask(p));
}
}
}
@ -275,9 +274,8 @@ inline oop PSPromotionManager::copy_to_survivor_space(oop o) {
new_obj->is_objArray() &&
PSChunkLargeArrays) {
// we'll chunk it
oop* const masked_o = mask_chunked_array_oop(o);
push_depth(masked_o);
TASKQUEUE_STATS_ONLY(++_arrays_chunked; ++_masked_pushes);
push_depth(ScannerTask(PartialArrayScanTask(o)));
TASKQUEUE_STATS_ONLY(++_arrays_chunked; ++_array_chunk_pushes);
} else {
// we'll just push its contents
push_contents(new_obj);
@ -318,7 +316,7 @@ inline oop PSPromotionManager::copy_to_survivor_space(oop o) {
// Attempt to "claim" oop at p via CAS, push the new obj if successful
// This version tests the oop* to make sure it is within the heap before
// attempting marking.
template <class T, bool promote_immediately>
template <bool promote_immediately, class T>
inline void PSPromotionManager::copy_and_push_safe_barrier(T* p) {
assert(should_scavenge(p, true), "revisiting object?");
@ -348,29 +346,28 @@ inline void PSPromotionManager::copy_and_push_safe_barrier(T* p) {
}
}
inline void PSPromotionManager::process_popped_location_depth(StarTask p) {
if (is_oop_masked(p)) {
inline void PSPromotionManager::process_popped_location_depth(ScannerTask task) {
if (task.is_partial_array_task()) {
assert(PSChunkLargeArrays, "invariant");
oop const old = unmask_chunked_array_oop(p);
process_array_chunk(old);
process_array_chunk(task.to_partial_array_task());
} else {
if (p.is_narrow()) {
if (task.is_narrow_oop_ptr()) {
assert(UseCompressedOops, "Error");
copy_and_push_safe_barrier<narrowOop, /*promote_immediately=*/false>(p);
copy_and_push_safe_barrier</*promote_immediately=*/false>(task.to_narrow_oop_ptr());
} else {
copy_and_push_safe_barrier<oop, /*promote_immediately=*/false>(p);
copy_and_push_safe_barrier</*promote_immediately=*/false>(task.to_oop_ptr());
}
}
}
inline bool PSPromotionManager::steal_depth(int queue_num, StarTask& t) {
inline bool PSPromotionManager::steal_depth(int queue_num, ScannerTask& t) {
return stack_array_depth()->steal(queue_num, t);
}
#if TASKQUEUE_STATS
void PSPromotionManager::record_steal(StarTask& p) {
if (is_oop_masked(p)) {
++_masked_steals;
void PSPromotionManager::record_steal(ScannerTask task) {
if (task.is_partial_array_task()) {
++_array_chunk_steals;
}
}
#endif // TASKQUEUE_STATS

10
src/hotspot/share/gc/parallel/psScavenge.cpp
View File

@ -149,10 +149,10 @@ static void steal_work(TaskTerminator& terminator, uint worker_id) {
"stacks should be empty at this point");
while (true) {
StarTask p;
if (PSPromotionManager::steal_depth(worker_id, p)) {
TASKQUEUE_STATS_ONLY(pm->record_steal(p));
pm->process_popped_location_depth(p);
ScannerTask task;
if (PSPromotionManager::steal_depth(worker_id, task)) {
TASKQUEUE_STATS_ONLY(pm->record_steal(task));
pm->process_popped_location_depth(task);
pm->drain_stacks_depth(true);
} else {
if (terminator.offer_termination()) {
@ -192,7 +192,7 @@ public:
// Weak refs may be visited more than once.
if (PSScavenge::should_scavenge(p, _to_space)) {
_promotion_manager->copy_and_push_safe_barrier<T, /*promote_immediately=*/false>(p);
_promotion_manager->copy_and_push_safe_barrier</*promote_immediately=*/false>(p);
}
}
virtual void do_oop(oop* p) { PSKeepAliveClosure::do_oop_work(p); }