stan/jdk-24
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

6923991: G1: improve scalability of RSet scanning

Browse Source 
Implemented block-based work stealing. Moved copying during the rset scanning phase to the main copying phase. Made the size of rset table depend on the region size.

Reviewed-by: apetrusenko, tonyp
This commit is contained in:
Igor Veresov 2010-02-11 15:52:19 -08:00
parent 0356567ed8
commit 52b92d5593
14 changed files with 263 additions and 168 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
hotspot/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp
View File

@ -2646,6 +2646,13 @@ G1CollectedHeap::cleanup_surviving_young_words() {
// </NEW PREDICTION> // </NEW PREDICTION>
struct PrepareForRSScanningClosure : public HeapRegionClosure {
bool doHeapRegion(HeapRegion *r) {
r->rem_set()->set_iter_claimed(0);
return false;
}
};
void void
G1CollectedHeap::do_collection_pause_at_safepoint() { G1CollectedHeap::do_collection_pause_at_safepoint() {
if (PrintHeapAtGC) { if (PrintHeapAtGC) {
@ -2784,6 +2791,8 @@ G1CollectedHeap::do_collection_pause_at_safepoint() {
gclog_or_tty->print_cr("\nAfter pause, heap:"); gclog_or_tty->print_cr("\nAfter pause, heap:");
print(); print();
#endif #endif
PrepareForRSScanningClosure prepare_for_rs_scan;
collection_set_iterate(&prepare_for_rs_scan);
setup_surviving_young_words(); setup_surviving_young_words();
@ -3781,22 +3790,16 @@ oop G1ParCopyHelper::copy_to_survivor_space(oop old) {
return obj; return obj;
} }
template <bool do_gen_barrier, G1Barrier barrier, bool do_mark_forwardee, bool skip_cset_test> template <bool do_gen_barrier, G1Barrier barrier, bool do_mark_forwardee>
template <class T> template <class T>
void G1ParCopyClosure <do_gen_barrier, barrier, do_mark_forwardee, skip_cset_test> void G1ParCopyClosure <do_gen_barrier, barrier, do_mark_forwardee>
::do_oop_work(T* p) { ::do_oop_work(T* p) {
oop obj = oopDesc::load_decode_heap_oop(p); oop obj = oopDesc::load_decode_heap_oop(p);
assert(barrier != G1BarrierRS || obj != NULL, assert(barrier != G1BarrierRS || obj != NULL,
"Precondition: G1BarrierRS implies obj is nonNull"); "Precondition: G1BarrierRS implies obj is nonNull");
// The only time we skip the cset test is when we're scanning
// references popped from the queue. And we only push on the queue
// references that we know point into the cset, so no point in
// checking again. But we'll leave an assert here for peace of mind.
assert(!skip_cset_test || _g1->obj_in_cs(obj), "invariant");
// here the null check is implicit in the cset_fast_test() test // here the null check is implicit in the cset_fast_test() test
if (skip_cset_test || _g1->in_cset_fast_test(obj)) { if (_g1->in_cset_fast_test(obj)) {
#if G1_REM_SET_LOGGING #if G1_REM_SET_LOGGING
gclog_or_tty->print_cr("Loc "PTR_FORMAT" contains pointer "PTR_FORMAT" " gclog_or_tty->print_cr("Loc "PTR_FORMAT" contains pointer "PTR_FORMAT" "
"into CS.", p, (void*) obj); "into CS.", p, (void*) obj);
@ -3813,7 +3816,6 @@ void G1ParCopyClosure <do_gen_barrier, barrier, do_mark_forwardee, skip_cset_tes
} }
} }
// When scanning moved objs, must look at all oops.
if (barrier == G1BarrierEvac && obj != NULL) { if (barrier == G1BarrierEvac && obj != NULL) {
_par_scan_state->update_rs(_from, p, _par_scan_state->queue_num()); _par_scan_state->update_rs(_from, p, _par_scan_state->queue_num());
} }
@ -3823,8 +3825,8 @@ void G1ParCopyClosure <do_gen_barrier, barrier, do_mark_forwardee, skip_cset_tes
} }
} }
template void G1ParCopyClosure<false, G1BarrierEvac, false, true>::do_oop_work(oop* p); template void G1ParCopyClosure<false, G1BarrierEvac, false>::do_oop_work(oop* p);
template void G1ParCopyClosure<false, G1BarrierEvac, false, true>::do_oop_work(narrowOop* p); template void G1ParCopyClosure<false, G1BarrierEvac, false>::do_oop_work(narrowOop* p);
template <class T> void G1ParScanPartialArrayClosure::do_oop_nv(T* p) { template <class T> void G1ParScanPartialArrayClosure::do_oop_nv(T* p) {
assert(has_partial_array_mask(p), "invariant"); assert(has_partial_array_mask(p), "invariant");
@ -3896,11 +3898,11 @@ public:
assert(UseCompressedOops, "Error"); assert(UseCompressedOops, "Error");
narrowOop* p = (narrowOop*) stolen_task; narrowOop* p = (narrowOop*) stolen_task;
assert(has_partial_array_mask(p) || assert(has_partial_array_mask(p) ||
_g1h->obj_in_cs(oopDesc::load_decode_heap_oop(p)), "Error"); _g1h->is_in_g1_reserved(oopDesc::load_decode_heap_oop(p)), "Error");
pss->push_on_queue(p); pss->push_on_queue(p);
} else { } else {
oop* p = (oop*) stolen_task; oop* p = (oop*) stolen_task;
assert(has_partial_array_mask(p) || _g1h->obj_in_cs(*p), "Error"); assert(has_partial_array_mask(p) || _g1h->is_in_g1_reserved(*p), "Error");
pss->push_on_queue(p); pss->push_on_queue(p);
} }
continue; continue;
@ -3962,6 +3964,7 @@ public:
G1ParScanExtRootClosure only_scan_root_cl(_g1h, &pss); G1ParScanExtRootClosure only_scan_root_cl(_g1h, &pss);
G1ParScanPermClosure only_scan_perm_cl(_g1h, &pss); G1ParScanPermClosure only_scan_perm_cl(_g1h, &pss);
G1ParScanHeapRSClosure only_scan_heap_rs_cl(_g1h, &pss); G1ParScanHeapRSClosure only_scan_heap_rs_cl(_g1h, &pss);
G1ParPushHeapRSClosure push_heap_rs_cl(_g1h, &pss);
G1ParScanAndMarkExtRootClosure scan_mark_root_cl(_g1h, &pss); G1ParScanAndMarkExtRootClosure scan_mark_root_cl(_g1h, &pss);
G1ParScanAndMarkPermClosure scan_mark_perm_cl(_g1h, &pss); G1ParScanAndMarkPermClosure scan_mark_perm_cl(_g1h, &pss);
@ -3985,7 +3988,7 @@ public:
_g1h->g1_process_strong_roots(/* not collecting perm */ false, _g1h->g1_process_strong_roots(/* not collecting perm */ false,
SharedHeap::SO_AllClasses, SharedHeap::SO_AllClasses,
scan_root_cl, scan_root_cl,
&only_scan_heap_rs_cl, &push_heap_rs_cl,
scan_so_cl, scan_so_cl,
scan_perm_cl, scan_perm_cl,
i); i);

24
hotspot/src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp
View File

@ -1623,7 +1623,7 @@ public:
template <class T> void push_on_queue(T* ref) { template <class T> void push_on_queue(T* ref) {
assert(ref != NULL, "invariant"); assert(ref != NULL, "invariant");
assert(has_partial_array_mask(ref) || assert(has_partial_array_mask(ref) ||
_g1h->obj_in_cs(oopDesc::load_decode_heap_oop(ref)), "invariant"); _g1h->is_in_g1_reserved(oopDesc::load_decode_heap_oop(ref)), "invariant");
#ifdef ASSERT #ifdef ASSERT
if (has_partial_array_mask(ref)) { if (has_partial_array_mask(ref)) {
oop p = clear_partial_array_mask(ref); oop p = clear_partial_array_mask(ref);
@ -1644,9 +1644,9 @@ public:
assert((oop*)ref != NULL, "pop_local() returned true"); assert((oop*)ref != NULL, "pop_local() returned true");
assert(UseCompressedOops || !ref.is_narrow(), "Error"); assert(UseCompressedOops || !ref.is_narrow(), "Error");
assert(has_partial_array_mask((oop*)ref) || assert(has_partial_array_mask((oop*)ref) ||
_g1h->obj_in_cs(ref.is_narrow() ? oopDesc::load_decode_heap_oop((narrowOop*)ref) _g1h->is_in_g1_reserved(ref.is_narrow() ? oopDesc::load_decode_heap_oop((narrowOop*)ref)
: oopDesc::load_decode_heap_oop((oop*)ref)), : oopDesc::load_decode_heap_oop((oop*)ref)),
"invariant"); "invariant");
IF_G1_DETAILED_STATS(note_pop()); IF_G1_DETAILED_STATS(note_pop());
} else { } else {
StarTask null_task; StarTask null_task;
@ -1659,9 +1659,9 @@ public:
assert((oop*)new_ref != NULL, "pop() from a local non-empty stack"); assert((oop*)new_ref != NULL, "pop() from a local non-empty stack");
assert(UseCompressedOops || !new_ref.is_narrow(), "Error"); assert(UseCompressedOops || !new_ref.is_narrow(), "Error");
assert(has_partial_array_mask((oop*)new_ref) || assert(has_partial_array_mask((oop*)new_ref) ||
_g1h->obj_in_cs(new_ref.is_narrow() ? oopDesc::load_decode_heap_oop((narrowOop*)new_ref) _g1h->is_in_g1_reserved(new_ref.is_narrow() ? oopDesc::load_decode_heap_oop((narrowOop*)new_ref)
: oopDesc::load_decode_heap_oop((oop*)new_ref)), : oopDesc::load_decode_heap_oop((oop*)new_ref)),
"invariant"); "invariant");
ref = new_ref; ref = new_ref;
} }
@ -1825,12 +1825,12 @@ public:
assert(UseCompressedOops, "Error"); assert(UseCompressedOops, "Error");
narrowOop* p = (narrowOop*)ref_to_scan; narrowOop* p = (narrowOop*)ref_to_scan;
assert(!has_partial_array_mask(p) && assert(!has_partial_array_mask(p) &&
_g1h->obj_in_cs(oopDesc::load_decode_heap_oop(p)), "sanity"); _g1h->is_in_g1_reserved(oopDesc::load_decode_heap_oop(p)), "sanity");
deal_with_reference(p); deal_with_reference(p);
} else { } else {
oop* p = (oop*)ref_to_scan; oop* p = (oop*)ref_to_scan;
assert((has_partial_array_mask(p) && _g1h->obj_in_cs(clear_partial_array_mask(p))) || assert((has_partial_array_mask(p) && _g1h->is_in_g1_reserved(clear_partial_array_mask(p))) ||
_g1h->obj_in_cs(oopDesc::load_decode_heap_oop(p)), "sanity"); _g1h->is_in_g1_reserved(oopDesc::load_decode_heap_oop(p)), "sanity");
deal_with_reference(p); deal_with_reference(p);
} }
} }
@ -1844,12 +1844,12 @@ public:
assert(UseCompressedOops, "Error"); assert(UseCompressedOops, "Error");
narrowOop* p = (narrowOop*)ref_to_scan; narrowOop* p = (narrowOop*)ref_to_scan;
assert(!has_partial_array_mask(p) && assert(!has_partial_array_mask(p) &&
_g1h->obj_in_cs(oopDesc::load_decode_heap_oop(p)), "sanity"); _g1h->is_in_g1_reserved(oopDesc::load_decode_heap_oop(p)), "sanity");
deal_with_reference(p); deal_with_reference(p);
} else { } else {
oop* p = (oop*)ref_to_scan; oop* p = (oop*)ref_to_scan;
assert((has_partial_array_mask(p) && _g1h->obj_in_cs(clear_partial_array_mask(p))) || assert((has_partial_array_mask(p) && _g1h->obj_in_cs(clear_partial_array_mask(p))) ||
_g1h->obj_in_cs(oopDesc::load_decode_heap_oop(p)), "sanity"); _g1h->is_in_g1_reserved(oopDesc::load_decode_heap_oop(p)), "sanity");
deal_with_reference(p); deal_with_reference(p);
} }
} }

1
hotspot/src/share/vm/gc_implementation/g1/g1CollectorPolicy.cpp
View File

@ -205,6 +205,7 @@ G1CollectorPolicy::G1CollectorPolicy() :
// policy is created before the heap, we have to set this up here, // policy is created before the heap, we have to set this up here,
// so it's done as soon as possible. // so it's done as soon as possible.
HeapRegion::setup_heap_region_size(Arguments::min_heap_size()); HeapRegion::setup_heap_region_size(Arguments::min_heap_size());
HeapRegionRemSet::setup_remset_size();
_recent_prev_end_times_for_all_gcs_sec->add(os::elapsedTime()); _recent_prev_end_times_for_all_gcs_sec->add(os::elapsedTime());
_prev_collection_pause_end_ms = os::elapsedTime() * 1000.0; _prev_collection_pause_end_ms = os::elapsedTime() * 1000.0;

26
hotspot/src/share/vm/gc_implementation/g1/g1OopClosures.hpp
View File

@ -53,6 +53,15 @@ public:
bool apply_to_weak_ref_discovered_field() { return true; } bool apply_to_weak_ref_discovered_field() { return true; }
}; };
class G1ParPushHeapRSClosure : public G1ParClosureSuper {
public:
G1ParPushHeapRSClosure(G1CollectedHeap* g1, G1ParScanThreadState* par_scan_state) :
G1ParClosureSuper(g1, par_scan_state) { }
template <class T> void do_oop_nv(T* p);
virtual void do_oop(oop* p) { do_oop_nv(p); }
virtual void do_oop(narrowOop* p) { do_oop_nv(p); }
};
class G1ParScanClosure : public G1ParClosureSuper { class G1ParScanClosure : public G1ParClosureSuper {
public: public:
G1ParScanClosure(G1CollectedHeap* g1, G1ParScanThreadState* par_scan_state) : G1ParScanClosure(G1CollectedHeap* g1, G1ParScanThreadState* par_scan_state) :
@ -100,7 +109,7 @@ public:
}; };
template<bool do_gen_barrier, G1Barrier barrier, template<bool do_gen_barrier, G1Barrier barrier,
bool do_mark_forwardee, bool skip_cset_test> bool do_mark_forwardee>
class G1ParCopyClosure : public G1ParCopyHelper { class G1ParCopyClosure : public G1ParCopyHelper {
G1ParScanClosure _scanner; G1ParScanClosure _scanner;
template <class T> void do_oop_work(T* p); template <class T> void do_oop_work(T* p);
@ -116,12 +125,13 @@ public:
virtual void do_oop(narrowOop* p) { do_oop_nv(p); } virtual void do_oop(narrowOop* p) { do_oop_nv(p); }
}; };
typedef G1ParCopyClosure<false, G1BarrierNone, false, false> G1ParScanExtRootClosure; typedef G1ParCopyClosure<false, G1BarrierNone, false> G1ParScanExtRootClosure;
typedef G1ParCopyClosure<true, G1BarrierNone, false, false> G1ParScanPermClosure; typedef G1ParCopyClosure<true, G1BarrierNone, false> G1ParScanPermClosure;
typedef G1ParCopyClosure<false, G1BarrierRS, false, false> G1ParScanHeapRSClosure; typedef G1ParCopyClosure<false, G1BarrierRS, false> G1ParScanHeapRSClosure;
typedef G1ParCopyClosure<false, G1BarrierNone, true, false> G1ParScanAndMarkExtRootClosure; typedef G1ParCopyClosure<false, G1BarrierNone, true> G1ParScanAndMarkExtRootClosure;
typedef G1ParCopyClosure<true, G1BarrierNone, true, false> G1ParScanAndMarkPermClosure; typedef G1ParCopyClosure<true, G1BarrierNone, true> G1ParScanAndMarkPermClosure;
typedef G1ParCopyClosure<false, G1BarrierRS, true, false> G1ParScanAndMarkHeapRSClosure; typedef G1ParCopyClosure<false, G1BarrierRS, true> G1ParScanAndMarkHeapRSClosure;
// This is the only case when we set skip_cset_test. Basically, this // This is the only case when we set skip_cset_test. Basically, this
// closure is (should?) only be called directly while we're draining // closure is (should?) only be called directly while we're draining
// the overflow and task queues. In that case we know that the // the overflow and task queues. In that case we know that the
@ -132,7 +142,7 @@ typedef G1ParCopyClosure<false, G1BarrierRS, true, false> G1ParScanAndMarkHea
// We need a separate closure to handle references during evacuation // We need a separate closure to handle references during evacuation
// failure processing, as we cannot asume that the reference already // failure processing, as we cannot asume that the reference already
// points into the collection set (like G1ParScanHeapEvacClosure does). // points into the collection set (like G1ParScanHeapEvacClosure does).
typedef G1ParCopyClosure<false, G1BarrierEvac, false, false> G1ParScanHeapEvacFailureClosure; typedef G1ParCopyClosure<false, G1BarrierEvac, false> G1ParScanHeapEvacFailureClosure;
class FilterIntoCSClosure: public OopClosure { class FilterIntoCSClosure: public OopClosure {
G1CollectedHeap* _g1; G1CollectedHeap* _g1;

13
hotspot/src/share/vm/gc_implementation/g1/g1OopClosures.inline.hpp
View File

@ -104,3 +104,16 @@ template <class T> inline void G1ParScanClosure::do_oop_nv(T* p) {
} }
} }
} }
template <class T> inline void G1ParPushHeapRSClosure::do_oop_nv(T* p) {
T heap_oop = oopDesc::load_heap_oop(p);
if (!oopDesc::is_null(heap_oop)) {
oop obj = oopDesc::decode_heap_oop_not_null(heap_oop);
if (_g1->in_cset_fast_test(obj)) {
Prefetch::write(obj->mark_addr(), 0);
Prefetch::read(obj->mark_addr(), (HeapWordSize*2));
_par_scan_state->push_on_queue(p);
}
}
}

61
hotspot/src/share/vm/gc_implementation/g1/g1RemSet.cpp
View File

@ -155,8 +155,8 @@ class ScanRSClosure : public HeapRegionClosure {
G1BlockOffsetSharedArray* _bot_shared; G1BlockOffsetSharedArray* _bot_shared;
CardTableModRefBS *_ct_bs; CardTableModRefBS *_ct_bs;
int _worker_i; int _worker_i;
int _block_size;
bool _try_claimed; bool _try_claimed;
size_t _min_skip_distance, _max_skip_distance;
public: public:
ScanRSClosure(OopsInHeapRegionClosure* oc, int worker_i) : ScanRSClosure(OopsInHeapRegionClosure* oc, int worker_i) :
_oc(oc), _oc(oc),
@ -168,8 +168,7 @@ public:
_g1h = G1CollectedHeap::heap(); _g1h = G1CollectedHeap::heap();
_bot_shared = _g1h->bot_shared(); _bot_shared = _g1h->bot_shared();
_ct_bs = (CardTableModRefBS*) (_g1h->barrier_set()); _ct_bs = (CardTableModRefBS*) (_g1h->barrier_set());
_min_skip_distance = 16; _block_size = MAX2<int>(G1RSetScanBlockSize, 1);
_max_skip_distance = 2 * _g1h->n_par_threads() * _min_skip_distance;
} }
void set_try_claimed() { _try_claimed = true; } void set_try_claimed() { _try_claimed = true; }
@ -225,12 +224,15 @@ public:
HeapRegionRemSetIterator* iter = _g1h->rem_set_iterator(_worker_i); HeapRegionRemSetIterator* iter = _g1h->rem_set_iterator(_worker_i);
hrrs->init_iterator(iter); hrrs->init_iterator(iter);
size_t card_index; size_t card_index;
size_t skip_distance = 0, current_card = 0, jump_to_card = 0;
while (iter->has_next(card_index)) { // We claim cards in block so as to recude the contention. The block size is determined by
if (current_card < jump_to_card) { // the G1RSetScanBlockSize parameter.
++current_card; size_t jump_to_card = hrrs->iter_claimed_next(_block_size);
continue; for (size_t current_card = 0; iter->has_next(card_index); current_card++) {
if (current_card >= jump_to_card + _block_size) {
jump_to_card = hrrs->iter_claimed_next(_block_size);
} }
if (current_card < jump_to_card) continue;
HeapWord* card_start = _g1h->bot_shared()->address_for_index(card_index); HeapWord* card_start = _g1h->bot_shared()->address_for_index(card_index);
#if 0 #if 0
gclog_or_tty->print("Rem set iteration yielded card [" PTR_FORMAT ", " PTR_FORMAT ").\n", gclog_or_tty->print("Rem set iteration yielded card [" PTR_FORMAT ", " PTR_FORMAT ").\n",
@ -247,22 +249,14 @@ public:
// If the card is dirty, then we will scan it during updateRS. // If the card is dirty, then we will scan it during updateRS.
if (!card_region->in_collection_set() && !_ct_bs->is_card_dirty(card_index)) { if (!card_region->in_collection_set() && !_ct_bs->is_card_dirty(card_index)) {
if (!_ct_bs->is_card_claimed(card_index) && _ct_bs->claim_card(card_index)) { // We make the card as "claimed" lazily (so races are possible but they're benign),
scanCard(card_index, card_region); // which reduces the number of duplicate scans (the rsets of the regions in the cset
} else if (_try_claimed) { // can intersect).
if (jump_to_card == 0 || jump_to_card != current_card) { if (!_ct_bs->is_card_claimed(card_index)) {
// We did some useful work in the previous iteration. _ct_bs->set_card_claimed(card_index);
// Decrease the distance. scanCard(card_index, card_region);
skip_distance = MAX2(skip_distance >> 1, _min_skip_distance); }
} else {
// Previous iteration resulted in a claim failure.
// Increase the distance.
skip_distance = MIN2(skip_distance << 1, _max_skip_distance);
}
jump_to_card = current_card + skip_distance;
}
} }
++current_card;
} }
if (!_try_claimed) { if (!_try_claimed) {
hrrs->set_iter_complete(); hrrs->set_iter_complete();
@ -299,30 +293,18 @@ void HRInto_G1RemSet::scanRS(OopsInHeapRegionClosure* oc, int worker_i) {
double rs_time_start = os::elapsedTime(); double rs_time_start = os::elapsedTime();
HeapRegion *startRegion = calculateStartRegion(worker_i); HeapRegion *startRegion = calculateStartRegion(worker_i);
BufferingOopsInHeapRegionClosure boc(oc); ScanRSClosure scanRScl(oc, worker_i);
ScanRSClosure scanRScl(&boc, worker_i);
_g1->collection_set_iterate_from(startRegion, &scanRScl); _g1->collection_set_iterate_from(startRegion, &scanRScl);
scanRScl.set_try_claimed(); scanRScl.set_try_claimed();
_g1->collection_set_iterate_from(startRegion, &scanRScl); _g1->collection_set_iterate_from(startRegion, &scanRScl);
boc.done(); double scan_rs_time_sec = os::elapsedTime() - rs_time_start;
double closure_app_time_sec = boc.closure_app_seconds();
double scan_rs_time_sec = (os::elapsedTime() - rs_time_start) -
closure_app_time_sec;
double closure_app_time_ms = closure_app_time_sec * 1000.0;
assert( _cards_scanned != NULL, "invariant" ); assert( _cards_scanned != NULL, "invariant" );
_cards_scanned[worker_i] = scanRScl.cards_done(); _cards_scanned[worker_i] = scanRScl.cards_done();
_g1p->record_scan_rs_start_time(worker_i, rs_time_start * 1000.0); _g1p->record_scan_rs_start_time(worker_i, rs_time_start * 1000.0);
_g1p->record_scan_rs_time(worker_i, scan_rs_time_sec * 1000.0); _g1p->record_scan_rs_time(worker_i, scan_rs_time_sec * 1000.0);
double scan_new_refs_time_ms = _g1p->get_scan_new_refs_time(worker_i);
if (scan_new_refs_time_ms > 0.0) {
closure_app_time_ms += scan_new_refs_time_ms;
}
_g1p->record_obj_copy_time(worker_i, closure_app_time_ms);
} }
void HRInto_G1RemSet::updateRS(int worker_i) { void HRInto_G1RemSet::updateRS(int worker_i) {
@ -449,9 +431,8 @@ HRInto_G1RemSet::scanNewRefsRS_work(OopsInHeapRegionClosure* oc,
oc->do_oop(p); oc->do_oop(p);
} }
} }
_g1p->record_scan_new_refs_time(worker_i, double scan_new_refs_time_ms = (os::elapsedTime() - scan_new_refs_start_sec) * 1000.0;
(os::elapsedTime() - scan_new_refs_start_sec) _g1p->record_scan_new_refs_time(worker_i, scan_new_refs_time_ms);
* 1000.0);
} }
void HRInto_G1RemSet::cleanupHRRS() { void HRInto_G1RemSet::cleanupHRRS() {

20
hotspot/src/share/vm/gc_implementation/g1/g1_globals.hpp
View File

@ -207,8 +207,20 @@
develop(bool, G1PrintOopAppls, false, \ develop(bool, G1PrintOopAppls, false, \
"When true, print applications of closures to external locs.") \ "When true, print applications of closures to external locs.") \
\ \
develop(intx, G1LogRSRegionEntries, 7, \ develop(intx, G1RSetRegionEntriesBase, 256, \
"Log_2 of max number of regions for which we keep bitmaps.") \ "Max number of regions in a fine-grain table per MB.") \
\
product(intx, G1RSetRegionEntries, 0, \
"Max number of regions for which we keep bitmaps." \
"Will be set ergonomically by default") \
\
develop(intx, G1RSetSparseRegionEntriesBase, 4, \
"Max number of entries per region in a sparse table " \
"per MB.") \
\
product(intx, G1RSetSparseRegionEntries, 0, \
"Max number of entries per region in a sparse table." \
"Will be set ergonomically by default.") \
\ \
develop(bool, G1RecordHRRSOops, false, \ develop(bool, G1RecordHRRSOops, false, \
"When true, record recent calls to rem set operations.") \ "When true, record recent calls to rem set operations.") \
@ -293,6 +305,10 @@
develop(bool, G1VerifyCTCleanup, false, \ develop(bool, G1VerifyCTCleanup, false, \
"Verify card table cleanup.") \ "Verify card table cleanup.") \
\ \
product(uintx, G1RSetScanBlockSize, 64, \
"Size of a work unit of cards claimed by a worker thread" \
"during RSet scanning.") \
\
develop(bool, ReduceInitialCardMarksForG1, false, \ develop(bool, ReduceInitialCardMarksForG1, false, \
"When ReduceInitialCardMarks is true, this flag setting " \ "When ReduceInitialCardMarks is true, this flag setting " \
" controls whether G1 allows the RICM optimization") " controls whether G1 allows the RICM optimization")

6
hotspot/src/share/vm/gc_implementation/g1/g1_specialized_oop_closures.hpp
View File

@ -33,11 +33,12 @@ enum G1Barrier {
}; };
template<bool do_gen_barrier, G1Barrier barrier, template<bool do_gen_barrier, G1Barrier barrier,
bool do_mark_forwardee, bool skip_cset_test> bool do_mark_forwardee>
class G1ParCopyClosure; class G1ParCopyClosure;
class G1ParScanClosure; class G1ParScanClosure;
class G1ParPushHeapRSClosure;
typedef G1ParCopyClosure<false, G1BarrierEvac, false, true> G1ParScanHeapEvacClosure; typedef G1ParCopyClosure<false, G1BarrierEvac, false> G1ParScanHeapEvacClosure;
class FilterIntoCSClosure; class FilterIntoCSClosure;
class FilterOutOfRegionClosure; class FilterOutOfRegionClosure;
@ -51,6 +52,7 @@ class FilterAndMarkInHeapRegionAndIntoCSClosure;
#define FURTHER_SPECIALIZED_OOP_OOP_ITERATE_CLOSURES(f) \ #define FURTHER_SPECIALIZED_OOP_OOP_ITERATE_CLOSURES(f) \
f(G1ParScanHeapEvacClosure,_nv) \ f(G1ParScanHeapEvacClosure,_nv) \
f(G1ParScanClosure,_nv) \ f(G1ParScanClosure,_nv) \
f(G1ParPushHeapRSClosure,_nv) \
f(FilterIntoCSClosure,_nv) \ f(FilterIntoCSClosure,_nv) \
f(FilterOutOfRegionClosure,_nv) \ f(FilterOutOfRegionClosure,_nv) \
f(FilterInHeapRegionAndIntoCSClosure,_nv) \ f(FilterInHeapRegionAndIntoCSClosure,_nv) \

35
hotspot/src/share/vm/gc_implementation/g1/heapRegionRemSet.cpp
View File

@ -505,12 +505,13 @@ OtherRegionsTable::OtherRegionsTable(HeapRegion* hr) :
typedef PosParPRT* PosParPRTPtr; typedef PosParPRT* PosParPRTPtr;
if (_max_fine_entries == 0) { if (_max_fine_entries == 0) {
assert(_mod_max_fine_entries_mask == 0, "Both or none."); assert(_mod_max_fine_entries_mask == 0, "Both or none.");
_max_fine_entries = (size_t)(1 << G1LogRSRegionEntries); size_t max_entries_log = (size_t)log2_long((jlong)G1RSetRegionEntries);
_max_fine_entries = (size_t)(1 << max_entries_log);
_mod_max_fine_entries_mask = _max_fine_entries - 1; _mod_max_fine_entries_mask = _max_fine_entries - 1;
#if SAMPLE_FOR_EVICTION #if SAMPLE_FOR_EVICTION
assert(_fine_eviction_sample_size == 0 assert(_fine_eviction_sample_size == 0
&& _fine_eviction_stride == 0, "All init at same time."); && _fine_eviction_stride == 0, "All init at same time.");
_fine_eviction_sample_size = MAX2((size_t)4, (size_t)G1LogRSRegionEntries); _fine_eviction_sample_size = MAX2((size_t)4, max_entries_log);
_fine_eviction_stride = _max_fine_entries / _fine_eviction_sample_size; _fine_eviction_stride = _max_fine_entries / _fine_eviction_sample_size;
#endif #endif
} }
@ -655,13 +656,6 @@ void OtherRegionsTable::add_reference(OopOrNarrowOopStar from, int tid) {
#endif #endif
} }
// Otherwise, transfer from sparse to fine-grain.
CardIdx_t cards[SparsePRTEntry::CardsPerEntry];
if (G1HRRSUseSparseTable) {
bool res = _sparse_table.get_cards(from_hrs_ind, &cards[0]);
assert(res, "There should have been an entry");
}
if (_n_fine_entries == _max_fine_entries) { if (_n_fine_entries == _max_fine_entries) {
prt = delete_region_table(); prt = delete_region_table();
} else { } else {
@ -676,10 +670,12 @@ void OtherRegionsTable::add_reference(OopOrNarrowOopStar from, int tid) {
_fine_grain_regions[ind] = prt; _fine_grain_regions[ind] = prt;
_n_fine_entries++; _n_fine_entries++;
// Add in the cards from the sparse table.
if (G1HRRSUseSparseTable) { if (G1HRRSUseSparseTable) {
for (int i = 0; i < SparsePRTEntry::CardsPerEntry; i++) { // Transfer from sparse to fine-grain.
CardIdx_t c = cards[i]; SparsePRTEntry *sprt_entry = _sparse_table.get_entry(from_hrs_ind);
assert(sprt_entry != NULL, "There should have been an entry");
for (int i = 0; i < SparsePRTEntry::cards_num(); i++) {
CardIdx_t c = sprt_entry->card(i);
if (c != SparsePRTEntry::NullEntry) { if (c != SparsePRTEntry::NullEntry) {
prt->add_card(c); prt->add_card(c);
} }
@ -1084,6 +1080,19 @@ HeapRegionRemSet::HeapRegionRemSet(G1BlockOffsetSharedArray* bosa,
{} {}
void HeapRegionRemSet::setup_remset_size() {
// Setup sparse and fine-grain tables sizes.
// table_size = base * (log(region_size / 1M) + 1)
int region_size_log_mb = MAX2((int)HeapRegion::LogOfHRGrainBytes - (int)LOG_M, 0);
if (FLAG_IS_DEFAULT(G1RSetSparseRegionEntries)) {
G1RSetSparseRegionEntries = G1RSetSparseRegionEntriesBase * (region_size_log_mb + 1);
}
if (FLAG_IS_DEFAULT(G1RSetRegionEntries)) {
G1RSetRegionEntries = G1RSetRegionEntriesBase * (region_size_log_mb + 1);
}
guarantee(G1RSetSparseRegionEntries > 0 && G1RSetRegionEntries > 0 , "Sanity");
}
void HeapRegionRemSet::init_for_par_iteration() { void HeapRegionRemSet::init_for_par_iteration() {
_iter_state = Unclaimed; _iter_state = Unclaimed;
} }
@ -1399,7 +1408,7 @@ void HeapRegionRemSet::test() {
os::sleep(Thread::current(), (jlong)5000, false); os::sleep(Thread::current(), (jlong)5000, false);
G1CollectedHeap* g1h = G1CollectedHeap::heap(); G1CollectedHeap* g1h = G1CollectedHeap::heap();
// Run with "-XX:G1LogRSRegionEntries=2", so that 1 and 5 end up in same // Run with "-XX:G1LogRSetRegionEntries=2", so that 1 and 5 end up in same
// hash bucket. // hash bucket.
HeapRegion* hr0 = g1h->region_at(0); HeapRegion* hr0 = g1h->region_at(0);
HeapRegion* hr1 = g1h->region_at(1); HeapRegion* hr1 = g1h->region_at(1);

17
hotspot/src/share/vm/gc_implementation/g1/heapRegionRemSet.hpp
View File

@ -187,7 +187,8 @@ private:
void clear_outgoing_entries(); void clear_outgoing_entries();
enum ParIterState { Unclaimed, Claimed, Complete }; enum ParIterState { Unclaimed, Claimed, Complete };
ParIterState _iter_state; volatile ParIterState _iter_state;
volatile jlong _iter_claimed;
// Unused unless G1RecordHRRSOops is true. // Unused unless G1RecordHRRSOops is true.
@ -209,6 +210,7 @@ public:
HeapRegion* hr); HeapRegion* hr);
static int num_par_rem_sets(); static int num_par_rem_sets();
static void setup_remset_size();
HeapRegion* hr() const { HeapRegion* hr() const {
return _other_regions.hr(); return _other_regions.hr();
@ -272,6 +274,19 @@ public:
// Returns "true" iff the region's iteration is complete. // Returns "true" iff the region's iteration is complete.
bool iter_is_complete(); bool iter_is_complete();
// Support for claiming blocks of cards during iteration
void set_iter_claimed(size_t x) { _iter_claimed = (jlong)x; }
size_t iter_claimed() const { return (size_t)_iter_claimed; }
// Claim the next block of cards
size_t iter_claimed_next(size_t step) {
size_t current, next;
do {
current = iter_claimed();
next = current + step;
} while (Atomic::cmpxchg((jlong)next, &_iter_claimed, (jlong)current) != (jlong)current);
return current;
}
// Initialize the given iterator to iterate over this rem set. // Initialize the given iterator to iterate over this rem set.
void init_iterator(HeapRegionRemSetIterator* iter) const; void init_iterator(HeapRegionRemSetIterator* iter) const;

143
hotspot/src/share/vm/gc_implementation/g1/sparsePRT.cpp
View File

@ -27,7 +27,7 @@
#define SPARSE_PRT_VERBOSE 0 #define SPARSE_PRT_VERBOSE 0
#define UNROLL_CARD_LOOPS 1 #define UNROLL_CARD_LOOPS 1
void SparsePRT::init_iterator(SparsePRTIter* sprt_iter) { void SparsePRT::init_iterator(SparsePRTIter* sprt_iter) {
sprt_iter->init(this); sprt_iter->init(this);
@ -36,27 +36,32 @@ void SparsePRT::init_iterator(SparsePRTIter* sprt_iter) {
void SparsePRTEntry::init(RegionIdx_t region_ind) { void SparsePRTEntry::init(RegionIdx_t region_ind) {
_region_ind = region_ind; _region_ind = region_ind;
_next_index = NullEntry; _next_index = NullEntry;
#if UNROLL_CARD_LOOPS #if UNROLL_CARD_LOOPS
assert(CardsPerEntry == 4, "Assumption. If changes, un-unroll."); assert((cards_num() & (UnrollFactor - 1)) == 0, "Invalid number of cards in the entry");
_cards[0] = NullEntry; for (int i = 0; i < cards_num(); i += UnrollFactor) {
_cards[1] = NullEntry; _cards[i] = NullEntry;
_cards[2] = NullEntry; _cards[i + 1] = NullEntry;
_cards[3] = NullEntry; _cards[i + 2] = NullEntry;
_cards[i + 3] = NullEntry;
}
#else #else
for (int i = 0; i < CardsPerEntry; i++) for (int i = 0; i < cards_num(); i++)
_cards[i] = NullEntry; _cards[i] = NullEntry;
#endif #endif
} }
bool SparsePRTEntry::contains_card(CardIdx_t card_index) const { bool SparsePRTEntry::contains_card(CardIdx_t card_index) const {
#if UNROLL_CARD_LOOPS #if UNROLL_CARD_LOOPS
assert(CardsPerEntry == 4, "Assumption. If changes, un-unroll."); assert((cards_num() & (UnrollFactor - 1)) == 0, "Invalid number of cards in the entry");
if (_cards[0] == card_index) return true; for (int i = 0; i < cards_num(); i += UnrollFactor) {
if (_cards[1] == card_index) return true; if (_cards[i] == card_index ||
if (_cards[2] == card_index) return true; _cards[i + 1] == card_index ||
if (_cards[3] == card_index) return true; _cards[i + 2] == card_index ||
_cards[i + 3] == card_index) return true;
}
#else #else
for (int i = 0; i < CardsPerEntry; i++) { for (int i = 0; i < cards_num(); i++) {
if (_cards[i] == card_index) return true; if (_cards[i] == card_index) return true;
} }
#endif #endif
@ -67,14 +72,16 @@ bool SparsePRTEntry::contains_card(CardIdx_t card_index) const {
int SparsePRTEntry::num_valid_cards() const { int SparsePRTEntry::num_valid_cards() const {
int sum = 0; int sum = 0;
#if UNROLL_CARD_LOOPS #if UNROLL_CARD_LOOPS
assert(CardsPerEntry == 4, "Assumption. If changes, un-unroll."); assert((cards_num() & (UnrollFactor - 1)) == 0, "Invalid number of cards in the entry");
if (_cards[0] != NullEntry) sum++; for (int i = 0; i < cards_num(); i += UnrollFactor) {
if (_cards[1] != NullEntry) sum++; sum += (_cards[i] != NullEntry);
if (_cards[2] != NullEntry) sum++; sum += (_cards[i + 1] != NullEntry);
if (_cards[3] != NullEntry) sum++; sum += (_cards[i + 2] != NullEntry);
sum += (_cards[i + 3] != NullEntry);
}
#else #else
for (int i = 0; i < CardsPerEntry; i++) { for (int i = 0; i < cards_num(); i++) {
if (_cards[i] != NulLEntry) sum++; sum += (_cards[i] != NullEntry);
} }
#endif #endif
// Otherwise, we're full. // Otherwise, we're full.
@ -83,27 +90,27 @@ int SparsePRTEntry::num_valid_cards() const {
SparsePRTEntry::AddCardResult SparsePRTEntry::add_card(CardIdx_t card_index) { SparsePRTEntry::AddCardResult SparsePRTEntry::add_card(CardIdx_t card_index) {
#if UNROLL_CARD_LOOPS #if UNROLL_CARD_LOOPS
assert(CardsPerEntry == 4, "Assumption. If changes, un-unroll."); assert((cards_num() & (UnrollFactor - 1)) == 0, "Invalid number of cards in the entry");
CardIdx_t c = _cards[0]; CardIdx_t c;
if (c == card_index) return found; for (int i = 0; i < cards_num(); i += UnrollFactor) {
if (c == NullEntry) { _cards[0] = card_index; return added; } c = _cards[i];
c = _cards[1]; if (c == card_index) return found;
if (c == card_index) return found; if (c == NullEntry) { _cards[i] = card_index; return added; }
if (c == NullEntry) { _cards[1] = card_index; return added; } c = _cards[i + 1];
c = _cards[2]; if (c == card_index) return found;
if (c == card_index) return found; if (c == NullEntry) { _cards[i + 1] = card_index; return added; }
if (c == NullEntry) { _cards[2] = card_index; return added; } c = _cards[i + 2];
c = _cards[3]; if (c == card_index) return found;
if (c == card_index) return found; if (c == NullEntry) { _cards[i + 2] = card_index; return added; }
if (c == NullEntry) { _cards[3] = card_index; return added; } c = _cards[i + 3];
if (c == card_index) return found;
if (c == NullEntry) { _cards[i + 3] = card_index; return added; }
}
#else #else
for (int i = 0; i < CardsPerEntry; i++) { for (int i = 0; i < cards_num(); i++) {
CardIdx_t c = _cards[i]; CardIdx_t c = _cards[i];
if (c == card_index) return found; if (c == card_index) return found;
if (c == NullEntry) { if (c == NullEntry) { _cards[i] = card_index; return added; }
_cards[i] = card_index;
return added;
}
} }
#endif #endif
// Otherwise, we're full. // Otherwise, we're full.
@ -112,13 +119,15 @@ SparsePRTEntry::AddCardResult SparsePRTEntry::add_card(CardIdx_t card_index) {
void SparsePRTEntry::copy_cards(CardIdx_t* cards) const { void SparsePRTEntry::copy_cards(CardIdx_t* cards) const {
#if UNROLL_CARD_LOOPS #if UNROLL_CARD_LOOPS
assert(CardsPerEntry == 4, "Assumption. If changes, un-unroll."); assert((cards_num() & (UnrollFactor - 1)) == 0, "Invalid number of cards in the entry");
cards[0] = _cards[0]; for (int i = 0; i < cards_num(); i += UnrollFactor) {
cards[1] = _cards[1]; cards[i] = _cards[i];
cards[2] = _cards[2]; cards[i + 1] = _cards[i + 1];
cards[3] = _cards[3]; cards[i + 2] = _cards[i + 2];
cards[i + 3] = _cards[i + 3];
}
#else #else
for (int i = 0; i < CardsPerEntry; i++) { for (int i = 0; i < cards_num(); i++) {
cards[i] = _cards[i]; cards[i] = _cards[i];
} }
#endif #endif
@ -133,7 +142,7 @@ void SparsePRTEntry::copy_cards(SparsePRTEntry* e) const {
RSHashTable::RSHashTable(size_t capacity) : RSHashTable::RSHashTable(size_t capacity) :
_capacity(capacity), _capacity_mask(capacity-1), _capacity(capacity), _capacity_mask(capacity-1),
_occupied_entries(0), _occupied_cards(0), _occupied_entries(0), _occupied_cards(0),
_entries(NEW_C_HEAP_ARRAY(SparsePRTEntry, capacity)), _entries((SparsePRTEntry*)NEW_C_HEAP_ARRAY(char, SparsePRTEntry::size() * capacity)),
_buckets(NEW_C_HEAP_ARRAY(int, capacity)), _buckets(NEW_C_HEAP_ARRAY(int, capacity)),
_free_list(NullEntry), _free_region(0) _free_list(NullEntry), _free_region(0)
{ {
@ -161,8 +170,8 @@ void RSHashTable::clear() {
"_capacity too large"); "_capacity too large");
// This will put -1 == NullEntry in the key field of all entries. // This will put -1 == NullEntry in the key field of all entries.
memset(_entries, -1, _capacity * sizeof(SparsePRTEntry)); memset(_entries, NullEntry, _capacity * SparsePRTEntry::size());
memset(_buckets, -1, _capacity * sizeof(int)); memset(_buckets, NullEntry, _capacity * sizeof(int));
_free_list = NullEntry; _free_list = NullEntry;
_free_region = 0; _free_region = 0;
} }
@ -175,8 +184,8 @@ bool RSHashTable::add_card(RegionIdx_t region_ind, CardIdx_t card_index) {
if (res == SparsePRTEntry::added) _occupied_cards++; if (res == SparsePRTEntry::added) _occupied_cards++;
#if SPARSE_PRT_VERBOSE #if SPARSE_PRT_VERBOSE
gclog_or_tty->print_cr(" after add_card[%d]: valid-cards = %d.", gclog_or_tty->print_cr(" after add_card[%d]: valid-cards = %d.",
pointer_delta(e, _entries, sizeof(SparsePRTEntry)), pointer_delta(e, _entries, SparsePRTEntry::size()),
e->num_valid_cards()); e->num_valid_cards());
#endif #endif
assert(e->num_valid_cards() > 0, "Postcondition"); assert(e->num_valid_cards() > 0, "Postcondition");
return res != SparsePRTEntry::overflow; return res != SparsePRTEntry::overflow;
@ -199,6 +208,22 @@ bool RSHashTable::get_cards(RegionIdx_t region_ind, CardIdx_t* cards) {
return true; return true;
} }
SparsePRTEntry* RSHashTable::get_entry(RegionIdx_t region_ind) {
int ind = (int) (region_ind & capacity_mask());
int cur_ind = _buckets[ind];
SparsePRTEntry* cur;
while (cur_ind != NullEntry &&
(cur = entry(cur_ind))->r_ind() != region_ind) {
cur_ind = cur->next_index();
}
if (cur_ind == NullEntry) return NULL;
// Otherwise...
assert(cur->r_ind() == region_ind, "Postcondition of loop + test above.");
assert(cur->num_valid_cards() > 0, "Inv");
return cur;
}
bool RSHashTable::delete_entry(RegionIdx_t region_ind) { bool RSHashTable::delete_entry(RegionIdx_t region_ind) {
int ind = (int) (region_ind & capacity_mask()); int ind = (int) (region_ind & capacity_mask());
int* prev_loc = &_buckets[ind]; int* prev_loc = &_buckets[ind];
@ -225,20 +250,8 @@ RSHashTable::entry_for_region_ind(RegionIdx_t region_ind) const {
int ind = (int) (region_ind & capacity_mask()); int ind = (int) (region_ind & capacity_mask());
int cur_ind = _buckets[ind]; int cur_ind = _buckets[ind];
SparsePRTEntry* cur; SparsePRTEntry* cur;
// XXX
// int k = 0;
while (cur_ind != NullEntry && while (cur_ind != NullEntry &&
(cur = entry(cur_ind))->r_ind() != region_ind) { (cur = entry(cur_ind))->r_ind() != region_ind) {
/*
k++;
if (k > 10) {
gclog_or_tty->print_cr("RSHashTable::entry_for_region_ind(%d): "
"k = %d, cur_ind = %d.", region_ind, k, cur_ind);
if (k >= 1000) {
while (1) ;
}
}
*/
cur_ind = cur->next_index(); cur_ind = cur->next_index();
} }
@ -319,7 +332,7 @@ size_t /* RSHashTable:: */ RSHashTableIter::compute_card_ind(CardIdx_t ci) {
bool /* RSHashTable:: */ RSHashTableIter::has_next(size_t& card_index) { bool /* RSHashTable:: */ RSHashTableIter::has_next(size_t& card_index) {
_card_ind++; _card_ind++;
CardIdx_t ci; CardIdx_t ci;
if (_card_ind < SparsePRTEntry::CardsPerEntry && if (_card_ind < SparsePRTEntry::cards_num() &&
((ci = _rsht->entry(_bl_ind)->card(_card_ind)) != ((ci = _rsht->entry(_bl_ind)->card(_card_ind)) !=
SparsePRTEntry::NullEntry)) { SparsePRTEntry::NullEntry)) {
card_index = compute_card_ind(ci); card_index = compute_card_ind(ci);
@ -359,7 +372,7 @@ bool RSHashTable::contains_card(RegionIdx_t region_index, CardIdx_t card_index)
size_t RSHashTable::mem_size() const { size_t RSHashTable::mem_size() const {
return sizeof(this) + return sizeof(this) +
capacity() * (sizeof(SparsePRTEntry) + sizeof(int)); capacity() * (SparsePRTEntry::size() + sizeof(int));
} }
// ---------------------------------------------------------------------- // ----------------------------------------------------------------------
@ -446,6 +459,10 @@ bool SparsePRT::get_cards(RegionIdx_t region_id, CardIdx_t* cards) {
return _next->get_cards(region_id, cards); return _next->get_cards(region_id, cards);
} }
SparsePRTEntry* SparsePRT::get_entry(RegionIdx_t region_id) {
return _next->get_entry(region_id);
}
bool SparsePRT::delete_entry(RegionIdx_t region_id) { bool SparsePRT::delete_entry(RegionIdx_t region_id) {
return _next->delete_entry(region_id); return _next->delete_entry(region_id);
} }

38
hotspot/src/share/vm/gc_implementation/g1/sparsePRT.hpp
View File

@ -32,21 +32,28 @@
// insertions only enqueue old versions for deletions, but do not delete // insertions only enqueue old versions for deletions, but do not delete
// old versions synchronously. // old versions synchronously.
class SparsePRTEntry: public CHeapObj { class SparsePRTEntry: public CHeapObj {
public: public:
enum SomePublicConstants { enum SomePublicConstants {
CardsPerEntry = 4, NullEntry = -1,
NullEntry = -1 UnrollFactor = 4
}; };
private: private:
RegionIdx_t _region_ind; RegionIdx_t _region_ind;
int _next_index; int _next_index;
CardIdx_t _cards[CardsPerEntry]; CardIdx_t _cards[1];
// WARNING: Don't put any data members beyond this line. Card array has, in fact, variable length.
// It should always be the last data member.
public: public:
// Returns the size of the entry, used for entry allocation.
static size_t size() { return sizeof(SparsePRTEntry) + sizeof(CardIdx_t) * (cards_num() - 1); }
// Returns the size of the card array.
static int cards_num() {
// The number of cards should be a multiple of 4, because that's our current
// unrolling factor.
static const int s = MAX2<int>(G1RSetSparseRegionEntries & ~(UnrollFactor - 1), UnrollFactor);
return s;
}
// Set the region_ind to the given value, and delete all cards. // Set the region_ind to the given value, and delete all cards.
inline void init(RegionIdx_t region_ind); inline void init(RegionIdx_t region_ind);
@ -134,12 +141,15 @@ public:
bool add_card(RegionIdx_t region_id, CardIdx_t card_index); bool add_card(RegionIdx_t region_id, CardIdx_t card_index);
bool get_cards(RegionIdx_t region_id, CardIdx_t* cards); bool get_cards(RegionIdx_t region_id, CardIdx_t* cards);
bool delete_entry(RegionIdx_t region_id); bool delete_entry(RegionIdx_t region_id);
bool contains_card(RegionIdx_t region_id, CardIdx_t card_index) const; bool contains_card(RegionIdx_t region_id, CardIdx_t card_index) const;
void add_entry(SparsePRTEntry* e); void add_entry(SparsePRTEntry* e);
SparsePRTEntry* get_entry(RegionIdx_t region_id);
void clear(); void clear();
size_t capacity() const { return _capacity; } size_t capacity() const { return _capacity; }
@ -148,7 +158,7 @@ public:
size_t occupied_cards() const { return _occupied_cards; } size_t occupied_cards() const { return _occupied_cards; }
size_t mem_size() const; size_t mem_size() const;
SparsePRTEntry* entry(int i) const { return &_entries[i]; } SparsePRTEntry* entry(int i) const { return (SparsePRTEntry*)((char*)_entries + SparsePRTEntry::size() * i); }
void print(); void print();
}; };
@ -157,7 +167,7 @@ public:
class RSHashTableIter VALUE_OBJ_CLASS_SPEC { class RSHashTableIter VALUE_OBJ_CLASS_SPEC {
int _tbl_ind; // [-1, 0.._rsht->_capacity) int _tbl_ind; // [-1, 0.._rsht->_capacity)
int _bl_ind; // [-1, 0.._rsht->_capacity) int _bl_ind; // [-1, 0.._rsht->_capacity)
short _card_ind; // [0..CardsPerEntry) short _card_ind; // [0..SparsePRTEntry::cards_num())
RSHashTable* _rsht; RSHashTable* _rsht;
size_t _heap_bot_card_ind; size_t _heap_bot_card_ind;
@ -176,7 +186,7 @@ public:
RSHashTableIter(size_t heap_bot_card_ind) : RSHashTableIter(size_t heap_bot_card_ind) :
_tbl_ind(RSHashTable::NullEntry), _tbl_ind(RSHashTable::NullEntry),
_bl_ind(RSHashTable::NullEntry), _bl_ind(RSHashTable::NullEntry),
_card_ind((SparsePRTEntry::CardsPerEntry-1)), _card_ind((SparsePRTEntry::cards_num() - 1)),
_rsht(NULL), _rsht(NULL),
_heap_bot_card_ind(heap_bot_card_ind) _heap_bot_card_ind(heap_bot_card_ind)
{} {}
@ -185,7 +195,7 @@ public:
_rsht = rsht; _rsht = rsht;
_tbl_ind = -1; // So that first increment gets to 0. _tbl_ind = -1; // So that first increment gets to 0.
_bl_ind = RSHashTable::NullEntry; _bl_ind = RSHashTable::NullEntry;
_card_ind = (SparsePRTEntry::CardsPerEntry-1); _card_ind = (SparsePRTEntry::cards_num() - 1);
} }
bool has_next(size_t& card_index); bool has_next(size_t& card_index);
@ -241,9 +251,13 @@ public:
// If the table hold an entry for "region_ind", Copies its // If the table hold an entry for "region_ind", Copies its
// cards into "cards", which must be an array of length at least // cards into "cards", which must be an array of length at least
// "CardsPerEntry", and returns "true"; otherwise, returns "false". // "SparePRTEntry::cards_num()", and returns "true"; otherwise,
// returns "false".
bool get_cards(RegionIdx_t region_ind, CardIdx_t* cards); bool get_cards(RegionIdx_t region_ind, CardIdx_t* cards);
// Return the pointer to the entry associated with the given region.
SparsePRTEntry* get_entry(RegionIdx_t region_ind);
// If there is an entry for "region_ind", removes it and return "true"; // If there is an entry for "region_ind", removes it and return "true";
// otherwise returns "false." // otherwise returns "false."
bool delete_entry(RegionIdx_t region_ind); bool delete_entry(RegionIdx_t region_ind);

10
hotspot/src/share/vm/memory/cardTableModRefBS.hpp
View File

@ -339,6 +339,16 @@ public:
return (val & (clean_card_mask_val() | claimed_card_val())) == claimed_card_val(); return (val & (clean_card_mask_val() | claimed_card_val())) == claimed_card_val();
} }
void set_card_claimed(size_t card_index) {
jbyte val = _byte_map[card_index];
if (val == clean_card_val()) {
val = (jbyte)claimed_card_val();
} else {
val |= (jbyte)claimed_card_val();
}
_byte_map[card_index] = val;
}
bool claim_card(size_t card_index); bool claim_card(size_t card_index);
bool is_card_clean(size_t card_index) { bool is_card_clean(size_t card_index) {

4
hotspot/src/share/vm/utilities/globalDefinitions.hpp
View File

@ -139,6 +139,10 @@ const size_t M = K*K;
const size_t G = M*K; const size_t G = M*K;
const size_t HWperKB = K / sizeof(HeapWord); const size_t HWperKB = K / sizeof(HeapWord);
const size_t LOG_K = 10;
const size_t LOG_M = 2 * LOG_K;
const size_t LOG_G = 2 * LOG_M;
const jint min_jint = (jint)1 << (sizeof(jint)*BitsPerByte-1); // 0x80000000 == smallest jint const jint min_jint = (jint)1 << (sizeof(jint)*BitsPerByte-1); // 0x80000000 == smallest jint
const jint max_jint = (juint)min_jint - 1; // 0x7FFFFFFF == largest jint const jint max_jint = (juint)min_jint - 1; // 0x7FFFFFFF == largest jint