stan/jdk-24
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

8276098: Do precise BOT updates in G1 evacuation phase

Browse Source 
Reviewed-by: tschatzl, ayang
This commit is contained in:
Stefan Johansson 2021-11-09 11:11:23 +00:00
parent 8747882e4c
commit 945f4085e5
15 changed files with 190 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
src/hotspot/share/gc/g1/g1AllocRegion.cpp
View File

@ -43,10 +43,7 @@ void G1AllocRegion::setup(G1CollectedHeap* g1h, HeapRegion* dummy_region) {
// Make sure that any allocation attempt on this region will fail
// and will not trigger any asserts.
assert(dummy_region->allocate_no_bot_updates(1) == NULL, "should fail");
assert(dummy_region->allocate(1) == NULL, "should fail");
DEBUG_ONLY(size_t assert_tmp);
assert(dummy_region->par_allocate_no_bot_updates(1, 1, &assert_tmp) == NULL, "should fail");
assert(dummy_region->par_allocate(1, 1, &assert_tmp) == NULL, "should fail");
_g1h = g1h;
@ -77,8 +74,9 @@ size_t G1AllocRegion::fill_up_remaining_space(HeapRegion* alloc_region) {
while (free_word_size >= min_word_size_to_fill) {
HeapWord* dummy = par_allocate(alloc_region, free_word_size);
if (dummy != NULL) {
// If the allocation was successful we should fill in the space.
CollectedHeap::fill_with_object(dummy, free_word_size);
// If the allocation was successful we should fill in the space. If the
// allocation was in old any necessary BOT updates will be done.
alloc_region->fill_with_dummy_object(dummy, free_word_size);
alloc_region->set_pre_dummy_top(dummy);
result += free_word_size * HeapWordSize;
break;
@ -256,7 +254,6 @@ G1AllocRegion::G1AllocRegion(const char* name,
: _alloc_region(NULL),
_count(0),
_used_bytes_before(0),
_bot_updates(bot_updates),
_name(name),
_node_index(node_index)
{ }
@ -389,7 +386,7 @@ HeapRegion* OldGCAllocRegion::release() {
// original problem cannot occur.
if (to_allocate_words >= G1CollectedHeap::min_fill_size()) {
HeapWord* dummy = attempt_allocation(to_allocate_words);
CollectedHeap::fill_with_object(dummy, to_allocate_words);
cur->fill_with_dummy_object(dummy, to_allocate_words);
}
}
}

3
src/hotspot/share/gc/g1/g1AllocRegion.hpp
View File

@ -68,9 +68,6 @@ private:
// we allocated in it.
size_t _used_bytes_before;
// When true, indicates that allocate calls should do BOT updates.
const bool _bot_updates;
// Useful for debugging and tracing.
const char* _name;

18
src/hotspot/share/gc/g1/g1AllocRegion.inline.hpp
View File

@ -31,9 +31,9 @@
#define assert_alloc_region(p, message) \
do { \
assert((p), "[%s] %s c: %u b: %s r: " PTR_FORMAT " u: " SIZE_FORMAT, \
_name, (message), _count, BOOL_TO_STR(_bot_updates), \
p2i(_alloc_region), _used_bytes_before); \
assert((p), "[%s] %s c: %u r: " PTR_FORMAT " u: " SIZE_FORMAT, \
_name, (message), _count, p2i(_alloc_region), \
_used_bytes_before); \
} while (0)
@ -45,11 +45,7 @@ inline HeapWord* G1AllocRegion::allocate(HeapRegion* alloc_region,
size_t word_size) {
assert(alloc_region != NULL, "pre-condition");
if (!_bot_updates) {
return alloc_region->allocate_no_bot_updates(word_size);
} else {
return alloc_region->allocate(word_size);
}
return alloc_region->allocate(word_size);
}
inline HeapWord* G1AllocRegion::par_allocate(HeapRegion* alloc_region, size_t word_size) {
@ -64,11 +60,7 @@ inline HeapWord* G1AllocRegion::par_allocate(HeapRegion* alloc_region,
assert(alloc_region != NULL, "pre-condition");
assert(!alloc_region->is_empty(), "pre-condition");
if (!_bot_updates) {
return alloc_region->par_allocate_no_bot_updates(min_word_size, desired_word_size, actual_word_size);
} else {
return alloc_region->par_allocate(min_word_size, desired_word_size, actual_word_size);
}
return alloc_region->par_allocate(min_word_size, desired_word_size, actual_word_size);
}
inline HeapWord* G1AllocRegion::attempt_allocation(size_t word_size) {

7
src/hotspot/share/gc/g1/g1Allocator.cpp
View File

@ -299,8 +299,12 @@ G1PLABAllocator::G1PLABAllocator(G1Allocator* allocator) :
_direct_allocated[state] = 0;
uint length = alloc_buffers_length(state);
_alloc_buffers[state] = NEW_C_HEAP_ARRAY(PLAB*, length, mtGC);
size_t word_sz = _g1h->desired_plab_sz(state);
for (uint node_index = 0; node_index < length; node_index++) {
_alloc_buffers[state][node_index] = new PLAB(_g1h->desired_plab_sz(state));
// Specialized PLABs for old that handle BOT updates for object allocations.
_alloc_buffers[state][node_index] = (state == G1HeapRegionAttr::Old)
? new G1BotUpdatingPLAB(word_sz)
: new PLAB(word_sz);
}
}
}
@ -360,6 +364,7 @@ HeapWord* G1PLABAllocator::allocate_direct_or_new_plab(G1HeapRegionAttr dest,
// Try direct allocation.
HeapWord* result = _allocator->par_allocate_during_gc(dest, word_sz, node_index);
if (result != NULL) {
update_bot_for_direct_allocation(dest, result, word_sz);
_direct_allocated[dest.type()] += word_sz;
}
return result;

26
src/hotspot/share/gc/g1/g1Allocator.hpp
View File

@ -145,6 +145,22 @@ public:
uint node_index);
};
// Specialized PLAB for old generation promotions. For old regions the
// BOT needs to be updated and the relevant data to do this efficiently
// is stored in the PLAB.
class G1BotUpdatingPLAB : public PLAB {
// An object spanning this threshold will cause a BOT update.
HeapWord* _next_bot_threshold;
// The region in which the PLAB resides.
HeapRegion* _region;
public:
G1BotUpdatingPLAB(size_t word_sz) : PLAB(word_sz) { }
// Sets the new PLAB buffer as well as updates the threshold and region.
void set_buf(HeapWord* buf, size_t word_sz) override;
// Updates the BOT if the last allocation crossed the threshold.
inline void update_bot(size_t word_sz);
};
// Manages the PLABs used during garbage collection. Interface for allocation from PLABs.
// Needs to handle multiple contexts, extra alignment in any "survivor" area and some
// statistics.
@ -165,11 +181,18 @@ private:
inline PLAB* alloc_buffer(G1HeapRegionAttr dest, uint node_index) const;
inline PLAB* alloc_buffer(region_type_t dest, uint node_index) const;
// Helpers to do explicit BOT updates for allocations in old generation regions.
void update_bot_for_direct_allocation(G1HeapRegionAttr attr, HeapWord* addr, size_t size);
// Returns the number of allocation buffers for the given dest.
// There is only 1 buffer for Old while Young may have multiple buffers depending on
// active NUMA nodes.
inline uint alloc_buffers_length(region_type_t dest) const;
// Returns if BOT updates are needed for the given destinaion. Currently we only have
// two destinations and BOT updates are only needed for the old generation.
inline bool needs_bot_update(G1HeapRegionAttr dest) const;
bool may_throw_away_buffer(size_t const allocation_word_sz, size_t const buffer_size) const;
public:
G1PLABAllocator(G1Allocator* allocator);
@ -198,6 +221,9 @@ public:
bool* refill_failed,
uint node_index);
// Update the BOT for the last PLAB allocation.
inline void update_bot_for_plab_allocation(G1HeapRegionAttr dest, size_t word_sz, uint node_index);
void undo_allocation(G1HeapRegionAttr dest, HeapWord* obj, size_t word_sz, uint node_index);
};

45
src/hotspot/share/gc/g1/g1Allocator.inline.hpp
View File

@ -133,4 +133,49 @@ inline HeapWord* G1PLABAllocator::allocate(G1HeapRegionAttr dest,
return allocate_direct_or_new_plab(dest, word_sz, refill_failed, node_index);
}
inline bool G1PLABAllocator::needs_bot_update(G1HeapRegionAttr dest) const {
return dest.is_old();
}
inline void G1PLABAllocator::update_bot_for_direct_allocation(G1HeapRegionAttr attr, HeapWord* addr, size_t size) {
if (!needs_bot_update(attr)) {
return;
}
// Out of PLAB allocations in an old generation region. Update BOT.
HeapRegion* region = _g1h->heap_region_containing(addr);
region->update_bot_at(addr, size);
}
inline void G1PLABAllocator::update_bot_for_plab_allocation(G1HeapRegionAttr dest, size_t word_sz, uint node_index) {
assert(needs_bot_update(dest), "Wrong destination: %s", dest.get_type_str());
G1BotUpdatingPLAB* plab = static_cast<G1BotUpdatingPLAB*>(alloc_buffer(dest, node_index));
plab->update_bot(word_sz);
}
inline void G1BotUpdatingPLAB::set_buf(HeapWord* buf, size_t word_sz) {
PLAB::set_buf(buf, word_sz);
// Update the region and threshold to allow efficient BOT updates.
_region = G1CollectedHeap::heap()->heap_region_containing(buf);
_next_bot_threshold = _region->bot_threshold_for_addr(buf);
}
inline void G1BotUpdatingPLAB::update_bot(size_t word_sz) {
// The last object end is at _top, if it did not cross the
// threshold, there is nothing to do.
if (_top <= _next_bot_threshold) {
return;
}
HeapWord* obj_start = _top - word_sz;
assert(contains(obj_start),
"Object start outside PLAB. bottom: " PTR_FORMAT " object: " PTR_FORMAT,
p2i(_bottom), p2i(obj_start));
assert(obj_start <= _next_bot_threshold,
"Object start not below or at threshold. threshold: " PTR_FORMAT " object: " PTR_FORMAT,
p2i(_next_bot_threshold), p2i(obj_start));
_region->update_bot_crossing_threshold(&_next_bot_threshold, obj_start, _top);
}
#endif // SHARE_GC_G1_G1ALLOCATOR_INLINE_HPP

9
src/hotspot/share/gc/g1/g1BlockOffsetTable.hpp
View File

@ -108,6 +108,7 @@ public:
class G1BlockOffsetTablePart {
friend class G1BlockOffsetTable;
friend class HeapRegion;
friend class VMStructs;
private:
// allocation boundary at which offset array must be updated
@ -181,6 +182,9 @@ public:
void verify() const;
// Given an address calculate where the next threshold needing an update is.
inline HeapWord* threshold_for_addr(const void* addr);
// Returns the address of the start of the block containing "addr", or
// else "null" if it is covered by no block. (May have side effects,
// namely updating of shared array entries that "point" too far
@ -204,6 +208,11 @@ public:
// updated.
HeapWord* threshold() const { return _next_offset_threshold; }
// Sets the threshold explicitly to keep it consistent with what has been
// updated. This needs to be done when the threshold is not used for updating
// the bot, for example when promoting to old in young collections.
void set_threshold(HeapWord* threshold) { _next_offset_threshold = threshold; }
// These must be guaranteed to work properly (i.e., do nothing)
// when "blk_start" ("blk" for second version) is "NULL". In this
// implementation, that's true because NULL is represented as 0, and thus

17
src/hotspot/share/gc/g1/g1BlockOffsetTable.inline.hpp
View File

@ -31,6 +31,20 @@
#include "gc/shared/memset_with_concurrent_readers.hpp"
#include "runtime/atomic.hpp"
inline HeapWord* G1BlockOffsetTablePart::threshold_for_addr(const void* addr) {
assert(addr >= _hr->bottom() && addr < _hr->top(), "invalid address");
size_t index = _bot->index_for(addr);
HeapWord* card_boundary = _bot->address_for_index(index);
// Address at card boundary, use as threshold.
if (card_boundary == addr) {
return card_boundary;
}
// Calculate next threshold.
HeapWord* threshold = card_boundary + BOTConstants::N_words;
return threshold;
}
inline HeapWord* G1BlockOffsetTablePart::block_start(const void* addr) {
assert(addr >= _hr->bottom() && addr < _hr->top(), "invalid address");
HeapWord* q = block_at_or_preceding(addr);
@ -109,9 +123,6 @@ inline HeapWord* G1BlockOffsetTablePart::block_at_or_preceding(const void* addr)
"Object crossed region boundary, found offset %u instead of 0",
(uint) _bot->offset_array(_bot->index_for(_hr->bottom())));
// We must make sure that the offset table entry we use is valid.
assert(addr < _next_offset_threshold, "Precondition");
size_t index = _bot->index_for(addr);
HeapWord* q = _bot->address_for_index(index);

6
src/hotspot/share/gc/g1/g1CollectedHeap.cpp
View File

@ -3293,6 +3293,7 @@ void G1CollectedHeap::retire_gc_alloc_region(HeapRegion* alloc_region,
_bytes_used_during_gc += allocated_bytes;
if (dest.is_old()) {
old_set_add(alloc_region);
alloc_region->update_bot_threshold();
} else {
assert(dest.is_young(), "Retiring alloc region should be young (%d)", dest.type());
_survivor.add_used_bytes(allocated_bytes);
@ -3445,3 +3446,8 @@ GrowableArray<GCMemoryManager*> G1CollectedHeap::memory_managers() {
GrowableArray<MemoryPool*> G1CollectedHeap::memory_pools() {
return _monitoring_support->memory_pools();
}
void G1CollectedHeap::fill_with_dummy_object(HeapWord* start, HeapWord* end, bool zap) {
HeapRegion* region = heap_region_containing(start);
region->fill_with_dummy_object(start, pointer_delta(end, start), zap);
}

2
src/hotspot/share/gc/g1/g1CollectedHeap.hpp
View File

@ -934,6 +934,8 @@ public:
virtual GrowableArray<GCMemoryManager*> memory_managers();
virtual GrowableArray<MemoryPool*> memory_pools();
virtual void fill_with_dummy_object(HeapWord* start, HeapWord* end, bool zap);
// Try to minimize the remembered set.
void scrub_rem_set();

5
src/hotspot/share/gc/g1/g1ParScanThreadState.cpp
View File

@ -498,6 +498,11 @@ oop G1ParScanThreadState::do_copy_to_survivor_space(G1HeapRegionAttr const regio
obj->incr_age();
}
_age_table.add(age, word_sz);
} else {
// Currently we only have two destinations and we only need BOT updates for
// old. If the current allocation was done outside the PLAB this call will
// have no effect since the _top of the PLAB has not changed.
_plab_allocator->update_bot_for_plab_allocation(dest_attr, word_sz, node_index);
}
// Most objects are not arrays, so do one array check rather than

10
src/hotspot/share/gc/g1/heapRegion.cpp
View File

@ -235,7 +235,6 @@ HeapRegion::HeapRegion(uint hrm_index,
_top(NULL),
_compaction_top(NULL),
_bot_part(bot, this),
_par_alloc_lock(Mutex::service-2, "HeapRegionParAlloc_lock"),
_pre_dummy_top(NULL),
_rem_set(NULL),
_hrm_index(hrm_index),
@ -818,3 +817,12 @@ void HeapRegion::object_iterate(ObjectClosure* blk) {
p += block_size(p);
}
}
void HeapRegion::fill_with_dummy_object(HeapWord* address, size_t word_size, bool zap) {
// Keep the BOT in sync for old generation regions.
if (is_old()) {
update_bot_at(address, word_size);
}
// Fill in the object.
CollectedHeap::fill_with_object(address, word_size, zap);
}

32
src/hotspot/share/gc/g1/heapRegion.hpp
View File

@ -78,7 +78,7 @@ class HeapRegion : public CHeapObj<mtGC> {
HeapWord* _compaction_top;
G1BlockOffsetTablePart _bot_part;
Mutex _par_alloc_lock;
// When we need to retire an allocation region, while other threads
// are also concurrently trying to allocate into it, we typically
// allocate a dummy object at the end of the region to ensure that
@ -153,18 +153,24 @@ public:
void object_iterate(ObjectClosure* blk);
// Allocation (return NULL if full). Assumes the caller has established
// mutually exclusive access to the HeapRegion.
HeapWord* allocate(size_t min_word_size, size_t desired_word_size, size_t* actual_word_size);
// Allocation (return NULL if full). Enforces mutual exclusion internally.
HeapWord* par_allocate(size_t min_word_size, size_t desired_word_size, size_t* actual_word_size);
// At the given address create an object with the given size. If the region
// is old the BOT will be updated if the object spans a threshold.
void fill_with_dummy_object(HeapWord* address, size_t word_size, bool zap = true);
HeapWord* allocate(size_t word_size);
HeapWord* par_allocate(size_t word_size);
// All allocations are done without updating the BOT. The BOT
// needs to be kept in sync for old generation regions and
// this is done by explicit updates when crossing thresholds.
inline HeapWord* par_allocate(size_t min_word_size, size_t desired_word_size, size_t* word_size);
inline HeapWord* allocate(size_t word_size);
inline HeapWord* allocate(size_t min_word_size, size_t desired_word_size, size_t* actual_size);
inline HeapWord* par_allocate_no_bot_updates(size_t min_word_size, size_t desired_word_size, size_t* word_size);
inline HeapWord* allocate_no_bot_updates(size_t word_size);
inline HeapWord* allocate_no_bot_updates(size_t min_word_size, size_t desired_word_size, size_t* actual_size);
// Update the BOT for the given address if it crosses the next
// BOT threshold at or after obj_start.
inline void update_bot_at(HeapWord* obj_start, size_t obj_size);
// Update BOT at the given threshold for the given object. The
// given object must cross the threshold.
inline void update_bot_crossing_threshold(HeapWord** threshold, HeapWord* obj_start, HeapWord* obj_end);
inline HeapWord* bot_threshold_for_addr(const void* addr);
// Full GC support methods.
@ -201,6 +207,10 @@ public:
_bot_part.update();
}
void update_bot_threshold() {
_bot_part.set_threshold(top());
}
private:
// The remembered set for this region.
HeapRegionRemSet* _rem_set;

77
src/hotspot/share/gc/g1/heapRegion.inline.hpp
View File

@ -79,36 +79,6 @@ inline HeapWord* HeapRegion::par_allocate_impl(size_t min_word_size,
} while (true);
}
inline HeapWord* HeapRegion::allocate(size_t min_word_size,
size_t desired_word_size,
size_t* actual_size) {
HeapWord* res = allocate_impl(min_word_size, desired_word_size, actual_size);
if (res != NULL) {
_bot_part.alloc_block(res, *actual_size);
}
return res;
}
inline HeapWord* HeapRegion::allocate(size_t word_size) {
size_t temp;
return allocate(word_size, word_size, &temp);
}
inline HeapWord* HeapRegion::par_allocate(size_t word_size) {
size_t temp;
return par_allocate(word_size, word_size, &temp);
}
// Because of the requirement of keeping "_offsets" up to date with the
// allocations, we sequentialize these with a lock. Therefore, best if
// this is used for larger LAB allocations only.
inline HeapWord* HeapRegion::par_allocate(size_t min_word_size,
size_t desired_word_size,
size_t* actual_size) {
MutexLocker x(&_par_alloc_lock, Mutex::_no_safepoint_check_flag);
return allocate(min_word_size, desired_word_size, actual_size);
}
inline HeapWord* HeapRegion::block_start(const void* p) {
return _bot_part.block_start(p);
}
@ -252,25 +222,52 @@ inline void HeapRegion::apply_to_marked_objects(G1CMBitMap* bitmap, ApplyToMarke
assert(next_addr == limit, "Should stop the scan at the limit.");
}
inline HeapWord* HeapRegion::par_allocate_no_bot_updates(size_t min_word_size,
size_t desired_word_size,
size_t* actual_word_size) {
assert(is_young(), "we can only skip BOT updates on young regions");
inline HeapWord* HeapRegion::par_allocate(size_t min_word_size,
size_t desired_word_size,
size_t* actual_word_size) {
return par_allocate_impl(min_word_size, desired_word_size, actual_word_size);
}
inline HeapWord* HeapRegion::allocate_no_bot_updates(size_t word_size) {
inline HeapWord* HeapRegion::allocate(size_t word_size) {
size_t temp;
return allocate_no_bot_updates(word_size, word_size, &temp);
return allocate(word_size, word_size, &temp);
}
inline HeapWord* HeapRegion::allocate_no_bot_updates(size_t min_word_size,
size_t desired_word_size,
size_t* actual_word_size) {
assert(is_young(), "we can only skip BOT updates on young regions");
inline HeapWord* HeapRegion::allocate(size_t min_word_size,
size_t desired_word_size,
size_t* actual_word_size) {
return allocate_impl(min_word_size, desired_word_size, actual_word_size);
}
inline HeapWord* HeapRegion::bot_threshold_for_addr(const void* addr) {
HeapWord* threshold = _bot_part.threshold_for_addr(addr);
assert(threshold >= addr,
"threshold must be at or after given address. " PTR_FORMAT " >= " PTR_FORMAT,
p2i(threshold), p2i(addr));
assert(is_old(),
"Should only calculate BOT threshold for old regions. addr: " PTR_FORMAT " region:" HR_FORMAT,
p2i(addr), HR_FORMAT_PARAMS(this));
return threshold;
}
inline void HeapRegion::update_bot_crossing_threshold(HeapWord** threshold, HeapWord* obj_start, HeapWord* obj_end) {
assert(is_old(), "should only do BOT updates for old regions");
assert(is_in(obj_start), "obj_start must be in this region: " HR_FORMAT
" obj_start " PTR_FORMAT " obj_end " PTR_FORMAT " threshold " PTR_FORMAT,
HR_FORMAT_PARAMS(this),
p2i(obj_start), p2i(obj_end), p2i(*threshold));
_bot_part.alloc_block_work(threshold, obj_start, obj_end);
}
inline void HeapRegion::update_bot_at(HeapWord* obj_start, size_t obj_size) {
HeapWord* threshold = bot_threshold_for_addr(obj_start);
HeapWord* obj_end = obj_start + obj_size;
if (obj_end > threshold) {
update_bot_crossing_threshold(&threshold, obj_start, obj_end);
}
}
inline void HeapRegion::note_start_of_marking() {
_next_marked_bytes = 0;
_next_top_at_mark_start = top();

2
src/hotspot/share/gc/shared/plab.hpp
View File

@ -119,7 +119,7 @@ public:
}
// Sets the space of the buffer to be [buf, space+word_sz()).
void set_buf(HeapWord* buf, size_t new_word_sz) {
virtual void set_buf(HeapWord* buf, size_t new_word_sz) {
assert(new_word_sz > AlignmentReserve, "Too small");
_word_sz = new_word_sz;