8254739: G1: Optimize evacuation failure for regions with few failed objects

Reviewed-by: tschatzl, ayang
This commit is contained in:
Hamlin Li 2021-11-05 23:24:45 +00:00
parent 59c3dcc761
commit ed7ecca401
12 changed files with 345 additions and 39 deletions

@ -25,6 +25,7 @@
#include "precompiled.hpp"
#include "gc/g1/g1CardSetMemory.inline.hpp"
#include "gc/g1/g1SegmentedArray.inline.hpp"
#include "logging/log.hpp"
#include "runtime/atomic.hpp"
#include "utilities/formatBuffer.hpp"
@ -46,6 +47,11 @@ G1CardSetAllocator<Elem>::G1CardSetAllocator(const char* name,
assert(elem_size >= sizeof(G1CardSetContainer), "Element instance size %u for allocator %s too small", elem_size, name);
}
template <class Elem>
G1CardSetAllocator<Elem>::~G1CardSetAllocator() {
drop_all();
}
template <class Elem>
bool G1CardSetAllocator<Elem>::try_transfer_pending() {
// Attempt to claim the lock.

@ -113,9 +113,7 @@ public:
G1CardSetAllocator(const char* name,
const G1CardSetAllocOptions* buffer_options,
G1CardSetBufferList* free_buffer_list);
~G1CardSetAllocator() {
drop_all();
}
~G1CardSetAllocator();
Elem* allocate();
void free(Elem* elem);

@ -68,40 +68,40 @@ public:
// dead too) already.
void do_object(oop obj) {
HeapWord* obj_addr = cast_from_oop<HeapWord*>(obj);
assert(_last_forwarded_object_end <= obj_addr, "should iterate in ascending address order");
assert(_hr->is_in(obj_addr), "sanity");
if (obj->is_forwarded() && obj->forwardee() == obj) {
// The object failed to move.
// The object failed to move.
assert(obj->is_forwarded() && obj->forwardee() == obj, "sanity");
zap_dead_objects(_last_forwarded_object_end, obj_addr);
// We consider all objects that we find self-forwarded to be
// live. What we'll do is that we'll update the prev marking
// info so that they are all under PTAMS and explicitly marked.
if (!_cm->is_marked_in_prev_bitmap(obj)) {
_cm->mark_in_prev_bitmap(obj);
}
if (_during_concurrent_start) {
// For the next marking info we'll only mark the
// self-forwarded objects explicitly if we are during
// concurrent start (since, normally, we only mark objects pointed
// to by roots if we succeed in copying them). By marking all
// self-forwarded objects we ensure that we mark any that are
// still pointed to be roots. During concurrent marking, and
// after concurrent start, we don't need to mark any objects
// explicitly and all objects in the CSet are considered
// (implicitly) live. So, we won't mark them explicitly and
// we'll leave them over NTAMS.
_cm->mark_in_next_bitmap(_worker_id, _hr, obj);
}
size_t obj_size = obj->size();
_marked_bytes += (obj_size * HeapWordSize);
PreservedMarks::init_forwarded_mark(obj);
HeapWord* obj_end = obj_addr + obj_size;
_last_forwarded_object_end = obj_end;
_hr->alloc_block_in_bot(obj_addr, obj_end);
zap_dead_objects(_last_forwarded_object_end, obj_addr);
// We consider all objects that we find self-forwarded to be
// live. What we'll do is that we'll update the prev marking
// info so that they are all under PTAMS and explicitly marked.
if (!_cm->is_marked_in_prev_bitmap(obj)) {
_cm->mark_in_prev_bitmap(obj);
}
if (_during_concurrent_start) {
// For the next marking info we'll only mark the
// self-forwarded objects explicitly if we are during
// concurrent start (since, normally, we only mark objects pointed
// to by roots if we succeed in copying them). By marking all
// self-forwarded objects we ensure that we mark any that are
// still pointed to be roots. During concurrent marking, and
// after concurrent start, we don't need to mark any objects
// explicitly and all objects in the CSet are considered
// (implicitly) live. So, we won't mark them explicitly and
// we'll leave them over NTAMS.
_cm->mark_in_next_bitmap(_worker_id, _hr, obj);
}
size_t obj_size = obj->size();
_marked_bytes += (obj_size * HeapWordSize);
PreservedMarks::init_forwarded_mark(obj);
HeapWord* obj_end = obj_addr + obj_size;
_last_forwarded_object_end = obj_end;
_hr->alloc_block_in_bot(obj_addr, obj_end);
}
// Fill the memory area from start to end with filler objects, and update the BOT
@ -164,7 +164,8 @@ public:
RemoveSelfForwardPtrObjClosure rspc(hr,
during_concurrent_start,
_worker_id);
hr->object_iterate(&rspc);
// Iterates evac failure objs which are recorded during evacuation.
hr->iterate_evac_failure_objs(&rspc);
// Need to zap the remainder area of the processed region.
rspc.zap_remainder();

@ -0,0 +1,131 @@
/*
* Copyright (c) 2021, Huawei and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*
*/
#include "precompiled.hpp"
#include "gc/g1/g1EvacFailureObjectsSet.hpp"
#include "gc/g1/g1CollectedHeap.hpp"
#include "gc/g1/g1SegmentedArray.inline.hpp"
#include "gc/g1/heapRegion.hpp"
#include "gc/g1/heapRegion.inline.hpp"
#include "utilities/quickSort.hpp"
const G1SegmentedArrayAllocOptions G1EvacFailureObjectsSet::_alloc_options =
G1SegmentedArrayAllocOptions((uint)sizeof(OffsetInRegion), BufferLength, UINT_MAX, Alignment);
G1SegmentedArrayBufferList<mtGC> G1EvacFailureObjectsSet::_free_buffer_list;
#ifdef ASSERT
void G1EvacFailureObjectsSet::assert_is_valid_offset(size_t offset) const {
const uint max_offset = 1u << (HeapRegion::LogOfHRGrainBytes - LogHeapWordSize);
assert(offset < max_offset, "must be, but is " SIZE_FORMAT, offset);
}
#endif
oop G1EvacFailureObjectsSet::from_offset(OffsetInRegion offset) const {
assert_is_valid_offset(offset);
return cast_to_oop(_bottom + offset);
}
G1EvacFailureObjectsSet::OffsetInRegion G1EvacFailureObjectsSet::to_offset(oop obj) const {
const HeapWord* o = cast_from_oop<const HeapWord*>(obj);
size_t offset = pointer_delta(o, _bottom);
assert(obj == from_offset(static_cast<OffsetInRegion>(offset)), "must be");
return static_cast<OffsetInRegion>(offset);
}
G1EvacFailureObjectsSet::G1EvacFailureObjectsSet(uint region_idx, HeapWord* bottom) :
DEBUG_ONLY(_region_idx(region_idx) COMMA)
_bottom(bottom),
_offsets(&_alloc_options, &_free_buffer_list) {
assert(HeapRegion::LogOfHRGrainBytes < 32, "must be");
}
void G1EvacFailureObjectsSet::record(oop obj) {
assert(obj != NULL, "must be");
assert(_region_idx == G1CollectedHeap::heap()->heap_region_containing(obj)->hrm_index(), "must be");
OffsetInRegion* e = _offsets.allocate();
*e = to_offset(obj);
}
// Helper class to join, sort and iterate over the previously collected segmented
// array of objects that failed evacuation.
class G1EvacFailureObjectsIterationHelper {
typedef G1EvacFailureObjectsSet::OffsetInRegion OffsetInRegion;
G1EvacFailureObjectsSet* _objects_set;
const G1SegmentedArray<OffsetInRegion, mtGC>* _segments;
OffsetInRegion* _offset_array;
uint _array_length;
static int order_oop(OffsetInRegion a, OffsetInRegion b) {
return static_cast<int>(a-b);
}
void join_and_sort() {
_segments->iterate_nodes(*this);
QuickSort::sort(_offset_array, _array_length, order_oop, true);
}
void iterate_internal(ObjectClosure* closure) {
for (uint i = 0; i < _array_length; i++) {
oop cur = _objects_set->from_offset(_offset_array[i]);
closure->do_object(cur);
}
}
public:
G1EvacFailureObjectsIterationHelper(G1EvacFailureObjectsSet* collector) :
_objects_set(collector),
_segments(&_objects_set->_offsets),
_offset_array(nullptr),
_array_length(0) { }
void iterate(ObjectClosure* closure) {
uint num = _segments->num_allocated_nodes();
_offset_array = NEW_C_HEAP_ARRAY(OffsetInRegion, num, mtGC);
join_and_sort();
assert(_array_length == num, "must be %u, %u", _array_length, num);
iterate_internal(closure);
FREE_C_HEAP_ARRAY(OffsetInRegion, _offset_array);
}
// Callback of G1SegmentedArray::iterate_nodes
void do_buffer(G1SegmentedArrayBuffer<mtGC>* node, uint length) {
node->copy_to(&_offset_array[_array_length]);
_array_length += length;
}
};
void G1EvacFailureObjectsSet::iterate(ObjectClosure* closure) {
assert_at_safepoint();
G1EvacFailureObjectsIterationHelper helper(this);
helper.iterate(closure);
_offsets.drop_all();
}

@ -0,0 +1,81 @@
/*
* Copyright (c) 2021, Huawei and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*
*/
#ifndef SHARE_GC_G1_G1EVACUATIONFAILUREOBJSINHR_HPP
#define SHARE_GC_G1_G1EVACUATIONFAILUREOBJSINHR_HPP
#include "gc/g1/g1SegmentedArray.hpp"
#include "memory/iterator.hpp"
#include "oops/oop.hpp"
class G1EvacFailureObjectsIterationHelper;
// This class collects addresses of objects that failed evacuation in a specific
// heap region.
// Provides sorted iteration of these elements for processing during the remove
// self forwards phase.
class G1EvacFailureObjectsSet {
friend class G1EvacFailureObjectsIterationHelper;
public:
// Storage type of an object that failed evacuation within a region. Given
// heap region size and possible object locations within a region, it is
// sufficient to use an uint here to save some space instead of full pointers.
typedef uint OffsetInRegion;
private:
static const uint BufferLength = 256;
static const uint Alignment = 4;
static const G1SegmentedArrayAllocOptions _alloc_options;
// This free list is shared among evacuation failure process in all regions.
static G1SegmentedArrayBufferList<mtGC> _free_buffer_list;
DEBUG_ONLY(const uint _region_idx;)
// Region bottom
const HeapWord* _bottom;
// Offsets within region containing objects that failed evacuation.
G1SegmentedArray<OffsetInRegion, mtGC> _offsets;
void assert_is_valid_offset(size_t offset) const NOT_DEBUG_RETURN;
// Converts between an offset within a region and an oop address.
oop from_offset(OffsetInRegion offset) const;
OffsetInRegion to_offset(oop obj) const;
public:
G1EvacFailureObjectsSet(uint region_idx, HeapWord* bottom);
// Record an object that failed evacuation.
void record(oop obj);
// Apply the given ObjectClosure to all objects that failed evacuation. Objects
// are passed in increasing address order.
void iterate(ObjectClosure* closure);
};
#endif //SHARE_GC_G1_G1EVACUATIONFAILUREOBJSINHR_HPP

@ -607,6 +607,9 @@ oop G1ParScanThreadState::handle_evacuation_failure_par(oop old, markWord m, siz
if (forward_ptr == NULL) {
// Forward-to-self succeeded. We are the "owner" of the object.
HeapRegion* r = _g1h->heap_region_containing(old);
// Records evac failure objs, this will help speed up iteration
// of these objs later in *remove self forward* phase of post evacuation.
r->record_evac_failure_obj(old);
if (_evac_failure_regions->record(r->hrm_index())) {
_g1h->hr_printer()->evac_failure(r);

@ -73,6 +73,17 @@ public:
size_t mem_size() const { return sizeof(*this) + (size_t)_num_elems * _elem_size; }
uint length() const {
// _next_allocate might grow larger than _num_elems in multi-thread environments
// due to races.
return MIN2(_next_allocate, _num_elems);
}
// Copies the (valid) contents of this buffer into the destination.
void copy_to(void* dest) const {
::memcpy(dest, _buffer, length() * _elem_size);
}
bool is_full() const { return _next_allocate >= _num_elems; }
};
@ -189,19 +200,23 @@ class G1SegmentedArray {
private:
inline G1SegmentedArrayBuffer<flag>* create_new_buffer(G1SegmentedArrayBuffer<flag>* const prev);
DEBUG_ONLY(uint calculate_length() const;)
public:
const G1SegmentedArrayBuffer<flag>* first_array_buffer() const { return Atomic::load(&_first); }
uint num_available_nodes() const { return Atomic::load(&_num_available_nodes); }
uint num_allocated_nodes() const { return Atomic::load(&_num_allocated_nodes); }
uint num_allocated_nodes() const {
uint allocated = Atomic::load(&_num_allocated_nodes);
assert(calculate_length() == allocated, "Must be");
return allocated;
}
inline uint elem_size() const;
G1SegmentedArray(const G1SegmentedArrayAllocOptions* buffer_options,
G1SegmentedArrayBufferList<flag>* free_buffer_list);
~G1SegmentedArray() {
drop_all();
}
~G1SegmentedArray();
// Deallocate all buffers to the free buffer list and reset this allocator. Must
// be called in a globally synchronized area.
@ -210,6 +225,9 @@ public:
inline Elem* allocate();
inline uint num_buffers() const;
template<typename BufferClosure>
void iterate_nodes(BufferClosure& closure) const;
};
#endif //SHARE_GC_G1_G1SEGMENTEDARRAY_HPP

@ -167,6 +167,11 @@ G1SegmentedArray<Elem, flag>::G1SegmentedArray(const G1SegmentedArrayAllocOption
assert(_free_buffer_list != nullptr, "precondition!");
}
template <class Elem, MEMFLAGS flag>
G1SegmentedArray<Elem, flag>::~G1SegmentedArray() {
drop_all();
}
template <class Elem, MEMFLAGS flag>
void G1SegmentedArray<Elem, flag>::drop_all() {
G1SegmentedArrayBuffer<flag>* cur = Atomic::load_acquire(&_first);
@ -232,4 +237,40 @@ inline uint G1SegmentedArray<Elem, flag>::num_buffers() const {
return Atomic::load(&_num_buffers);
}
#ifdef ASSERT
template <MEMFLAGS flag>
class LengthClosure {
uint _total;
public:
LengthClosure() : _total(0) {}
void do_buffer(G1SegmentedArrayBuffer<flag>* node, uint limit) {
_total += limit;
}
uint length() const {
return _total;
}
};
template <class Elem, MEMFLAGS flag>
uint G1SegmentedArray<Elem, flag>::calculate_length() const {
LengthClosure<flag> closure;
iterate_nodes(closure);
return closure.length();
}
#endif
template <class Elem, MEMFLAGS flag>
template <typename BufferClosure>
void G1SegmentedArray<Elem, flag>::iterate_nodes(BufferClosure& closure) const {
G1SegmentedArrayBuffer<flag>* cur = Atomic::load_acquire(&_first);
assert((cur != nullptr) == (_last != nullptr),
"If there is at least one element, there must be a last one");
while (cur != nullptr) {
closure.do_buffer(cur, cur->length());
cur = cur->next();
}
}
#endif //SHARE_GC_G1_G1SEGMENTEDARRAY_INLINE_HPP

@ -106,6 +106,10 @@ void HeapRegion::handle_evacuation_failure() {
_next_marked_bytes = 0;
}
void HeapRegion::iterate_evac_failure_objs(ObjectClosure* closure) {
_evac_failure_objs.iterate(closure);
}
void HeapRegion::unlink_from_list() {
set_next(NULL);
set_prev(NULL);
@ -246,7 +250,8 @@ HeapRegion::HeapRegion(uint hrm_index,
_prev_marked_bytes(0), _next_marked_bytes(0),
_young_index_in_cset(-1),
_surv_rate_group(NULL), _age_index(G1SurvRateGroup::InvalidAgeIndex), _gc_efficiency(-1.0),
_node_index(G1NUMA::UnknownNodeIndex)
_node_index(G1NUMA::UnknownNodeIndex),
_evac_failure_objs(hrm_index, _bottom)
{
assert(Universe::on_page_boundary(mr.start()) && Universe::on_page_boundary(mr.end()),
"invalid space boundaries");

@ -26,6 +26,7 @@
#define SHARE_GC_G1_HEAPREGION_HPP
#include "gc/g1/g1BlockOffsetTable.hpp"
#include "gc/g1/g1EvacFailureObjectsSet.hpp"
#include "gc/g1/g1HeapRegionTraceType.hpp"
#include "gc/g1/g1SurvRateGroup.hpp"
#include "gc/g1/heapRegionTracer.hpp"
@ -258,6 +259,8 @@ private:
uint _node_index;
G1EvacFailureObjectsSet _evac_failure_objs;
void report_region_type_change(G1HeapRegionTraceType::Type to);
// Returns whether the given object address refers to a dead object, and either the
@ -554,6 +557,11 @@ public:
// Update the region state after a failed evacuation.
void handle_evacuation_failure();
// Record an object that failed evacuation within this region.
void record_evac_failure_obj(oop obj);
// Applies the given closure to all previously recorded objects
// that failed evacuation in ascending address order.
void iterate_evac_failure_objs(ObjectClosure* closure);
// Iterate over the objects overlapping the given memory region, applying cl
// to all references in the region. This is a helper for

@ -31,6 +31,7 @@
#include "gc/g1/g1CollectedHeap.inline.hpp"
#include "gc/g1/g1ConcurrentMarkBitMap.inline.hpp"
#include "gc/g1/g1Predictions.hpp"
#include "gc/g1/g1SegmentedArray.inline.hpp"
#include "oops/oop.inline.hpp"
#include "runtime/atomic.hpp"
#include "runtime/prefetch.inline.hpp"
@ -450,4 +451,8 @@ inline void HeapRegion::record_surv_words_in_group(size_t words_survived) {
_surv_rate_group->record_surviving_words(age_in_group, words_survived);
}
inline void HeapRegion::record_evac_failure_obj(oop obj) {
_evac_failure_objs.record(obj);
}
#endif // SHARE_GC_G1_HEAPREGION_INLINE_HPP

@ -22,7 +22,16 @@
*/
#include "precompiled.hpp"
#include "gc/g1/g1BlockOffsetTable.inline.hpp"
#include "gc/g1/g1CardSet.inline.hpp"
#include "gc/g1/g1CollectedHeap.inline.hpp"
#include "gc/g1/g1RegionToSpaceMapper.hpp"
#include "gc/g1/heapRegion.inline.hpp"
#include "gc/g1/heapRegionSet.hpp"
#include "memory/allocation.hpp"
#include "memory/memRegion.hpp"
#include "memory/virtualspace.hpp"
#include "unittest.hpp"
// @requires UseG1GC