stan/jdk-24
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

8320864: Serial: Extract out Full GC related fields from ContiguousSpace

Browse Source 
Reviewed-by: kbarrett, sjohanss
This commit is contained in:
Albert Mingkun Yang 2024-01-09 06:11:44 +00:00
parent 176606d0cb
commit 07fce8eff2
11 changed files with 391 additions and 616 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

508
src/hotspot/share/gc/serial/genMarkSweep.cpp
View File

@ -36,6 +36,7 @@
#include "gc/serial/defNewGeneration.hpp"
#include "gc/serial/generation.hpp"
#include "gc/serial/genMarkSweep.hpp"
#include "gc/serial/markSweep.inline.hpp"
#include "gc/serial/serialGcRefProcProxyTask.hpp"
#include "gc/serial/serialHeap.hpp"
#include "gc/shared/classUnloadingContext.hpp"
@ -48,7 +49,7 @@
#include "gc/shared/preservedMarks.inline.hpp"
#include "gc/shared/referencePolicy.hpp"
#include "gc/shared/referenceProcessorPhaseTimes.hpp"
#include "gc/shared/space.hpp"
#include "gc/shared/space.inline.hpp"
#include "gc/shared/strongRootsScope.hpp"
#include "gc/shared/weakProcessor.hpp"
#include "memory/universe.hpp"
@ -57,6 +58,7 @@
#include "prims/jvmtiExport.hpp"
#include "runtime/handles.inline.hpp"
#include "runtime/javaThread.hpp"
#include "runtime/prefetch.inline.hpp"
#include "runtime/synchronizer.hpp"
#include "runtime/vmThread.hpp"
#include "utilities/copy.hpp"
@ -66,98 +68,281 @@
#include "jvmci/jvmci.hpp"
#endif
void GenMarkSweep::invoke_at_safepoint(bool clear_all_softrefs) {
assert(SafepointSynchronize::is_at_safepoint(), "must be at a safepoint");
class DeadSpacer : StackObj {
size_t _allowed_deadspace_words;
bool _active;
ContiguousSpace* _space;
SerialHeap* gch = SerialHeap::heap();
#ifdef ASSERT
if (gch->soft_ref_policy()->should_clear_all_soft_refs()) {
assert(clear_all_softrefs, "Policy should have been checked earlier");
}
#endif
public:
DeadSpacer(ContiguousSpace* space) : _allowed_deadspace_words(0), _space(space) {
size_t ratio = _space->allowed_dead_ratio();
_active = ratio > 0;
gch->trace_heap_before_gc(_gc_tracer);
// Increment the invocation count
_total_invocations++;
// Capture used regions for each generation that will be
// subject to collection, so that card table adjustments can
// be made intelligently (see clear / invalidate further below).
gch->save_used_regions();
allocate_stacks();
mark_sweep_phase1(clear_all_softrefs);
mark_sweep_phase2();
// Don't add any more derived pointers during phase3
#if COMPILER2_OR_JVMCI
assert(DerivedPointerTable::is_active(), "Sanity");
DerivedPointerTable::set_active(false);
#endif
mark_sweep_phase3();
mark_sweep_phase4();
restore_marks();
// Set saved marks for allocation profiler (and other things? -- dld)
// (Should this be in general part?)
gch->save_marks();
deallocate_stacks();
MarkSweep::_string_dedup_requests->flush();
bool is_young_gen_empty = (gch->young_gen()->used() == 0);
gch->rem_set()->maintain_old_to_young_invariant(gch->old_gen(), is_young_gen_empty);
gch->prune_scavengable_nmethods();
// Update heap occupancy information which is used as
// input to soft ref clearing policy at the next gc.
Universe::heap()->update_capacity_and_used_at_gc();
// Signal that we have completed a visit to all live objects.
Universe::heap()->record_whole_heap_examined_timestamp();
gch->trace_heap_after_gc(_gc_tracer);
}
void GenMarkSweep::allocate_stacks() {
void* scratch = nullptr;
size_t num_words;
DefNewGeneration* young_gen = (DefNewGeneration*)SerialHeap::heap()->young_gen();
young_gen->contribute_scratch(scratch, num_words);
if (scratch != nullptr) {
_preserved_count_max = num_words * HeapWordSize / sizeof(PreservedMark);
} else {
_preserved_count_max = 0;
if (_active) {
// We allow some amount of garbage towards the bottom of the space, so
// we don't start compacting before there is a significant gain to be made.
// Occasionally, we want to ensure a full compaction, which is determined
// by the MarkSweepAlwaysCompactCount parameter.
if ((MarkSweep::total_invocations() % MarkSweepAlwaysCompactCount) != 0) {
_allowed_deadspace_words = (space->capacity() * ratio / 100) / HeapWordSize;
} else {
_active = false;
}
}
}
_preserved_marks = (PreservedMark*)scratch;
_preserved_count = 0;
bool insert_deadspace(HeapWord* dead_start, HeapWord* dead_end) {
if (!_active) {
return false;
}
_preserved_overflow_stack_set.init(1);
}
size_t dead_length = pointer_delta(dead_end, dead_start);
if (_allowed_deadspace_words >= dead_length) {
_allowed_deadspace_words -= dead_length;
CollectedHeap::fill_with_object(dead_start, dead_length);
oop obj = cast_to_oop(dead_start);
// obj->set_mark(obj->mark().set_marked());
assert(dead_length == obj->size(), "bad filler object size");
log_develop_trace(gc, compaction)("Inserting object to dead space: " PTR_FORMAT ", " PTR_FORMAT ", " SIZE_FORMAT "b",
p2i(dead_start), p2i(dead_end), dead_length * HeapWordSize);
void GenMarkSweep::deallocate_stacks() {
if (_preserved_count_max != 0) {
DefNewGeneration* young_gen = (DefNewGeneration*)SerialHeap::heap()->young_gen();
young_gen->reset_scratch();
return true;
} else {
_active = false;
return false;
}
}
};
// Implement the "compaction" part of the mark-compact GC algorithm.
class Compacter {
// There are four spaces in total, but only the first three can be used after
// compact. IOW, old and eden/from must be enough for all live objs
static constexpr uint max_num_spaces = 4;
struct CompactionSpace {
ContiguousSpace* _space;
// Will be the new top after compaction is complete.
HeapWord* _compaction_top;
// The first dead word in this contiguous space. It's an optimization to
// skip large chunk of live objects at the beginning.
HeapWord* _first_dead;
void init(ContiguousSpace* space) {
_space = space;
_compaction_top = space->bottom();
_first_dead = nullptr;
}
};
CompactionSpace _spaces[max_num_spaces];
// The num of spaces to be compacted, i.e. containing live objs.
uint _num_spaces;
uint _index;
HeapWord* get_compaction_top(uint index) const {
return _spaces[index]._compaction_top;
}
_preserved_overflow_stack_set.reclaim();
_marking_stack.clear();
_objarray_stack.clear(true);
}
HeapWord* get_first_dead(uint index) const {
return _spaces[index]._first_dead;
}
void GenMarkSweep::mark_sweep_phase1(bool clear_all_softrefs) {
ContiguousSpace* get_space(uint index) const {
return _spaces[index]._space;
}
void record_first_dead(uint index, HeapWord* first_dead) {
assert(_spaces[index]._first_dead == nullptr, "should write only once");
_spaces[index]._first_dead = first_dead;
}
HeapWord* alloc(size_t words) {
while (true) {
if (words <= pointer_delta(_spaces[_index]._space->end(),
_spaces[_index]._compaction_top)) {
HeapWord* result = _spaces[_index]._compaction_top;
_spaces[_index]._compaction_top += words;
if (_index == 0) {
// old-gen requires BOT update
static_cast<TenuredSpace*>(_spaces[0]._space)->update_for_block(result, result + words);
}
return result;
}
// out-of-memory in this space
_index++;
assert(_index < max_num_spaces - 1, "the last space should not be used");
}
}
static void prefetch_read_scan(void* p) {
if (PrefetchScanIntervalInBytes >= 0) {
Prefetch::read(p, PrefetchScanIntervalInBytes);
}
}
static void prefetch_write_scan(void* p) {
if (PrefetchScanIntervalInBytes >= 0) {
Prefetch::write(p, PrefetchScanIntervalInBytes);
}
}
static void prefetch_write_copy(void* p) {
if (PrefetchCopyIntervalInBytes >= 0) {
Prefetch::write(p, PrefetchCopyIntervalInBytes);
}
}
static void forward_obj(oop obj, HeapWord* new_addr) {
prefetch_write_scan(obj);
if (cast_from_oop<HeapWord*>(obj) != new_addr) {
obj->forward_to(cast_to_oop(new_addr));
} else {
assert(obj->is_gc_marked(), "inv");
// This obj will stay in-place. Fix the markword.
obj->init_mark();
}
}
static HeapWord* find_next_live_addr(HeapWord* start, HeapWord* end) {
for (HeapWord* i_addr = start; i_addr < end; /* empty */) {
prefetch_read_scan(i_addr);
oop obj = cast_to_oop(i_addr);
if (obj->is_gc_marked()) {
return i_addr;
}
i_addr += obj->size();
}
return end;
};
static size_t relocate(HeapWord* addr) {
// Prefetch source and destination
prefetch_read_scan(addr);
oop obj = cast_to_oop(addr);
oop new_obj = obj->forwardee();
HeapWord* new_addr = cast_from_oop<HeapWord*>(new_obj);
assert(addr != new_addr, "inv");
prefetch_write_copy(new_addr);
size_t obj_size = obj->size();
Copy::aligned_conjoint_words(addr, new_addr, obj_size);
new_obj->init_mark();
return obj_size;
}
public:
explicit Compacter(SerialHeap* heap) {
// In this order so that heap is compacted towards old-gen.
_spaces[0].init(heap->old_gen()->space());
_spaces[1].init(heap->young_gen()->eden());
_spaces[2].init(heap->young_gen()->from());
bool is_promotion_failed = (heap->young_gen()->from()->next_compaction_space() != nullptr);
if (is_promotion_failed) {
_spaces[3].init(heap->young_gen()->to());
_num_spaces = 4;
} else {
_num_spaces = 3;
}
_index = 0;
}
void phase2_calculate_new_addr() {
for (uint i = 0; i < _num_spaces; ++i) {
ContiguousSpace* space = get_space(i);
HeapWord* cur_addr = space->bottom();
HeapWord* top = space->top();
bool record_first_dead_done = false;
DeadSpacer dead_spacer(space);
while (cur_addr < top) {
oop obj = cast_to_oop(cur_addr);
size_t obj_size = obj->size();
if (obj->is_gc_marked()) {
HeapWord* new_addr = alloc(obj_size);
forward_obj(obj, new_addr);
cur_addr += obj_size;
} else {
// Skipping the current known-unmarked obj
HeapWord* next_live_addr = find_next_live_addr(cur_addr + obj_size, top);
if (dead_spacer.insert_deadspace(cur_addr, next_live_addr)) {
// Register space for the filler obj
alloc(pointer_delta(next_live_addr, cur_addr));
} else {
if (!record_first_dead_done) {
record_first_dead(i, cur_addr);
record_first_dead_done = true;
}
*(HeapWord**)cur_addr = next_live_addr;
}
cur_addr = next_live_addr;
}
}
if (!record_first_dead_done) {
record_first_dead(i, top);
}
}
}
void phase3_adjust_pointers() {
for (uint i = 0; i < _num_spaces; ++i) {
ContiguousSpace* space = get_space(i);
HeapWord* cur_addr = space->bottom();
HeapWord* const top = space->top();
HeapWord* const first_dead = get_first_dead(i);
while (cur_addr < top) {
prefetch_write_scan(cur_addr);
if (cur_addr < first_dead || cast_to_oop(cur_addr)->is_gc_marked()) {
size_t size = MarkSweep::adjust_pointers(cast_to_oop(cur_addr));
cur_addr += size;
} else {
assert(*(HeapWord**)cur_addr > cur_addr, "forward progress");
cur_addr = *(HeapWord**)cur_addr;
}
}
}
}
void phase4_compact() {
for (uint i = 0; i < _num_spaces; ++i) {
ContiguousSpace* space = get_space(i);
HeapWord* cur_addr = space->bottom();
HeapWord* top = space->top();
// Check if the first obj inside this space is forwarded.
if (!cast_to_oop(cur_addr)->is_forwarded()) {
// Jump over consecutive (in-place) live-objs-chunk
cur_addr = get_first_dead(i);
}
while (cur_addr < top) {
if (!cast_to_oop(cur_addr)->is_forwarded()) {
cur_addr = *(HeapWord**) cur_addr;
continue;
}
cur_addr += relocate(cur_addr);
}
// Reset top and unused memory
space->set_top(get_compaction_top(i));
if (ZapUnusedHeapArea) {
space->mangle_unused_area();
}
}
}
};
void GenMarkSweep::phase1_mark(bool clear_all_softrefs) {
// Recursively traverse all live objects and mark them
GCTraceTime(Info, gc, phases) tm("Phase 1: Mark live objects", _gc_timer);
@ -241,54 +426,121 @@ void GenMarkSweep::mark_sweep_phase1(bool clear_all_softrefs) {
}
}
void GenMarkSweep::invoke_at_safepoint(bool clear_all_softrefs) {
assert(SafepointSynchronize::is_at_safepoint(), "must be at a safepoint");
void GenMarkSweep::mark_sweep_phase2() {
// Now all live objects are marked, compute the new object addresses.
GCTraceTime(Info, gc, phases) tm("Phase 2: Compute new object addresses", _gc_timer);
SerialHeap::heap()->prepare_for_compaction();
}
class GenAdjustPointersClosure: public SerialHeap::GenClosure {
public:
void do_generation(Generation* gen) {
gen->adjust_pointers();
}
};
void GenMarkSweep::mark_sweep_phase3() {
SerialHeap* gch = SerialHeap::heap();
// Adjust the pointers to reflect the new locations
GCTraceTime(Info, gc, phases) tm("Phase 3: Adjust pointers", gc_timer());
ClassLoaderDataGraph::verify_claimed_marks_cleared(ClassLoaderData::_claim_stw_fullgc_adjust);
CodeBlobToOopClosure code_closure(&adjust_pointer_closure, CodeBlobToOopClosure::FixRelocations);
gch->process_roots(SerialHeap::SO_AllCodeCache,
&adjust_pointer_closure,
&adjust_cld_closure,
&adjust_cld_closure,
&code_closure);
gch->gen_process_weak_roots(&adjust_pointer_closure);
adjust_marks();
GenAdjustPointersClosure blk;
gch->generation_iterate(&blk, true);
}
class GenCompactClosure: public SerialHeap::GenClosure {
public:
void do_generation(Generation* gen) {
gen->compact();
#ifdef ASSERT
if (gch->soft_ref_policy()->should_clear_all_soft_refs()) {
assert(clear_all_softrefs, "Policy should have been checked earlier");
}
};
#endif
void GenMarkSweep::mark_sweep_phase4() {
// All pointers are now adjusted, move objects accordingly
GCTraceTime(Info, gc, phases) tm("Phase 4: Move objects", _gc_timer);
gch->trace_heap_before_gc(_gc_tracer);
GenCompactClosure blk;
SerialHeap::heap()->generation_iterate(&blk, true);
// Increment the invocation count
_total_invocations++;
// Capture used regions for each generation that will be
// subject to collection, so that card table adjustments can
// be made intelligently (see clear / invalidate further below).
gch->save_used_regions();
allocate_stacks();
phase1_mark(clear_all_softrefs);
Compacter compacter{gch};
{
// Now all live objects are marked, compute the new object addresses.
GCTraceTime(Info, gc, phases) tm("Phase 2: Compute new object addresses", _gc_timer);
compacter.phase2_calculate_new_addr();
}
// Don't add any more derived pointers during phase3
#if COMPILER2_OR_JVMCI
assert(DerivedPointerTable::is_active(), "Sanity");
DerivedPointerTable::set_active(false);
#endif
{
// Adjust the pointers to reflect the new locations
GCTraceTime(Info, gc, phases) tm("Phase 3: Adjust pointers", gc_timer());
ClassLoaderDataGraph::verify_claimed_marks_cleared(ClassLoaderData::_claim_stw_fullgc_adjust);
CodeBlobToOopClosure code_closure(&adjust_pointer_closure, CodeBlobToOopClosure::FixRelocations);
gch->process_roots(SerialHeap::SO_AllCodeCache,
&adjust_pointer_closure,
&adjust_cld_closure,
&adjust_cld_closure,
&code_closure);
WeakProcessor::oops_do(&adjust_pointer_closure);
adjust_marks();
compacter.phase3_adjust_pointers();
}
{
// All pointers are now adjusted, move objects accordingly
GCTraceTime(Info, gc, phases) tm("Phase 4: Move objects", _gc_timer);
compacter.phase4_compact();
}
restore_marks();
// Set saved marks for allocation profiler (and other things? -- dld)
// (Should this be in general part?)
gch->save_marks();
deallocate_stacks();
MarkSweep::_string_dedup_requests->flush();
bool is_young_gen_empty = (gch->young_gen()->used() == 0);
gch->rem_set()->maintain_old_to_young_invariant(gch->old_gen(), is_young_gen_empty);
gch->prune_scavengable_nmethods();
// Update heap occupancy information which is used as
// input to soft ref clearing policy at the next gc.
Universe::heap()->update_capacity_and_used_at_gc();
// Signal that we have completed a visit to all live objects.
Universe::heap()->record_whole_heap_examined_timestamp();
gch->trace_heap_after_gc(_gc_tracer);
}
void GenMarkSweep::allocate_stacks() {
void* scratch = nullptr;
size_t num_words;
DefNewGeneration* young_gen = (DefNewGeneration*)SerialHeap::heap()->young_gen();
young_gen->contribute_scratch(scratch, num_words);
if (scratch != nullptr) {
_preserved_count_max = num_words * HeapWordSize / sizeof(PreservedMark);
} else {
_preserved_count_max = 0;
}
_preserved_marks = (PreservedMark*)scratch;
_preserved_count = 0;
_preserved_overflow_stack_set.init(1);
}
void GenMarkSweep::deallocate_stacks() {
if (_preserved_count_max != 0) {
DefNewGeneration* young_gen = (DefNewGeneration*)SerialHeap::heap()->young_gen();
young_gen->reset_scratch();
}
_preserved_overflow_stack_set.reclaim();
_marking_stack.clear();
_objarray_stack.clear(true);
}

9
src/hotspot/share/gc/serial/genMarkSweep.hpp
View File

@ -32,15 +32,8 @@ class GenMarkSweep : public MarkSweep {
static void invoke_at_safepoint(bool clear_all_softrefs);
private:
// Mark live objects
static void mark_sweep_phase1(bool clear_all_softrefs);
// Calculate new addresses
static void mark_sweep_phase2();
// Update pointers
static void mark_sweep_phase3();
// Move objects to new positions
static void mark_sweep_phase4();
static void phase1_mark(bool clear_all_softrefs);
// Temporary data structures for traversal and storing/restoring marks
static void allocate_stacks();

31
src/hotspot/share/gc/serial/generation.cpp
View File

@ -218,34 +218,3 @@ void Generation::object_iterate(ObjectClosure* cl) {
GenerationObjIterateClosure blk(cl);
space_iterate(&blk);
}
void Generation::prepare_for_compaction(CompactPoint* cp) {
// Generic implementation, can be specialized
ContiguousSpace* space = first_compaction_space();
while (space != nullptr) {
space->prepare_for_compaction(cp);
space = space->next_compaction_space();
}
}
class AdjustPointersClosure: public SpaceClosure {
public:
void do_space(Space* sp) {
sp->adjust_pointers();
}
};
void Generation::adjust_pointers() {
// Note that this is done over all spaces, not just the compactible
// ones.
AdjustPointersClosure blk;
space_iterate(&blk, true);
}
void Generation::compact() {
ContiguousSpace* sp = first_compaction_space();
while (sp != nullptr) {
sp->compact();
sp = sp->next_compaction_space();
}
}

9
src/hotspot/share/gc/serial/generation.hpp
View File

@ -51,7 +51,7 @@
class DefNewGeneration;
class GCMemoryManager;
class ContiguousSpace;
class CompactPoint;
class OopClosure;
class GCStats;
@ -286,13 +286,6 @@ class Generation: public CHeapObj<mtGC> {
GCStats* gc_stats() const { return _gc_stats; }
virtual void update_gc_stats(Generation* current_generation, bool full) {}
// Mark sweep support phase2
virtual void prepare_for_compaction(CompactPoint* cp);
// Mark sweep support phase3
virtual void adjust_pointers();
// Mark sweep support phase4
virtual void compact();
// Accessing "marks".
// This function gives a generation a chance to note a point between

4
src/hotspot/share/gc/serial/tenuredGeneration.hpp
View File

@ -70,8 +70,6 @@ class TenuredGeneration: public Generation {
GenerationCounters* _gen_counters;
CSpaceCounters* _space_counters;
// Accessing spaces
TenuredSpace* space() const { return _the_space; }
// Attempt to expand the generation by "bytes". Expand by at a
// minimum "expand_bytes". Return true if some amount (not
@ -85,6 +83,8 @@ class TenuredGeneration: public Generation {
public:
virtual void compute_new_size();
TenuredSpace* space() const { return _the_space; }
// Grow generation with specified size (returns false if unable to grow)
bool grow_by(size_t bytes);
// Grow generation to reserved size.

13
src/hotspot/share/gc/shared/genCollectedHeap.cpp
View File

@ -713,10 +713,6 @@ void GenCollectedHeap::process_roots(ScanningOption so,
DEBUG_ONLY(ScavengableNMethods::asserted_non_scavengable_nmethods_do(&assert_code_is_non_scavengable));
}
void GenCollectedHeap::gen_process_weak_roots(OopClosure* root_closure) {
WeakProcessor::oops_do(root_closure);
}
bool GenCollectedHeap::no_allocs_since_save_marks() {
return _young_gen->no_allocs_since_save_marks() &&
_old_gen->no_allocs_since_save_marks();
@ -911,15 +907,6 @@ GenCollectedHeap* GenCollectedHeap::heap() {
return named_heap<GenCollectedHeap>(CollectedHeap::Serial);
}
#if INCLUDE_SERIALGC
void GenCollectedHeap::prepare_for_compaction() {
// Start by compacting into same gen.
CompactPoint cp(_old_gen);
_old_gen->prepare_for_compaction(&cp);
_young_gen->prepare_for_compaction(&cp);
}
#endif // INCLUDE_SERIALGC
void GenCollectedHeap::verify(VerifyOption option /* ignored */) {
log_debug(gc, verify)("%s", _old_gen->name());
_old_gen->verify();

12
src/hotspot/share/gc/shared/genCollectedHeap.hpp
View File

@ -292,11 +292,6 @@ public:
CLDClosure* weak_cld_closure,
CodeBlobToOopClosure* code_roots);
// Apply "root_closure" to all the weak roots of the system.
// These include JNI weak roots, string table,
// and referents of reachable weak refs.
void gen_process_weak_roots(OopClosure* root_closure);
// Set the saved marks of generations, if that makes sense.
// In particular, if any generation might iterate over the oops
// in other generations, it should call this method.
@ -340,13 +335,6 @@ private:
HeapWord* mem_allocate_work(size_t size,
bool is_tlab);
#if INCLUDE_SERIALGC
// For use by mark-sweep. As implemented, mark-sweep-compact is global
// in an essential way: compaction is performed across generations, by
// iterating over spaces.
void prepare_for_compaction();
#endif
// Save the tops of the spaces in all generations
void record_gen_tops_before_GC() PRODUCT_RETURN;

239
src/hotspot/share/gc/shared/space.cpp
View File

@ -35,19 +35,13 @@
#include "oops/oop.inline.hpp"
#include "runtime/atomic.hpp"
#include "runtime/java.hpp"
#include "runtime/prefetch.inline.hpp"
#include "runtime/safepoint.hpp"
#include "utilities/align.hpp"
#include "utilities/copy.hpp"
#include "utilities/globalDefinitions.hpp"
#include "utilities/macros.hpp"
#if INCLUDE_SERIALGC
#include "gc/serial/serialBlockOffsetTable.inline.hpp"
#include "gc/serial/defNewGeneration.hpp"
#endif
ContiguousSpace::ContiguousSpace(): Space(),
_compaction_top(nullptr),
_next_compaction_space(nullptr),
_top(nullptr) {
_mangler = new GenSpaceMangler(this);
@ -59,8 +53,7 @@ ContiguousSpace::~ContiguousSpace() {
void ContiguousSpace::initialize(MemRegion mr,
bool clear_space,
bool mangle_space)
{
bool mangle_space) {
HeapWord* bottom = mr.start();
HeapWord* end = mr.end();
assert(Universe::on_page_boundary(bottom) && Universe::on_page_boundary(end),
@ -70,7 +63,6 @@ void ContiguousSpace::initialize(MemRegion mr,
if (clear_space) {
clear(mangle_space);
}
set_compaction_top(bottom);
_next_compaction_space = nullptr;
}
@ -80,7 +72,6 @@ void ContiguousSpace::clear(bool mangle_space) {
if (ZapUnusedHeapArea && mangle_space) {
mangle_unused_area();
}
_compaction_top = bottom();
}
bool ContiguousSpace::is_free_block(const HeapWord* p) const {
@ -115,230 +106,6 @@ void ContiguousSpace::mangle_unused_area_complete() {
#endif // NOT_PRODUCT
HeapWord* ContiguousSpace::forward(oop q, size_t size,
CompactPoint* cp, HeapWord* compact_top) {
// q is alive
// First check if we should switch compaction space
assert(this == cp->space, "'this' should be current compaction space.");
size_t compaction_max_size = pointer_delta(end(), compact_top);
while (size > compaction_max_size) {
// switch to next compaction space
cp->space->set_compaction_top(compact_top);
cp->space = cp->space->next_compaction_space();
if (cp->space == nullptr) {
cp->gen = GenCollectedHeap::heap()->young_gen();
assert(cp->gen != nullptr, "compaction must succeed");
cp->space = cp->gen->first_compaction_space();
assert(cp->space != nullptr, "generation must have a first compaction space");
}
compact_top = cp->space->bottom();
cp->space->set_compaction_top(compact_top);
compaction_max_size = pointer_delta(cp->space->end(), compact_top);
}
// store the forwarding pointer into the mark word
if (cast_from_oop<HeapWord*>(q) != compact_top) {
q->forward_to(cast_to_oop(compact_top));
assert(q->is_gc_marked(), "encoding the pointer should preserve the mark");
} else {
// if the object isn't moving we can just set the mark to the default
// mark and handle it specially later on.
q->init_mark();
assert(!q->is_forwarded(), "should not be forwarded");
}
compact_top += size;
// We need to update the offset table so that the beginnings of objects can be
// found during scavenge. Note that we are updating the offset table based on
// where the object will be once the compaction phase finishes.
cp->space->update_for_block(compact_top - size, compact_top);
return compact_top;
}
#if INCLUDE_SERIALGC
void ContiguousSpace::prepare_for_compaction(CompactPoint* cp) {
// Compute the new addresses for the live objects and store it in the mark
// Used by universe::mark_sweep_phase2()
// We're sure to be here before any objects are compacted into this
// space, so this is a good time to initialize this:
set_compaction_top(bottom());
if (cp->space == nullptr) {
assert(cp->gen != nullptr, "need a generation");
assert(cp->gen->first_compaction_space() == this, "just checking");
cp->space = cp->gen->first_compaction_space();
cp->space->set_compaction_top(cp->space->bottom());
}
HeapWord* compact_top = cp->space->compaction_top(); // This is where we are currently compacting to.
DeadSpacer dead_spacer(this);
HeapWord* end_of_live = bottom(); // One byte beyond the last byte of the last live object.
HeapWord* first_dead = nullptr; // The first dead object.
const intx interval = PrefetchScanIntervalInBytes;
HeapWord* cur_obj = bottom();
HeapWord* scan_limit = top();
while (cur_obj < scan_limit) {
if (cast_to_oop(cur_obj)->is_gc_marked()) {
// prefetch beyond cur_obj
Prefetch::write(cur_obj, interval);
size_t size = cast_to_oop(cur_obj)->size();
compact_top = cp->space->forward(cast_to_oop(cur_obj), size, cp, compact_top);
cur_obj += size;
end_of_live = cur_obj;
} else {
// run over all the contiguous dead objects
HeapWord* end = cur_obj;
do {
// prefetch beyond end
Prefetch::write(end, interval);
end += cast_to_oop(end)->size();
} while (end < scan_limit && !cast_to_oop(end)->is_gc_marked());
// see if we might want to pretend this object is alive so that
// we don't have to compact quite as often.
if (cur_obj == compact_top && dead_spacer.insert_deadspace(cur_obj, end)) {
oop obj = cast_to_oop(cur_obj);
compact_top = cp->space->forward(obj, obj->size(), cp, compact_top);
end_of_live = end;
} else {
// otherwise, it really is a free region.
// cur_obj is a pointer to a dead object. Use this dead memory to store a pointer to the next live object.
*(HeapWord**)cur_obj = end;
// see if this is the first dead region.
if (first_dead == nullptr) {
first_dead = cur_obj;
}
}
// move on to the next object
cur_obj = end;
}
}
assert(cur_obj == scan_limit, "just checking");
_end_of_live = end_of_live;
if (first_dead != nullptr) {
_first_dead = first_dead;
} else {
_first_dead = end_of_live;
}
// save the compaction_top of the compaction space.
cp->space->set_compaction_top(compact_top);
}
void ContiguousSpace::adjust_pointers() {
// Check first is there is any work to do.
if (used() == 0) {
return; // Nothing to do.
}
// adjust all the interior pointers to point at the new locations of objects
// Used by MarkSweep::mark_sweep_phase3()
HeapWord* cur_obj = bottom();
HeapWord* const end_of_live = _end_of_live; // Established by prepare_for_compaction().
HeapWord* const first_dead = _first_dead; // Established by prepare_for_compaction().
assert(first_dead <= end_of_live, "Stands to reason, no?");
const intx interval = PrefetchScanIntervalInBytes;
debug_only(HeapWord* prev_obj = nullptr);
while (cur_obj < end_of_live) {
Prefetch::write(cur_obj, interval);
if (cur_obj < first_dead || cast_to_oop(cur_obj)->is_gc_marked()) {
// cur_obj is alive
// point all the oops to the new location
size_t size = MarkSweep::adjust_pointers(cast_to_oop(cur_obj));
debug_only(prev_obj = cur_obj);
cur_obj += size;
} else {
debug_only(prev_obj = cur_obj);
// cur_obj is not a live object, instead it points at the next live object
cur_obj = *(HeapWord**)cur_obj;
assert(cur_obj > prev_obj, "we should be moving forward through memory, cur_obj: " PTR_FORMAT ", prev_obj: " PTR_FORMAT, p2i(cur_obj), p2i(prev_obj));
}
}
assert(cur_obj == end_of_live, "just checking");
}
void ContiguousSpace::compact() {
// Copy all live objects to their new location
// Used by MarkSweep::mark_sweep_phase4()
verify_up_to_first_dead(this);
HeapWord* const start = bottom();
HeapWord* const end_of_live = _end_of_live;
assert(_first_dead <= end_of_live, "Invariant. _first_dead: " PTR_FORMAT " <= end_of_live: " PTR_FORMAT, p2i(_first_dead), p2i(end_of_live));
if (_first_dead == end_of_live && (start == end_of_live || !cast_to_oop(start)->is_gc_marked())) {
// Nothing to compact. The space is either empty or all live object should be left in place.
clear_empty_region(this);
return;
}
const intx scan_interval = PrefetchScanIntervalInBytes;
const intx copy_interval = PrefetchCopyIntervalInBytes;
assert(start < end_of_live, "bottom: " PTR_FORMAT " should be < end_of_live: " PTR_FORMAT, p2i(start), p2i(end_of_live));
HeapWord* cur_obj = start;
if (_first_dead > cur_obj && !cast_to_oop(cur_obj)->is_gc_marked()) {
// All object before _first_dead can be skipped. They should not be moved.
// A pointer to the first live object is stored at the memory location for _first_dead.
cur_obj = *(HeapWord**)(_first_dead);
}
debug_only(HeapWord* prev_obj = nullptr);
while (cur_obj < end_of_live) {
if (!cast_to_oop(cur_obj)->is_forwarded()) {
debug_only(prev_obj = cur_obj);
// The first word of the dead object contains a pointer to the next live object or end of space.
cur_obj = *(HeapWord**)cur_obj;
assert(cur_obj > prev_obj, "we should be moving forward through memory");
} else {
// prefetch beyond q
Prefetch::read(cur_obj, scan_interval);
// size and destination
size_t size = cast_to_oop(cur_obj)->size();
HeapWord* compaction_top = cast_from_oop<HeapWord*>(cast_to_oop(cur_obj)->forwardee());
// prefetch beyond compaction_top
Prefetch::write(compaction_top, copy_interval);
// copy object and reinit its mark
assert(cur_obj != compaction_top, "everything in this pass should be moving");
Copy::aligned_conjoint_words(cur_obj, compaction_top, size);
oop new_obj = cast_to_oop(compaction_top);
ContinuationGCSupport::transform_stack_chunk(new_obj);
new_obj->init_mark();
assert(new_obj->klass() != nullptr, "should have a class");
debug_only(prev_obj = cur_obj);
cur_obj += size;
}
}
clear_empty_region(this);
}
#endif // INCLUDE_SERIALGC
void Space::print_short() const { print_short_on(tty); }
void Space::print_short_on(outputStream* st) const {
@ -481,10 +248,6 @@ HeapWord* ContiguousSpace::par_allocate(size_t size) {
}
#if INCLUDE_SERIALGC
void TenuredSpace::update_for_block(HeapWord* start, HeapWord* end) {
_offsets.update_for_block(start, end);
}
HeapWord* TenuredSpace::block_start_const(const void* addr) const {
HeapWord* cur_block = _offsets.block_start_reaching_into_card(addr);

87
src/hotspot/share/gc/shared/space.hpp
View File

@ -184,29 +184,12 @@ class Space: public CHeapObj<mtGC> {
// Allocation (return null if full). Enforces mutual exclusion internally.
virtual HeapWord* par_allocate(size_t word_size) = 0;
#if INCLUDE_SERIALGC
// Mark-sweep-compact support: all spaces can update pointers to objects
// moving as a part of compaction.
virtual void adjust_pointers() = 0;
#endif
void print() const;
virtual void print_on(outputStream* st) const;
void print_short() const;
void print_short_on(outputStream* st) const;
};
// A structure to represent a point at which objects are being copied
// during compaction.
class CompactPoint : public StackObj {
public:
Generation* gen;
ContiguousSpace* space;
CompactPoint(Generation* g = nullptr) :
gen(g), space(nullptr) {}
};
class GenSpaceMangler;
// A space in which the free area is contiguous. It therefore supports
@ -215,26 +198,13 @@ class ContiguousSpace: public Space {
friend class VMStructs;
private:
HeapWord* _compaction_top;
ContiguousSpace* _next_compaction_space;
static inline void verify_up_to_first_dead(ContiguousSpace* space) NOT_DEBUG_RETURN;
static inline void clear_empty_region(ContiguousSpace* space);
protected:
protected:
HeapWord* _top;
// A helper for mangling the unused area of the space in debug builds.
GenSpaceMangler* _mangler;
// Used during compaction.
HeapWord* _first_dead;
HeapWord* _end_of_live;
// This the function to invoke when an allocation of an object covering
// "start" to "end" occurs to update other internal data structures.
virtual void update_for_block(HeapWord* start, HeapWord* the_end) { }
GenSpaceMangler* mangler() { return _mangler; }
// Allocation helpers (return null if full).
@ -254,23 +224,13 @@ private:
// The "clear" method must be called on a region that may have
// had allocation performed in it, but is now to be considered empty.
virtual void clear(bool mangle_space);
// Used temporarily during a compaction phase to hold the value
// top should have when compaction is complete.
HeapWord* compaction_top() const { return _compaction_top; }
void set_compaction_top(HeapWord* value) {
assert(value == nullptr || (value >= bottom() && value <= end()),
"should point inside space");
_compaction_top = value;
}
void clear(bool mangle_space);
// Returns the next space (in the current generation) to be compacted in
// the global compaction order. Also is used to select the next
// space into which to compact.
virtual ContiguousSpace* next_compaction_space() const {
ContiguousSpace* next_compaction_space() const {
return _next_compaction_space;
}
@ -278,42 +238,10 @@ private:
_next_compaction_space = csp;
}
#if INCLUDE_SERIALGC
// MarkSweep support phase2
// Start the process of compaction of the current space: compute
// post-compaction addresses, and insert forwarding pointers. The fields
// "cp->gen" and "cp->compaction_space" are the generation and space into
// which we are currently compacting. This call updates "cp" as necessary,
// and leaves the "compaction_top" of the final value of
// "cp->compaction_space" up-to-date. Offset tables may be updated in
// this phase as if the final copy had occurred; if so, "cp->threshold"
// indicates when the next such action should be taken.
void prepare_for_compaction(CompactPoint* cp);
// MarkSweep support phase3
void adjust_pointers() override;
// MarkSweep support phase4
virtual void compact();
#endif // INCLUDE_SERIALGC
// The maximum percentage of objects that can be dead in the compacted
// live part of a compacted space ("deadwood" support.)
virtual size_t allowed_dead_ratio() const { return 0; };
// "q" is an object of the given "size" that should be forwarded;
// "cp" names the generation ("gen") and containing "this" (which must
// also equal "cp->space"). "compact_top" is where in "this" the
// next object should be forwarded to. If there is room in "this" for
// the object, insert an appropriate forwarding pointer in "q".
// If not, go to the next compaction space (there must
// be one, since compaction must succeed -- we go to the first space of
// the previous generation if necessary, updating "cp"), reset compact_top
// and then forward. In either case, returns the new value of "compact_top".
// Invokes the "update_for_block" function of the then-current compaction
// space.
virtual HeapWord* forward(oop q, size_t size, CompactPoint* cp,
HeapWord* compact_top);
// Accessors
HeapWord* top() const { return _top; }
void set_top(HeapWord* value) { _top = value; }
@ -359,12 +287,6 @@ private:
// Iteration
void object_iterate(ObjectClosure* blk) override;
// Compaction support
void reset_after_compaction() {
assert(compaction_top() >= bottom() && compaction_top() <= end(), "should point inside space");
set_top(compaction_top());
}
// Apply "blk->do_oop" to the addresses of all reference fields in objects
// starting with the _saved_mark_word, which was noted during a generation's
// save_marks and is required to denote the head of an object.
@ -419,8 +341,7 @@ class TenuredSpace: public ContiguousSpace {
inline HeapWord* allocate(size_t word_size) override;
inline HeapWord* par_allocate(size_t word_size) override;
// MarkSweep support phase3
void update_for_block(HeapWord* start, HeapWord* end) override;
inline void update_for_block(HeapWord* start, HeapWord* end);
void print_on(outputStream* st) const override;
};

91
src/hotspot/share/gc/shared/space.inline.hpp
View File

@ -27,17 +27,12 @@
#include "gc/shared/space.hpp"
#include "gc/serial/generation.hpp"
#include "gc/shared/collectedHeap.hpp"
#include "gc/shared/spaceDecorator.hpp"
#include "oops/oop.inline.hpp"
#include "oops/oopsHierarchy.hpp"
#include "runtime/prefetch.inline.hpp"
#include "runtime/safepoint.hpp"
#if INCLUDE_SERIALGC
#include "gc/serial/serialBlockOffsetTable.inline.hpp"
#include "gc/serial/markSweep.inline.hpp"
#endif
inline HeapWord* Space::block_start(const void* p) {
return block_start_const(p);
@ -60,90 +55,8 @@ inline HeapWord* TenuredSpace::par_allocate(size_t size) {
return res;
}
class DeadSpacer : StackObj {
size_t _allowed_deadspace_words;
bool _active;
ContiguousSpace* _space;
public:
DeadSpacer(ContiguousSpace* space) : _allowed_deadspace_words(0), _space(space) {
size_t ratio = _space->allowed_dead_ratio();
_active = ratio > 0;
if (_active) {
assert(!UseG1GC, "G1 should not be using dead space");
// We allow some amount of garbage towards the bottom of the space, so
// we don't start compacting before there is a significant gain to be made.
// Occasionally, we want to ensure a full compaction, which is determined
// by the MarkSweepAlwaysCompactCount parameter.
if ((MarkSweep::total_invocations() % MarkSweepAlwaysCompactCount) != 0) {
_allowed_deadspace_words = (space->capacity() * ratio / 100) / HeapWordSize;
} else {
_active = false;
}
}
}
bool insert_deadspace(HeapWord* dead_start, HeapWord* dead_end) {
if (!_active) {
return false;
}
size_t dead_length = pointer_delta(dead_end, dead_start);
if (_allowed_deadspace_words >= dead_length) {
_allowed_deadspace_words -= dead_length;
CollectedHeap::fill_with_object(dead_start, dead_length);
oop obj = cast_to_oop(dead_start);
obj->set_mark(obj->mark().set_marked());
assert(dead_length == obj->size(), "bad filler object size");
log_develop_trace(gc, compaction)("Inserting object to dead space: " PTR_FORMAT ", " PTR_FORMAT ", " SIZE_FORMAT "b",
p2i(dead_start), p2i(dead_end), dead_length * HeapWordSize);
return true;
} else {
_active = false;
return false;
}
}
};
#ifdef ASSERT
inline void ContiguousSpace::verify_up_to_first_dead(ContiguousSpace* space) {
HeapWord* cur_obj = space->bottom();
if (cur_obj < space->_end_of_live && space->_first_dead > cur_obj && !cast_to_oop(cur_obj)->is_gc_marked()) {
// we have a chunk of the space which hasn't moved and we've reinitialized
// the mark word during the previous pass, so we can't use is_gc_marked for
// the traversal.
HeapWord* prev_obj = nullptr;
while (cur_obj < space->_first_dead) {
size_t size = cast_to_oop(cur_obj)->size();
assert(!cast_to_oop(cur_obj)->is_gc_marked(), "should be unmarked (special dense prefix handling)");
prev_obj = cur_obj;
cur_obj += size;
}
}
}
#endif
inline void ContiguousSpace::clear_empty_region(ContiguousSpace* space) {
// Let's remember if we were empty before we did the compaction.
bool was_empty = space->used_region().is_empty();
// Reset space after compaction is complete
space->reset_after_compaction();
// We do this clear, below, since it has overloaded meanings for some
// space subtypes. For example, TenuredSpace's that were
// compacted into will have had their offset table thresholds updated
// continuously, but those that weren't need to have their thresholds
// re-initialized. Also mangles unused area for debugging.
if (space->used_region().is_empty()) {
if (!was_empty) space->clear(SpaceDecorator::Mangle);
} else {
if (ZapUnusedHeapArea) space->mangle_unused_area();
}
inline void TenuredSpace::update_for_block(HeapWord* start, HeapWord* end) {
_offsets.update_for_block(start, end);
}
#endif // INCLUDE_SERIALGC

4
src/hotspot/share/gc/shared/vmStructs_gc.hpp
View File

@ -99,10 +99,6 @@
nonstatic_field(CollectedHeap, _is_gc_active, bool) \
nonstatic_field(CollectedHeap, _total_collections, unsigned int) \
\
nonstatic_field(ContiguousSpace, _compaction_top, HeapWord*) \
nonstatic_field(ContiguousSpace, _first_dead, HeapWord*) \
nonstatic_field(ContiguousSpace, _end_of_live, HeapWord*) \
\
nonstatic_field(ContiguousSpace, _top, HeapWord*) \
nonstatic_field(ContiguousSpace, _saved_mark_word, HeapWord*) \
\