stan/jdk-24
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

8282072: G1: Rename CardSetPtr to CardSetContainerPtr

Browse Source 
Reviewed-by: ayang, tschatzl
This commit is contained in:
Ivan Walulya 2022-03-17 08:54:29 +00:00
parent 3da5204b3c
commit a5ebcc0c04
8 changed files with 346 additions and 353 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

405
src/hotspot/share/gc/g1/g1CardSet.cpp
View File

@ -26,25 +26,19 @@
#include "gc/g1/g1CardSet.inline.hpp" #include "gc/g1/g1CardSet.inline.hpp"
#include "gc/g1/g1CardSetContainers.inline.hpp" #include "gc/g1/g1CardSetContainers.inline.hpp"
#include "gc/g1/g1CardSetMemory.inline.hpp" #include "gc/g1/g1CardSetMemory.inline.hpp"
#include "gc/g1/g1FromCardCache.hpp"
#include "gc/g1/heapRegion.inline.hpp" #include "gc/g1/heapRegion.inline.hpp"
#include "gc/shared/gcLogPrecious.hpp"
#include "gc/shared/gcTraceTime.inline.hpp"
#include "memory/allocation.inline.hpp" #include "memory/allocation.inline.hpp"
#include "runtime/atomic.hpp" #include "runtime/atomic.hpp"
#include "runtime/globals_extension.hpp" #include "runtime/globals_extension.hpp"
#include "runtime/mutex.hpp"
#include "utilities/bitMap.inline.hpp" #include "utilities/bitMap.inline.hpp"
#include "utilities/concurrentHashTable.inline.hpp" #include "utilities/concurrentHashTable.inline.hpp"
#include "utilities/globalDefinitions.hpp" #include "utilities/globalDefinitions.hpp"
#include "utilities/lockFreeStack.hpp"
#include "utilities/spinYield.hpp"
#include "gc/shared/gcLogPrecious.hpp" G1CardSet::ContainerPtr G1CardSet::FullCardSet = (G1CardSet::ContainerPtr)-1;
#include "gc/shared/gcTraceTime.inline.hpp"
#include "runtime/java.hpp"
G1CardSet::CardSetPtr G1CardSet::FullCardSet = (G1CardSet::CardSetPtr)-1; static uint default_log2_card_regions_per_region() {
static uint default_log2_card_region_per_region() {
uint log2_card_regions_per_heap_region = 0; uint log2_card_regions_per_heap_region = 0;
const uint card_container_limit = G1CardSetContainer::LogCardsPerRegionLimit; const uint card_container_limit = G1CardSetContainer::LogCardsPerRegionLimit;
@ -62,7 +56,7 @@ G1CardSetConfiguration::G1CardSetConfiguration() :
G1RemSetHowlNumBuckets, /* num_buckets_in_howl */ G1RemSetHowlNumBuckets, /* num_buckets_in_howl */
(double)G1RemSetCoarsenHowlToFullPercent / 100, /* cards_in_howl_threshold_percent */ (double)G1RemSetCoarsenHowlToFullPercent / 100, /* cards_in_howl_threshold_percent */
(uint)HeapRegion::CardsPerRegion, /* max_cards_in_cardset */ (uint)HeapRegion::CardsPerRegion, /* max_cards_in_cardset */
default_log2_card_region_per_region()) /* log2_card_region_per_region */ default_log2_card_regions_per_region()) /* log2_card_regions_per_region */
{ {
assert((_log2_card_regions_per_heap_region + _log2_cards_per_card_region) == (uint)HeapRegion::LogCardsPerRegion, assert((_log2_card_regions_per_heap_region + _log2_cards_per_card_region) == (uint)HeapRegion::LogCardsPerRegion,
"inconsistent heap region virtualization setup"); "inconsistent heap region virtualization setup");
@ -73,7 +67,7 @@ G1CardSetConfiguration::G1CardSetConfiguration(uint max_cards_in_array,
uint max_buckets_in_howl, uint max_buckets_in_howl,
double cards_in_howl_threshold_percent, double cards_in_howl_threshold_percent,
uint max_cards_in_card_set, uint max_cards_in_card_set,
uint log2_card_region_per_region) : uint log2_card_regions_per_region) :
G1CardSetConfiguration(log2i_exact(max_cards_in_card_set), /* inline_ptr_bits_per_card */ G1CardSetConfiguration(log2i_exact(max_cards_in_card_set), /* inline_ptr_bits_per_card */
max_cards_in_array, /* max_cards_in_array */ max_cards_in_array, /* max_cards_in_array */
cards_in_bitmap_threshold_percent, /* cards_in_bitmap_threshold_percent */ cards_in_bitmap_threshold_percent, /* cards_in_bitmap_threshold_percent */
@ -82,7 +76,7 @@ G1CardSetConfiguration::G1CardSetConfiguration(uint max_cards_in_array,
max_buckets_in_howl), max_buckets_in_howl),
cards_in_howl_threshold_percent, /* cards_in_howl_threshold_percent */ cards_in_howl_threshold_percent, /* cards_in_howl_threshold_percent */
max_cards_in_card_set, /* max_cards_in_cardset */ max_cards_in_card_set, /* max_cards_in_cardset */
log2_card_region_per_region) log2_card_regions_per_region)
{ } { }
G1CardSetConfiguration::G1CardSetConfiguration(uint inline_ptr_bits_per_card, G1CardSetConfiguration::G1CardSetConfiguration(uint inline_ptr_bits_per_card,
@ -197,7 +191,7 @@ void G1CardSetCoarsenStats::print_on(outputStream* out) {
} }
class G1CardSetHashTable : public CHeapObj<mtGCCardSet> { class G1CardSetHashTable : public CHeapObj<mtGCCardSet> {
using CardSetPtr = G1CardSet::CardSetPtr; using ContainerPtr = G1CardSet::ContainerPtr;
// Did we insert at least one card in the table? // Did we insert at least one card in the table?
bool volatile _inserted_card; bool volatile _inserted_card;
@ -231,12 +225,12 @@ class G1CardSetHashTable : public CHeapObj<mtGCCardSet> {
}; };
class G1CardSetHashTableScan : public StackObj { class G1CardSetHashTableScan : public StackObj {
G1CardSet::CardSetPtrClosure* _scan_f; G1CardSet::ContainerPtrClosure* _scan_f;
public: public:
explicit G1CardSetHashTableScan(G1CardSet::CardSetPtrClosure* f) : _scan_f(f) { } explicit G1CardSetHashTableScan(G1CardSet::ContainerPtrClosure* f) : _scan_f(f) { }
bool operator()(G1CardSetHashTableValue* value) { bool operator()(G1CardSetHashTableValue* value) {
_scan_f->do_cardsetptr(value->_region_idx, value->_num_occupied, value->_card_set); _scan_f->do_containerptr(value->_region_idx, value->_num_occupied, value->_container);
return true; return true;
} }
}; };
@ -284,19 +278,19 @@ public:
return found.value(); return found.value();
} }
void iterate_safepoint(G1CardSet::CardSetPtrClosure* cl2) { void iterate_safepoint(G1CardSet::ContainerPtrClosure* cl2) {
G1CardSetHashTableScan cl(cl2); G1CardSetHashTableScan cl(cl2);
_table.do_safepoint_scan(cl); _table.do_safepoint_scan(cl);
} }
void iterate(G1CardSet::CardSetPtrClosure* cl2) { void iterate(G1CardSet::ContainerPtrClosure* cl2) {
G1CardSetHashTableScan cl(cl2); G1CardSetHashTableScan cl(cl2);
_table.do_scan(Thread::current(), cl); _table.do_scan(Thread::current(), cl);
} }
void reset() { void reset() {
if (Atomic::load(&_inserted_card)) { if (Atomic::load(&_inserted_card)) {
_table.unsafe_reset(InitialLogTableSize); _table.unsafe_reset(InitialLogTableSize);
Atomic::store(&_inserted_card, false); Atomic::store(&_inserted_card, false);
} }
} }
@ -343,93 +337,93 @@ G1CardSet::~G1CardSet() {
_mm->flush(); _mm->flush();
} }
uint G1CardSet::card_set_type_to_mem_object_type(uintptr_t type) const { uint G1CardSet::container_type_to_mem_object_type(uintptr_t type) const {
assert(type == G1CardSet::CardSetArrayOfCards || assert(type == G1CardSet::ContainerArrayOfCards ||
type == G1CardSet::CardSetBitMap || type == G1CardSet::ContainerBitMap ||
type == G1CardSet::CardSetHowl, "should not allocate card set type %zu", type); type == G1CardSet::ContainerHowl, "should not allocate container type %zu", type);
return (uint)type; return (uint)type;
} }
uint8_t* G1CardSet::allocate_mem_object(uintptr_t type) { uint8_t* G1CardSet::allocate_mem_object(uintptr_t type) {
return _mm->allocate(card_set_type_to_mem_object_type(type)); return _mm->allocate(container_type_to_mem_object_type(type));
} }
void G1CardSet::free_mem_object(CardSetPtr card_set) { void G1CardSet::free_mem_object(ContainerPtr container) {
assert(card_set != G1CardSet::FreeCardSet, "should not free Free card set"); assert(container != G1CardSet::FreeCardSet, "should not free container FreeCardSet");
assert(card_set != G1CardSet::FullCardSet, "should not free Full card set"); assert(container != G1CardSet::FullCardSet, "should not free container FullCardSet");
uintptr_t type = card_set_type(card_set); uintptr_t type = container_type(container);
void* value = strip_card_set_type(card_set); void* value = strip_container_type(container);
assert(type == G1CardSet::CardSetArrayOfCards || assert(type == G1CardSet::ContainerArrayOfCards ||
type == G1CardSet::CardSetBitMap || type == G1CardSet::ContainerBitMap ||
type == G1CardSet::CardSetHowl, "should not free card set type %zu", type); type == G1CardSet::ContainerHowl, "should not free card set type %zu", type);
assert(static_cast<G1CardSetContainer*>(value)->refcount() == 1, "must be"); assert(static_cast<G1CardSetContainer*>(value)->refcount() == 1, "must be");
_mm->free(card_set_type_to_mem_object_type(type), value); _mm->free(container_type_to_mem_object_type(type), value);
} }
G1CardSet::CardSetPtr G1CardSet::acquire_card_set(CardSetPtr volatile* card_set_addr) { G1CardSet::ContainerPtr G1CardSet::acquire_container(ContainerPtr volatile* container_addr) {
// Update reference counts under RCU critical section to avoid a // Update reference counts under RCU critical section to avoid a
// use-after-cleapup bug where we increment a reference count for // use-after-cleapup bug where we increment a reference count for
// an object whose memory has already been cleaned up and reused. // an object whose memory has already been cleaned up and reused.
GlobalCounter::CriticalSection cs(Thread::current()); GlobalCounter::CriticalSection cs(Thread::current());
while (true) { while (true) {
// Get cardsetptr and increment refcount atomically wrt to memory reuse. // Get ContainerPtr and increment refcount atomically wrt to memory reuse.
CardSetPtr card_set = Atomic::load_acquire(card_set_addr); ContainerPtr container = Atomic::load_acquire(container_addr);
uint cs_type = card_set_type(card_set); uint cs_type = container_type(container);
if (card_set == FullCardSet || cs_type == CardSetInlinePtr) { if (container == FullCardSet || cs_type == ContainerInlinePtr) {
return card_set; return container;
} }
G1CardSetContainer* card_set_on_heap = (G1CardSetContainer*)strip_card_set_type(card_set); G1CardSetContainer* container_on_heap = (G1CardSetContainer*)strip_container_type(container);
if (card_set_on_heap->try_increment_refcount()) { if (container_on_heap->try_increment_refcount()) {
assert(card_set_on_heap->refcount() >= 3, "Smallest value is 3"); assert(container_on_heap->refcount() >= 3, "Smallest value is 3");
return card_set; return container;
} }
} }
} }
bool G1CardSet::release_card_set(CardSetPtr card_set) { bool G1CardSet::release_container(ContainerPtr container) {
uint cs_type = card_set_type(card_set); uint cs_type = container_type(container);
if (card_set == FullCardSet || cs_type == CardSetInlinePtr) { if (container == FullCardSet || cs_type == ContainerInlinePtr) {
return false; return false;
} }
G1CardSetContainer* card_set_on_heap = (G1CardSetContainer*)strip_card_set_type(card_set); G1CardSetContainer* container_on_heap = (G1CardSetContainer*)strip_container_type(container);
return card_set_on_heap->decrement_refcount() == 1; return container_on_heap->decrement_refcount() == 1;
} }
void G1CardSet::release_and_maybe_free_card_set(CardSetPtr card_set) { void G1CardSet::release_and_maybe_free_container(ContainerPtr container) {
if (release_card_set(card_set)) { if (release_container(container)) {
free_mem_object(card_set); free_mem_object(container);
} }
} }
void G1CardSet::release_and_must_free_card_set(CardSetPtr card_set) { void G1CardSet::release_and_must_free_container(ContainerPtr container) {
bool should_free = release_card_set(card_set); bool should_free = release_container(container);
assert(should_free, "should have been the only one having a reference"); assert(should_free, "should have been the only one having a reference");
free_mem_object(card_set); free_mem_object(container);
} }
class G1ReleaseCardsets : public StackObj { class G1ReleaseCardsets : public StackObj {
G1CardSet* _card_set; G1CardSet* _card_set;
using CardSetPtr = G1CardSet::CardSetPtr; using ContainerPtr = G1CardSet::ContainerPtr;
void coarsen_to_full(CardSetPtr* card_set_addr) { void coarsen_to_full(ContainerPtr* container_addr) {
while (true) { while (true) {
CardSetPtr cur_card_set = Atomic::load_acquire(card_set_addr); ContainerPtr cur_container = Atomic::load_acquire(container_addr);
uint cs_type = G1CardSet::card_set_type(cur_card_set); uint cs_type = G1CardSet::container_type(cur_container);
if (cur_card_set == G1CardSet::FullCardSet) { if (cur_container == G1CardSet::FullCardSet) {
return; return;
} }
CardSetPtr old_value = Atomic::cmpxchg(card_set_addr, cur_card_set, G1CardSet::FullCardSet); ContainerPtr old_value = Atomic::cmpxchg(container_addr, cur_container, G1CardSet::FullCardSet);
if (old_value == cur_card_set) { if (old_value == cur_container) {
_card_set->release_and_maybe_free_card_set(cur_card_set); _card_set->release_and_maybe_free_container(cur_container);
return; return;
} }
} }
@ -438,51 +432,51 @@ class G1ReleaseCardsets : public StackObj {
public: public:
explicit G1ReleaseCardsets(G1CardSet* card_set) : _card_set(card_set) { } explicit G1ReleaseCardsets(G1CardSet* card_set) : _card_set(card_set) { }
void operator ()(CardSetPtr* card_set_addr) { void operator ()(ContainerPtr* container_addr) {
coarsen_to_full(card_set_addr); coarsen_to_full(container_addr);
} }
}; };
G1AddCardResult G1CardSet::add_to_array(CardSetPtr card_set, uint card_in_region) { G1AddCardResult G1CardSet::add_to_array(ContainerPtr container, uint card_in_region) {
G1CardSetArray* array = card_set_ptr<G1CardSetArray>(card_set); G1CardSetArray* array = container_ptr<G1CardSetArray>(container);
return array->add(card_in_region); return array->add(card_in_region);
} }
G1AddCardResult G1CardSet::add_to_howl(CardSetPtr parent_card_set, G1AddCardResult G1CardSet::add_to_howl(ContainerPtr parent_container,
uint card_region, uint card_region,
uint card_in_region, uint card_in_region,
bool increment_total) { bool increment_total) {
G1CardSetHowl* howl = card_set_ptr<G1CardSetHowl>(parent_card_set); G1CardSetHowl* howl = container_ptr<G1CardSetHowl>(parent_container);
G1AddCardResult add_result; G1AddCardResult add_result;
CardSetPtr to_transfer = nullptr; ContainerPtr to_transfer = nullptr;
CardSetPtr card_set; ContainerPtr container;
uint bucket = _config->howl_bucket_index(card_in_region); uint bucket = _config->howl_bucket_index(card_in_region);
volatile CardSetPtr* bucket_entry = howl->get_card_set_addr(bucket); ContainerPtr volatile* bucket_entry = howl->get_container_addr(bucket);
while (true) { while (true) {
if (Atomic::load(&howl->_num_entries) >= _config->cards_in_howl_threshold()) { if (Atomic::load(&howl->_num_entries) >= _config->cards_in_howl_threshold()) {
return Overflow; return Overflow;
} }
card_set = acquire_card_set(bucket_entry); container = acquire_container(bucket_entry);
add_result = add_to_card_set(bucket_entry, card_set, card_region, card_in_region); add_result = add_to_container(bucket_entry, container, card_region, card_in_region);
if (add_result != Overflow) { if (add_result != Overflow) {
break; break;
} }
// Card set has overflown. Coarsen or retry. // Card set container has overflown. Coarsen or retry.
bool coarsened = coarsen_card_set(bucket_entry, card_set, card_in_region, true /* within_howl */); bool coarsened = coarsen_container(bucket_entry, container, card_in_region, true /* within_howl */);
_coarsen_stats.record_coarsening(card_set_type(card_set) + G1CardSetCoarsenStats::CoarsenHowlOffset, !coarsened); _coarsen_stats.record_coarsening(container_type(container) + G1CardSetCoarsenStats::CoarsenHowlOffset, !coarsened);
if (coarsened) { if (coarsened) {
// We have been the one coarsening this card set (and in the process added that card). // We successful coarsened this card set container (and in the process added the card).
add_result = Added; add_result = Added;
to_transfer = card_set; to_transfer = container;
break; break;
} }
// Somebody else beat us to coarsening. Retry. // Somebody else beat us to coarsening. Retry.
release_and_maybe_free_card_set(card_set); release_and_maybe_free_container(container);
} }
if (increment_total && add_result == Added) { if (increment_total && add_result == Added) {
@ -490,91 +484,91 @@ G1AddCardResult G1CardSet::add_to_howl(CardSetPtr parent_card_set,
} }
if (to_transfer != nullptr) { if (to_transfer != nullptr) {
transfer_cards_in_howl(parent_card_set, to_transfer, card_region); transfer_cards_in_howl(parent_container, to_transfer, card_region);
} }
release_and_maybe_free_card_set(card_set); release_and_maybe_free_container(container);
return add_result; return add_result;
} }
G1AddCardResult G1CardSet::add_to_bitmap(CardSetPtr card_set, uint card_in_region) { G1AddCardResult G1CardSet::add_to_bitmap(ContainerPtr container, uint card_in_region) {
G1CardSetBitMap* bitmap = card_set_ptr<G1CardSetBitMap>(card_set); G1CardSetBitMap* bitmap = container_ptr<G1CardSetBitMap>(container);
uint card_offset = _config->howl_bitmap_offset(card_in_region); uint card_offset = _config->howl_bitmap_offset(card_in_region);
return bitmap->add(card_offset, _config->cards_in_howl_bitmap_threshold(), _config->max_cards_in_howl_bitmap()); return bitmap->add(card_offset, _config->cards_in_howl_bitmap_threshold(), _config->max_cards_in_howl_bitmap());
} }
G1AddCardResult G1CardSet::add_to_inline_ptr(CardSetPtr volatile* card_set_addr, CardSetPtr card_set, uint card_in_region) { G1AddCardResult G1CardSet::add_to_inline_ptr(ContainerPtr volatile* container_addr, ContainerPtr container, uint card_in_region) {
G1CardSetInlinePtr value(card_set_addr, card_set); G1CardSetInlinePtr value(container_addr, container);
return value.add(card_in_region, _config->inline_ptr_bits_per_card(), _config->max_cards_in_inline_ptr()); return value.add(card_in_region, _config->inline_ptr_bits_per_card(), _config->max_cards_in_inline_ptr());
} }
G1CardSet::CardSetPtr G1CardSet::create_coarsened_array_of_cards(uint card_in_region, bool within_howl) { G1CardSet::ContainerPtr G1CardSet::create_coarsened_array_of_cards(uint card_in_region, bool within_howl) {
uint8_t* data = nullptr; uint8_t* data = nullptr;
CardSetPtr new_card_set; ContainerPtr new_container;
if (within_howl) { if (within_howl) {
uint const size_in_bits = _config->max_cards_in_howl_bitmap(); uint const size_in_bits = _config->max_cards_in_howl_bitmap();
uint card_offset = _config->howl_bitmap_offset(card_in_region); uint container_offset = _config->howl_bitmap_offset(card_in_region);
data = allocate_mem_object(CardSetBitMap); data = allocate_mem_object(ContainerBitMap);
new (data) G1CardSetBitMap(card_offset, size_in_bits); new (data) G1CardSetBitMap(container_offset, size_in_bits);
new_card_set = make_card_set_ptr(data, CardSetBitMap); new_container = make_container_ptr(data, ContainerBitMap);
} else { } else {
data = allocate_mem_object(CardSetHowl); data = allocate_mem_object(ContainerHowl);
new (data) G1CardSetHowl(card_in_region, _config); new (data) G1CardSetHowl(card_in_region, _config);
new_card_set = make_card_set_ptr(data, CardSetHowl); new_container = make_container_ptr(data, ContainerHowl);
} }
return new_card_set; return new_container;
} }
bool G1CardSet::coarsen_card_set(volatile CardSetPtr* card_set_addr, bool G1CardSet::coarsen_container(ContainerPtr volatile* container_addr,
CardSetPtr cur_card_set, ContainerPtr cur_container,
uint card_in_region, uint card_in_region,
bool within_howl) { bool within_howl) {
CardSetPtr new_card_set = nullptr; ContainerPtr new_container = nullptr;
switch (card_set_type(cur_card_set)) { switch (container_type(cur_container)) {
case CardSetArrayOfCards : { case ContainerArrayOfCards: {
new_card_set = create_coarsened_array_of_cards(card_in_region, within_howl); new_container = create_coarsened_array_of_cards(card_in_region, within_howl);
break; break;
} }
case CardSetBitMap: { case ContainerBitMap: {
new_card_set = FullCardSet; new_container = FullCardSet;
break; break;
} }
case CardSetInlinePtr: { case ContainerInlinePtr: {
uint const size = _config->max_cards_in_array(); uint const size = _config->max_cards_in_array();
uint8_t* data = allocate_mem_object(CardSetArrayOfCards); uint8_t* data = allocate_mem_object(ContainerArrayOfCards);
new (data) G1CardSetArray(card_in_region, size); new (data) G1CardSetArray(card_in_region, size);
new_card_set = make_card_set_ptr(data, CardSetArrayOfCards); new_container = make_container_ptr(data, ContainerArrayOfCards);
break; break;
} }
case CardSetHowl: { case ContainerHowl: {
new_card_set = FullCardSet; // anything will do at this point. new_container = FullCardSet; // anything will do at this point.
break; break;
} }
default: default:
ShouldNotReachHere(); ShouldNotReachHere();
} }
CardSetPtr old_value = Atomic::cmpxchg(card_set_addr, cur_card_set, new_card_set); // Memory order? ContainerPtr old_value = Atomic::cmpxchg(container_addr, cur_container, new_container); // Memory order?
if (old_value == cur_card_set) { if (old_value == cur_container) {
// Success. Indicate that the cards from the current card set must be transferred // Success. Indicate that the cards from the current card set must be transferred
// by this caller. // by this caller.
// Release the hash table reference to the card. The caller still holds the // Release the hash table reference to the card. The caller still holds the
// reference to this card set, so it can never be released (and we do not need to // reference to this card set, so it can never be released (and we do not need to
// check its result). // check its result).
bool should_free = release_card_set(cur_card_set); bool should_free = release_container(cur_container);
assert(!should_free, "must have had more than one reference"); assert(!should_free, "must have had more than one reference");
// Free containers if cur_card_set is CardSetHowl // Free containers if cur_container is ContainerHowl
if (card_set_type(cur_card_set) == CardSetHowl) { if (container_type(cur_container) == ContainerHowl) {
G1ReleaseCardsets rel(this); G1ReleaseCardsets rel(this);
card_set_ptr<G1CardSetHowl>(cur_card_set)->iterate(rel, _config->num_buckets_in_howl()); container_ptr<G1CardSetHowl>(cur_container)->iterate(rel, _config->num_buckets_in_howl());
} }
return true; return true;
} else { } else {
// Somebody else beat us to coarsening that card set. Exit, but clean up first. // Somebody else beat us to coarsening that card set. Exit, but clean up first.
if (new_card_set != FullCardSet) { if (new_container != FullCardSet) {
assert(new_card_set != nullptr, "must not be"); assert(new_container != nullptr, "must not be");
release_and_must_free_card_set(new_card_set); release_and_must_free_container(new_container);
} }
return false; return false;
} }
@ -591,34 +585,34 @@ public:
} }
}; };
void G1CardSet::transfer_cards(G1CardSetHashTableValue* table_entry, CardSetPtr source_card_set, uint card_region) { void G1CardSet::transfer_cards(G1CardSetHashTableValue* table_entry, ContainerPtr source_container, uint card_region) {
assert(source_card_set != FullCardSet, "Should not need to transfer from full"); assert(source_container != FullCardSet, "Should not need to transfer from FullCardSet");
// Need to transfer old entries unless there is a Full card set in place now, i.e. // Need to transfer old entries unless there is a Full card set container in place now, i.e.
// the old type has been CardSetBitMap. "Full" contains all elements anyway. // the old type has been ContainerBitMap. "Full" contains all elements anyway.
if (card_set_type(source_card_set) != CardSetHowl) { if (container_type(source_container) != ContainerHowl) {
G1TransferCard iter(this, card_region); G1TransferCard iter(this, card_region);
iterate_cards_during_transfer(source_card_set, iter); iterate_cards_during_transfer(source_container, iter);
} else { } else {
assert(card_set_type(source_card_set) == CardSetHowl, "must be"); assert(container_type(source_container) == ContainerHowl, "must be");
// Need to correct for that the Full remembered set occupies more cards than the // Need to correct for that the Full remembered set occupies more cards than the
// AoCS before. // AoCS before.
Atomic::add(&_num_occupied, _config->max_cards_in_region() - table_entry->_num_occupied, memory_order_relaxed); Atomic::add(&_num_occupied, _config->max_cards_in_region() - table_entry->_num_occupied, memory_order_relaxed);
} }
} }
void G1CardSet::transfer_cards_in_howl(CardSetPtr parent_card_set, void G1CardSet::transfer_cards_in_howl(ContainerPtr parent_container,
CardSetPtr source_card_set, ContainerPtr source_container,
uint card_region) { uint card_region) {
assert(card_set_type(parent_card_set) == CardSetHowl, "must be"); assert(container_type(parent_container) == ContainerHowl, "must be");
assert(source_card_set != FullCardSet, "Should not need to transfer from full"); assert(source_container != FullCardSet, "Should not need to transfer from full");
// Need to transfer old entries unless there is a Full card set in place now, i.e. // Need to transfer old entries unless there is a Full card set in place now, i.e.
// the old type has been CardSetBitMap. // the old type has been ContainerBitMap.
if (card_set_type(source_card_set) != CardSetBitMap) { if (container_type(source_container) != ContainerBitMap) {
// We only need to transfer from anything below CardSetBitMap. // We only need to transfer from anything below ContainerBitMap.
G1TransferCard iter(this, card_region); G1TransferCard iter(this, card_region);
iterate_cards_during_transfer(source_card_set, iter); iterate_cards_during_transfer(source_container, iter);
} else { } else {
uint diff = _config->max_cards_in_howl_bitmap() - card_set_ptr<G1CardSetBitMap>(source_card_set)->num_bits_set(); uint diff = _config->max_cards_in_howl_bitmap() - container_ptr<G1CardSetBitMap>(source_container)->num_bits_set();
// Need to correct for that the Full remembered set occupies more cards than the // Need to correct for that the Full remembered set occupies more cards than the
// bitmap before. // bitmap before.
@ -627,10 +621,10 @@ void G1CardSet::transfer_cards_in_howl(CardSetPtr parent_card_set,
// G1CardSet::add_to_howl after coarsening. // G1CardSet::add_to_howl after coarsening.
diff -= 1; diff -= 1;
G1CardSetHowl* howling_array = card_set_ptr<G1CardSetHowl>(parent_card_set); G1CardSetHowl* howling_array = container_ptr<G1CardSetHowl>(parent_container);
Atomic::add(&howling_array->_num_entries, diff, memory_order_relaxed); Atomic::add(&howling_array->_num_entries, diff, memory_order_relaxed);
G1CardSetHashTableValue* table_entry = get_card_set(card_region); G1CardSetHashTableValue* table_entry = get_container(card_region);
assert(table_entry != nullptr, "Table entry not found for transferred cards"); assert(table_entry != nullptr, "Table entry not found for transferred cards");
Atomic::add(&table_entry->_num_occupied, diff, memory_order_relaxed); Atomic::add(&table_entry->_num_occupied, diff, memory_order_relaxed);
@ -639,72 +633,75 @@ void G1CardSet::transfer_cards_in_howl(CardSetPtr parent_card_set,
} }
} }
G1AddCardResult G1CardSet::add_to_card_set(volatile CardSetPtr* card_set_addr, CardSetPtr card_set, uint card_region, uint card_in_region, bool increment_total) { G1AddCardResult G1CardSet::add_to_container(ContainerPtr volatile* container_addr,
assert(card_set_addr != nullptr, "Cannot add to empty cardset"); ContainerPtr container,
uint card_region,
uint card_in_region,
bool increment_total) {
assert(container_addr != nullptr, "must be");
G1AddCardResult add_result; G1AddCardResult add_result;
switch (card_set_type(card_set)) { switch (container_type(container)) {
case CardSetInlinePtr: { case ContainerInlinePtr: {
add_result = add_to_inline_ptr(card_set_addr, card_set, card_in_region); add_result = add_to_inline_ptr(container_addr, container, card_in_region);
break; break;
} }
case CardSetArrayOfCards : { case ContainerArrayOfCards: {
add_result = add_to_array(card_set, card_in_region); add_result = add_to_array(container, card_in_region);
break; break;
} }
case CardSetBitMap: { case ContainerBitMap: {
add_result = add_to_bitmap(card_set, card_in_region); add_result = add_to_bitmap(container, card_in_region);
break; break;
} }
case CardSetHowl: { case ContainerHowl: {
assert(CardSetHowl == card_set_type(FullCardSet), "must be"); assert(ContainerHowl == container_type(FullCardSet), "must be");
if (card_set == FullCardSet) { if (container == FullCardSet) {
return Found; return Found;
} }
add_result = add_to_howl(card_set, card_region, card_in_region, increment_total); add_result = add_to_howl(container, card_region, card_in_region, increment_total);
break; break;
} }
default: default:
ShouldNotReachHere(); ShouldNotReachHere();
} }
return add_result; return add_result;
} }
G1CardSetHashTableValue* G1CardSet::get_or_add_card_set(uint card_region, bool* should_grow_table) { G1CardSetHashTableValue* G1CardSet::get_or_add_container(uint card_region, bool* should_grow_table) {
return _table->get_or_add(card_region, should_grow_table); return _table->get_or_add(card_region, should_grow_table);
} }
G1CardSetHashTableValue* G1CardSet::get_card_set(uint card_region) { G1CardSetHashTableValue* G1CardSet::get_container(uint card_region) {
return _table->get(card_region); return _table->get(card_region);
} }
G1AddCardResult G1CardSet::add_card(uint card_region, uint card_in_region, bool increment_total) { G1AddCardResult G1CardSet::add_card(uint card_region, uint card_in_region, bool increment_total) {
G1AddCardResult add_result; G1AddCardResult add_result;
CardSetPtr to_transfer = nullptr; ContainerPtr to_transfer = nullptr;
CardSetPtr card_set; ContainerPtr container;
bool should_grow_table = false; bool should_grow_table = false;
G1CardSetHashTableValue* table_entry = get_or_add_card_set(card_region, &should_grow_table); G1CardSetHashTableValue* table_entry = get_or_add_container(card_region, &should_grow_table);
while (true) { while (true) {
card_set = acquire_card_set(&table_entry->_card_set); container = acquire_container(&table_entry->_container);
add_result = add_to_card_set(&table_entry->_card_set, card_set, card_region, card_in_region, increment_total); add_result = add_to_container(&table_entry->_container, container, card_region, card_in_region, increment_total);
if (add_result != Overflow) { if (add_result != Overflow) {
break; break;
} }
// Card set has overflown. Coarsen or retry. // Card set has overflown. Coarsen or retry.
bool coarsened = coarsen_card_set(&table_entry->_card_set, card_set, card_in_region); bool coarsened = coarsen_container(&table_entry->_container, container, card_in_region);
_coarsen_stats.record_coarsening(card_set_type(card_set), !coarsened); _coarsen_stats.record_coarsening(container_type(container), !coarsened);
if (coarsened) { if (coarsened) {
// We have been the one coarsening this card set (and in the process added that card). // We successful coarsened this card set container (and in the process added the card).
add_result = Added; add_result = Added;
to_transfer = card_set; to_transfer = container;
break; break;
} }
// Somebody else beat us to coarsening. Retry. // Somebody else beat us to coarsening. Retry.
release_and_maybe_free_card_set(card_set); release_and_maybe_free_container(container);
} }
if (increment_total && add_result == Added) { if (increment_total && add_result == Added) {
@ -718,7 +715,7 @@ G1AddCardResult G1CardSet::add_card(uint card_region, uint card_in_region, bool
transfer_cards(table_entry, to_transfer, card_region); transfer_cards(table_entry, to_transfer, card_region);
} }
release_and_maybe_free_card_set(card_set); release_and_maybe_free_container(container);
return add_result; return add_result;
} }
@ -727,29 +724,29 @@ bool G1CardSet::contains_card(uint card_region, uint card_in_region) {
assert(card_in_region < _config->max_cards_in_region(), assert(card_in_region < _config->max_cards_in_region(),
"Card %u is beyond max %u", card_in_region, _config->max_cards_in_region()); "Card %u is beyond max %u", card_in_region, _config->max_cards_in_region());
// Protect the card set from reclamation. // Protect the card set container from reclamation.
GlobalCounter::CriticalSection cs(Thread::current()); GlobalCounter::CriticalSection cs(Thread::current());
G1CardSetHashTableValue* table_entry = get_card_set(card_region); G1CardSetHashTableValue* table_entry = get_container(card_region);
if (table_entry == nullptr) { if (table_entry == nullptr) {
return false; return false;
} }
CardSetPtr card_set = table_entry->_card_set; ContainerPtr container = table_entry->_container;
if (card_set == FullCardSet) { if (container == FullCardSet) {
// contains_card() is not a performance critical method so we do not hide that // contains_card() is not a performance critical method so we do not hide that
// case in the switch below. // case in the switch below.
return true; return true;
} }
switch (card_set_type(card_set)) { switch (container_type(container)) {
case CardSetInlinePtr: { case ContainerInlinePtr: {
G1CardSetInlinePtr ptr(card_set); G1CardSetInlinePtr ptr(container);
return ptr.contains(card_in_region, _config->inline_ptr_bits_per_card()); return ptr.contains(card_in_region, _config->inline_ptr_bits_per_card());
} }
case CardSetArrayOfCards : return card_set_ptr<G1CardSetArray>(card_set)->contains(card_in_region); case ContainerArrayOfCards: return container_ptr<G1CardSetArray>(container)->contains(card_in_region);
case CardSetBitMap: return card_set_ptr<G1CardSetBitMap>(card_set)->contains(card_in_region, _config->max_cards_in_howl_bitmap()); case ContainerBitMap: return container_ptr<G1CardSetBitMap>(container)->contains(card_in_region, _config->max_cards_in_howl_bitmap());
case CardSetHowl: { case ContainerHowl: {
G1CardSetHowl* howling_array = card_set_ptr<G1CardSetHowl>(card_set); G1CardSetHowl* howling_array = container_ptr<G1CardSetHowl>(container);
return howling_array->contains(card_in_region, _config); return howling_array->contains(card_in_region, _config);
} }
@ -759,53 +756,53 @@ bool G1CardSet::contains_card(uint card_region, uint card_in_region) {
} }
void G1CardSet::print_info(outputStream* st, uint card_region, uint card_in_region) { void G1CardSet::print_info(outputStream* st, uint card_region, uint card_in_region) {
G1CardSetHashTableValue* table_entry = get_card_set(card_region); G1CardSetHashTableValue* table_entry = get_container(card_region);
if (table_entry == nullptr) { if (table_entry == nullptr) {
st->print("NULL card set"); st->print("NULL card set");
return; return;
} }
CardSetPtr card_set = table_entry->_card_set; ContainerPtr container = table_entry->_container;
if (card_set == FullCardSet) { if (container == FullCardSet) {
st->print("FULL card set)"); st->print("FULL card set)");
return; return;
} }
switch (card_set_type(card_set)) { switch (container_type(container)) {
case CardSetInlinePtr: { case ContainerInlinePtr: {
st->print("InlinePtr not containing %u", card_in_region); st->print("InlinePtr not containing %u", card_in_region);
break; break;
} }
case CardSetArrayOfCards : { case ContainerArrayOfCards: {
st->print("AoC not containing %u", card_in_region); st->print("AoC not containing %u", card_in_region);
break; break;
} }
case CardSetBitMap: { case ContainerBitMap: {
st->print("BitMap not containing %u", card_in_region); st->print("BitMap not containing %u", card_in_region);
break; break;
} }
case CardSetHowl: { case ContainerHowl: {
st->print("CardSetHowl not containing %u", card_in_region); st->print("ContainerHowl not containing %u", card_in_region);
break; break;
} }
default: st->print("Unknown card set type %u", card_set_type(card_set)); ShouldNotReachHere(); break; default: st->print("Unknown card set container type %u", container_type(container)); ShouldNotReachHere(); break;
} }
} }
template <class CardVisitor> template <class CardVisitor>
void G1CardSet::iterate_cards_during_transfer(CardSetPtr const card_set, CardVisitor& cl) { void G1CardSet::iterate_cards_during_transfer(ContainerPtr const container, CardVisitor& cl) {
uint type = card_set_type(card_set); uint type = container_type(container);
assert(type == CardSetInlinePtr || type == CardSetArrayOfCards, assert(type == ContainerInlinePtr || type == ContainerArrayOfCards,
"invalid card set type %d to transfer from", "invalid card set type %d to transfer from",
card_set_type(card_set)); container_type(container));
switch (type) { switch (type) {
case CardSetInlinePtr: { case ContainerInlinePtr: {
G1CardSetInlinePtr ptr(card_set); G1CardSetInlinePtr ptr(container);
ptr.iterate(cl, _config->inline_ptr_bits_per_card()); ptr.iterate(cl, _config->inline_ptr_bits_per_card());
return; return;
} }
case CardSetArrayOfCards : { case ContainerArrayOfCards: {
card_set_ptr<G1CardSetArray>(card_set)->iterate(cl); container_ptr<G1CardSetArray>(container)->iterate(cl);
return; return;
} }
default: default:
@ -813,7 +810,7 @@ void G1CardSet::iterate_cards_during_transfer(CardSetPtr const card_set, CardVis
} }
} }
void G1CardSet::iterate_containers(CardSetPtrClosure* cl, bool at_safepoint) { void G1CardSet::iterate_containers(ContainerPtrClosure* cl, bool at_safepoint) {
if (at_safepoint) { if (at_safepoint) {
_table->iterate_safepoint(cl); _table->iterate_safepoint(cl);
} else { } else {
@ -844,7 +841,7 @@ public:
}; };
template <typename Closure, template <typename> class CardOrRanges> template <typename Closure, template <typename> class CardOrRanges>
class G1CardSetContainersClosure : public G1CardSet::CardSetPtrClosure { class G1CardSetContainersClosure : public G1CardSet::ContainerPtrClosure {
G1CardSet* _card_set; G1CardSet* _card_set;
Closure& _cl; Closure& _cl;
@ -855,9 +852,9 @@ public:
_card_set(card_set), _card_set(card_set),
_cl(cl) { } _cl(cl) { }
void do_cardsetptr(uint region_idx, size_t num_occupied, G1CardSet::CardSetPtr card_set) override { void do_containerptr(uint region_idx, size_t num_occupied, G1CardSet::ContainerPtr container) override {
CardOrRanges<Closure> cl(_cl, region_idx); CardOrRanges<Closure> cl(_cl, region_idx);
_card_set->iterate_cards_or_ranges_in_container(card_set, cl); _card_set->iterate_cards_or_ranges_in_container(container, cl);
} }
}; };
@ -879,13 +876,13 @@ size_t G1CardSet::occupied() const {
} }
size_t G1CardSet::num_containers() { size_t G1CardSet::num_containers() {
class GetNumberOfContainers : public CardSetPtrClosure { class GetNumberOfContainers : public ContainerPtrClosure {
public: public:
size_t _count; size_t _count;
GetNumberOfContainers() : CardSetPtrClosure(), _count(0) { } GetNumberOfContainers() : ContainerPtrClosure(), _count(0) { }
void do_cardsetptr(uint region_idx, size_t num_occupied, CardSetPtr card_set) override { void do_containerptr(uint region_idx, size_t num_occupied, ContainerPtr container) override {
_count++; _count++;
} }
} cl; } cl;

133
src/hotspot/share/gc/g1/g1CardSet.hpp
View File

@ -26,10 +26,7 @@
#define SHARE_GC_G1_G1CARDSET_HPP #define SHARE_GC_G1_G1CARDSET_HPP
#include "memory/allocation.hpp" #include "memory/allocation.hpp"
#include "memory/padded.hpp"
#include "oops/oopsHierarchy.hpp"
#include "utilities/concurrentHashTable.hpp" #include "utilities/concurrentHashTable.hpp"
#include "utilities/lockFreeStack.hpp"
class G1CardSetAllocOptions; class G1CardSetAllocOptions;
class G1CardSetHashTable; class G1CardSetHashTable;
@ -147,10 +144,10 @@ public:
class G1CardSetCoarsenStats { class G1CardSetCoarsenStats {
public: public:
// Number of entries in the statistics tables: since we index with the source // Number of entries in the statistics tables: since we index with the source
// cardset of the coarsening, this is the total number of combinations of // container of the coarsening, this is the total number of combinations of
// card sets - 1. // card set containers - 1.
static constexpr size_t NumCoarsenCategories = 7; static constexpr size_t NumCoarsenCategories = 7;
// Coarsening statistics for the possible CardSetPtr in the Howl card set // Coarsening statistics for the possible ContainerPtr in the Howl card set
// start from this offset. // start from this offset.
static constexpr size_t CoarsenHowlOffset = 4; static constexpr size_t CoarsenHowlOffset = 4;
@ -173,14 +170,14 @@ public:
void print_on(outputStream* out); void print_on(outputStream* out);
}; };
// Sparse set of card indexes comprising a remembered set on the Java heap. Card // Set of card indexes comprising a remembered set on the Java heap. Card
// size is assumed to be card table card size. // size is assumed to be card table card size.
// //
// Technically it is implemented using a ConcurrentHashTable that stores a card // Technically it is implemented using a ConcurrentHashTable that stores a card
// set container for every region containing at least one card. // set container for every region containing at least one card.
// //
// There are in total five different containers, encoded in the ConcurrentHashTable // There are in total five different containers, encoded in the ConcurrentHashTable
// node as CardSetPtr. A CardSetPtr may cover the whole region or just a part of // node as ContainerPtr. A ContainerPtr may cover the whole region or just a part of
// it. // it.
// See its description below for more information. // See its description below for more information.
class G1CardSet : public CHeapObj<mtGCCardSet> { class G1CardSet : public CHeapObj<mtGCCardSet> {
@ -194,46 +191,46 @@ class G1CardSet : public CHeapObj<mtGCCardSet> {
static G1CardSetCoarsenStats _coarsen_stats; // Coarsening statistics since VM start. static G1CardSetCoarsenStats _coarsen_stats; // Coarsening statistics since VM start.
static G1CardSetCoarsenStats _last_coarsen_stats; // Coarsening statistics at last GC. static G1CardSetCoarsenStats _last_coarsen_stats; // Coarsening statistics at last GC.
public: public:
// Two lower bits are used to encode the card storage types // Two lower bits are used to encode the card set container types
static const uintptr_t CardSetPtrHeaderSize = 2; static const uintptr_t ContainerPtrHeaderSize = 2;
// CardSetPtr represents the card storage type of a given covered area. It encodes // ContainerPtr represents the card set container type of a given covered area.
// a type in the LSBs, in addition to having a few significant values. // It encodes a type in the LSBs, in addition to having a few significant values.
// //
// Possible encodings: // Possible encodings:
// //
// 0...00000 free (Empty, should never happen) // 0...00000 free (Empty, should never happen)
// 1...11111 full All card indexes in the whole area this CardSetPtr covers are part of this container. // 1...11111 full All card indexes in the whole area this ContainerPtr covers are part of this container.
// X...XXX00 inline-ptr-cards A handful of card indexes covered by this CardSetPtr are encoded within the CardSetPtr. // X...XXX00 inline-ptr-cards A handful of card indexes covered by this ContainerPtr are encoded within the ContainerPtr.
// X...XXX01 array of cards The container is a contiguous array of card indexes. // X...XXX01 array of cards The container is a contiguous array of card indexes.
// X...XXX10 bitmap The container uses a bitmap to determine whether a given index is part of this set. // X...XXX10 bitmap The container uses a bitmap to determine whether a given index is part of this set.
// X...XXX11 howl This is a card set container containing an array of CardSetPtr, with each CardSetPtr // X...XXX11 howl This is a card set container containing an array of ContainerPtr, with each ContainerPtr
// limited to a sub-range of the original range. Currently only one level of this // limited to a sub-range of the original range. Currently only one level of this
// container is supported. // container is supported.
typedef void* CardSetPtr; using ContainerPtr = void*;
// Coarsening happens in the order below: // Coarsening happens in the order below:
// CardSetInlinePtr -> CardSetArrayOfCards -> CardSetHowl -> Full // ContainerInlinePtr -> ContainerArrayOfCards -> ContainerHowl -> Full
// Corsening of containers inside the CardSetHowl happens in the order: // Corsening of containers inside the ContainerHowl happens in the order:
// CardSetInlinePtr -> CardSetArrayOfCards -> CardSetBitMap -> Full // ContainerInlinePtr -> ContainerArrayOfCards -> ContainerBitMap -> Full
static const uintptr_t CardSetInlinePtr = 0x0; static const uintptr_t ContainerInlinePtr = 0x0;
static const uintptr_t CardSetArrayOfCards = 0x1; static const uintptr_t ContainerArrayOfCards = 0x1;
static const uintptr_t CardSetBitMap = 0x2; static const uintptr_t ContainerBitMap = 0x2;
static const uintptr_t CardSetHowl = 0x3; static const uintptr_t ContainerHowl = 0x3;
// The special sentinel values // The special sentinel values
static constexpr CardSetPtr FreeCardSet = nullptr; static constexpr ContainerPtr FreeCardSet = nullptr;
// Unfortunately we can't make (G1CardSet::CardSetPtr)-1 constexpr because // Unfortunately we can't make (G1CardSet::ContainerPtr)-1 constexpr because
// reinterpret_casts are forbidden in constexprs. Use a regular static instead. // reinterpret_casts are forbidden in constexprs. Use a regular static instead.
static CardSetPtr FullCardSet; static ContainerPtr FullCardSet;
static const uintptr_t CardSetPtrTypeMask = ((uintptr_t)1 << CardSetPtrHeaderSize) - 1; static const uintptr_t ContainerPtrTypeMask = ((uintptr_t)1 << ContainerPtrHeaderSize) - 1;
static CardSetPtr strip_card_set_type(CardSetPtr ptr) { return (CardSetPtr)((uintptr_t)ptr & ~CardSetPtrTypeMask); } static ContainerPtr strip_container_type(ContainerPtr ptr) { return (ContainerPtr)((uintptr_t)ptr & ~ContainerPtrTypeMask); }
static uint card_set_type(CardSetPtr ptr) { return (uintptr_t)ptr & CardSetPtrTypeMask; } static uint container_type(ContainerPtr ptr) { return (uintptr_t)ptr & ContainerPtrTypeMask; }
template <class T> template <class T>
static T* card_set_ptr(CardSetPtr ptr); static T* container_ptr(ContainerPtr ptr);
private: private:
G1CardSetMemoryManager* _mm; G1CardSetMemoryManager* _mm;
@ -245,42 +242,42 @@ private:
// be (slightly) more cards in the card set than this value in reality. // be (slightly) more cards in the card set than this value in reality.
size_t _num_occupied; size_t _num_occupied;
CardSetPtr make_card_set_ptr(void* value, uintptr_t type); ContainerPtr make_container_ptr(void* value, uintptr_t type);
CardSetPtr acquire_card_set(CardSetPtr volatile* card_set_addr); ContainerPtr acquire_container(ContainerPtr volatile* container_addr);
// Returns true if the card set should be released // Returns true if the card set container should be released
bool release_card_set(CardSetPtr card_set); bool release_container(ContainerPtr container);
// Release card set and free if needed. // Release card set and free if needed.
void release_and_maybe_free_card_set(CardSetPtr card_set); void release_and_maybe_free_container(ContainerPtr container);
// Release card set and free (and it must be freeable). // Release card set and free (and it must be freeable).
void release_and_must_free_card_set(CardSetPtr card_set); void release_and_must_free_container(ContainerPtr container);
// Coarsens the CardSet cur_card_set to the next level; tries to replace the // Coarsens the card set container cur_container to the next level; tries to replace the
// previous CardSet with a new one which includes the given card_in_region. // previous ContainerPtr with a new one which includes the given card_in_region.
// coarsen_card_set does not transfer cards from cur_card_set // coarsen_container does not transfer cards from cur_container
// to the new card_set. Transfer is achieved by transfer_cards. // to the new container. Transfer is achieved by transfer_cards.
// Returns true if this was the thread that coarsened the CardSet (and added the card). // Returns true if this was the thread that coarsened the container (and added the card).
bool coarsen_card_set(CardSetPtr volatile* card_set_addr, bool coarsen_container(ContainerPtr volatile* container_addr,
CardSetPtr cur_card_set, ContainerPtr cur_container,
uint card_in_region, bool within_howl = false); uint card_in_region, bool within_howl = false);
CardSetPtr create_coarsened_array_of_cards(uint card_in_region, bool within_howl); ContainerPtr create_coarsened_array_of_cards(uint card_in_region, bool within_howl);
// Transfer entries from source_card_set to a recently installed coarser storage type // Transfer entries from source_card_set to a recently installed coarser storage type
// We only need to transfer anything finer than CardSetBitMap. "Full" contains // We only need to transfer anything finer than ContainerBitMap. "Full" contains
// all elements anyway. // all elements anyway.
void transfer_cards(G1CardSetHashTableValue* table_entry, CardSetPtr source_card_set, uint card_region); void transfer_cards(G1CardSetHashTableValue* table_entry, ContainerPtr source_container, uint card_region);
void transfer_cards_in_howl(CardSetPtr parent_card_set, CardSetPtr source_card_set, uint card_region); void transfer_cards_in_howl(ContainerPtr parent_container, ContainerPtr source_container, uint card_region);
G1AddCardResult add_to_card_set(CardSetPtr volatile* card_set_addr, CardSetPtr card_set, uint card_region, uint card, bool increment_total = true); G1AddCardResult add_to_container(ContainerPtr volatile* container_addr, ContainerPtr container, uint card_region, uint card, bool increment_total = true);
G1AddCardResult add_to_inline_ptr(CardSetPtr volatile* card_set_addr, CardSetPtr card_set, uint card_in_region); G1AddCardResult add_to_inline_ptr(ContainerPtr volatile* container_addr, ContainerPtr container, uint card_in_region);
G1AddCardResult add_to_array(CardSetPtr card_set, uint card_in_region); G1AddCardResult add_to_array(ContainerPtr container, uint card_in_region);
G1AddCardResult add_to_bitmap(CardSetPtr card_set, uint card_in_region); G1AddCardResult add_to_bitmap(ContainerPtr container, uint card_in_region);
G1AddCardResult add_to_howl(CardSetPtr parent_card_set, uint card_region, uint card_in_region, bool increment_total = true); G1AddCardResult add_to_howl(ContainerPtr parent_container, uint card_region, uint card_in_region, bool increment_total = true);
G1CardSetHashTableValue* get_or_add_card_set(uint card_region, bool* should_grow_table); G1CardSetHashTableValue* get_or_add_container(uint card_region, bool* should_grow_table);
G1CardSetHashTableValue* get_card_set(uint card_region); G1CardSetHashTableValue* get_container(uint card_region);
// Iterate over cards of a card set container during transfer of the cards from // Iterate over cards of a card set container during transfer of the cards from
// one container to another. Executes // one container to another. Executes
@ -289,11 +286,11 @@ private:
// //
// on the given class. // on the given class.
template <class CardVisitor> template <class CardVisitor>
void iterate_cards_during_transfer(CardSetPtr const card_set, CardVisitor& vl); void iterate_cards_during_transfer(ContainerPtr const container, CardVisitor& vl);
uint card_set_type_to_mem_object_type(uintptr_t type) const; uint container_type_to_mem_object_type(uintptr_t type) const;
uint8_t* allocate_mem_object(uintptr_t type); uint8_t* allocate_mem_object(uintptr_t type);
void free_mem_object(CardSetPtr card_set); void free_mem_object(ContainerPtr container);
public: public:
G1CardSetConfiguration* config() const { return _config; } G1CardSetConfiguration* config() const { return _config; }
@ -302,8 +299,8 @@ public:
G1CardSet(G1CardSetConfiguration* config, G1CardSetMemoryManager* mm); G1CardSet(G1CardSetConfiguration* config, G1CardSetMemoryManager* mm);
virtual ~G1CardSet(); virtual ~G1CardSet();
// Adds the given card to this set, returning an appropriate result. If added, // Adds the given card to this set, returning an appropriate result.
// updates the total count. // If incremental_count is true and the card has been added, updates the total count.
G1AddCardResult add_card(uint card_region, uint card_in_region, bool increment_total = true); G1AddCardResult add_card(uint card_region, uint card_in_region, bool increment_total = true);
bool contains_card(uint card_region, uint card_in_region); bool contains_card(uint card_region, uint card_in_region);
@ -351,14 +348,14 @@ public:
// start_iterate(). // start_iterate().
// //
template <class CardOrRangeVisitor> template <class CardOrRangeVisitor>
void iterate_cards_or_ranges_in_container(CardSetPtr const card_set, CardOrRangeVisitor& cl); void iterate_cards_or_ranges_in_container(ContainerPtr const container, CardOrRangeVisitor& cl);
class CardSetPtrClosure { class ContainerPtrClosure {
public: public:
virtual void do_cardsetptr(uint region_idx, size_t num_occupied, CardSetPtr card_set) = 0; virtual void do_containerptr(uint region_idx, size_t num_occupied, ContainerPtr container) = 0;
}; };
void iterate_containers(CardSetPtrClosure* cl, bool safepoint = false); void iterate_containers(ContainerPtrClosure* cl, bool safepoint = false);
class CardClosure { class CardClosure {
public: public:
@ -370,13 +367,13 @@ public:
class G1CardSetHashTableValue { class G1CardSetHashTableValue {
public: public:
using CardSetPtr = G1CardSet::CardSetPtr; using ContainerPtr = G1CardSet::ContainerPtr;
const uint _region_idx; const uint _region_idx;
uint volatile _num_occupied; uint volatile _num_occupied;
CardSetPtr volatile _card_set; ContainerPtr volatile _container;
G1CardSetHashTableValue(uint region_idx, CardSetPtr card_set) : _region_idx(region_idx), _num_occupied(0), _card_set(card_set) { } G1CardSetHashTableValue(uint region_idx, ContainerPtr container) : _region_idx(region_idx), _num_occupied(0), _container(container) { }
}; };
class G1CardSetHashTableConfig : public StackObj { class G1CardSetHashTableConfig : public StackObj {
@ -391,6 +388,6 @@ public:
static void free_node(void* context, void* memory, Value const& value); static void free_node(void* context, void* memory, Value const& value);
}; };
typedef ConcurrentHashTable<G1CardSetHashTableConfig, mtGCCardSet> CardSetHash; using CardSetHash = ConcurrentHashTable<G1CardSetHashTableConfig, mtGCCardSet>;
#endif // SHARE_GC_G1_G1CARDSET_HPP #endif // SHARE_GC_G1_G1CARDSET_HPP

35
src/hotspot/share/gc/g1/g1CardSet.inline.hpp
View File

@ -28,55 +28,54 @@
#include "gc/g1/g1CardSet.hpp" #include "gc/g1/g1CardSet.hpp"
#include "gc/g1/g1CardSetContainers.inline.hpp" #include "gc/g1/g1CardSetContainers.inline.hpp"
#include "gc/g1/g1GCPhaseTimes.hpp" #include "gc/g1/g1GCPhaseTimes.hpp"
#include "runtime/atomic.hpp"
#include "logging/log.hpp" #include "logging/log.hpp"
template <class T> template <class T>
inline T* G1CardSet::card_set_ptr(CardSetPtr ptr) { inline T* G1CardSet::container_ptr(ContainerPtr ptr) {
return (T*)strip_card_set_type(ptr); return (T*)strip_container_type(ptr);
} }
inline G1CardSet::CardSetPtr G1CardSet::make_card_set_ptr(void* value, uintptr_t type) { inline G1CardSet::ContainerPtr G1CardSet::make_container_ptr(void* value, uintptr_t type) {
assert(card_set_type(value) == 0, "Given ptr " PTR_FORMAT " already has type bits set", p2i(value)); assert(container_type(value) == 0, "Given ptr " PTR_FORMAT " already has type bits set", p2i(value));
return (CardSetPtr)((uintptr_t)value | type); return (ContainerPtr)((uintptr_t)value | type);
} }
template <class CardOrRangeVisitor> template <class CardOrRangeVisitor>
inline void G1CardSet::iterate_cards_or_ranges_in_container(CardSetPtr const card_set, CardOrRangeVisitor& cl) { inline void G1CardSet::iterate_cards_or_ranges_in_container(ContainerPtr const container, CardOrRangeVisitor& cl) {
switch (card_set_type(card_set)) { switch (container_type(container)) {
case CardSetInlinePtr: { case ContainerInlinePtr: {
if (cl.start_iterate(G1GCPhaseTimes::MergeRSMergedInline)) { if (cl.start_iterate(G1GCPhaseTimes::MergeRSMergedInline)) {
G1CardSetInlinePtr ptr(card_set); G1CardSetInlinePtr ptr(container);
ptr.iterate(cl, _config->inline_ptr_bits_per_card()); ptr.iterate(cl, _config->inline_ptr_bits_per_card());
} }
return; return;
} }
case CardSetArrayOfCards : { case ContainerArrayOfCards: {
if (cl.start_iterate(G1GCPhaseTimes::MergeRSMergedArrayOfCards)) { if (cl.start_iterate(G1GCPhaseTimes::MergeRSMergedArrayOfCards)) {
card_set_ptr<G1CardSetArray>(card_set)->iterate(cl); container_ptr<G1CardSetArray>(container)->iterate(cl);
} }
return; return;
} }
case CardSetBitMap: { case ContainerBitMap: {
// There is no first-level bitmap spanning the whole area. // There is no first-level bitmap spanning the whole area.
ShouldNotReachHere(); ShouldNotReachHere();
return; return;
} }
case CardSetHowl: { case ContainerHowl: {
assert(card_set_type(FullCardSet) == CardSetHowl, "Must be"); assert(container_type(FullCardSet) == ContainerHowl, "Must be");
if (card_set == FullCardSet) { if (container == FullCardSet) {
if (cl.start_iterate(G1GCPhaseTimes::MergeRSMergedFull)) { if (cl.start_iterate(G1GCPhaseTimes::MergeRSMergedFull)) {
cl(0, _config->max_cards_in_region()); cl(0, _config->max_cards_in_region());
} }
return; return;
} }
if (cl.start_iterate(G1GCPhaseTimes::MergeRSMergedHowl)) { if (cl.start_iterate(G1GCPhaseTimes::MergeRSMergedHowl)) {
card_set_ptr<G1CardSetHowl>(card_set)->iterate(cl, _config); container_ptr<G1CardSetHowl>(container)->iterate(cl, _config);
} }
return; return;
} }
} }
log_error(gc)("Unkown card set type %u", card_set_type(card_set)); log_error(gc)("Unkown card set container type %u", container_type(container));
ShouldNotReachHere(); ShouldNotReachHere();
} }

58
src/hotspot/share/gc/g1/g1CardSetContainers.hpp
View File

@ -31,7 +31,7 @@
#include "utilities/bitMap.hpp" #include "utilities/bitMap.hpp"
#include "utilities/globalDefinitions.hpp" #include "utilities/globalDefinitions.hpp"
// A helper class to encode a few card indexes within a CardSetPtr. // A helper class to encode a few card indexes within a ContainerPtr.
// //
// The pointer value (either 32 or 64 bits) is split into two areas: // The pointer value (either 32 or 64 bits) is split into two areas:
// //
@ -65,16 +65,16 @@
class G1CardSetInlinePtr : public StackObj { class G1CardSetInlinePtr : public StackObj {
friend class G1CardSetContainersTest; friend class G1CardSetContainersTest;
typedef G1CardSet::CardSetPtr CardSetPtr; using ContainerPtr = G1CardSet::ContainerPtr;
CardSetPtr volatile * _value_addr; ContainerPtr volatile * _value_addr;
CardSetPtr _value; ContainerPtr _value;
static const uint SizeFieldLen = 3; static const uint SizeFieldLen = 3;
static const uint SizeFieldPos = 2; static const uint SizeFieldPos = 2;
static const uint HeaderSize = G1CardSet::CardSetPtrHeaderSize + SizeFieldLen; static const uint HeaderSize = G1CardSet::ContainerPtrHeaderSize + SizeFieldLen;
static const uint BitsInValue = sizeof(CardSetPtr) * BitsPerByte; static const uint BitsInValue = sizeof(ContainerPtr) * BitsPerByte;
static const uintptr_t SizeFieldMask = (((uint)1 << SizeFieldLen) - 1) << SizeFieldPos; static const uintptr_t SizeFieldMask = (((uint)1 << SizeFieldLen) - 1) << SizeFieldPos;
@ -82,9 +82,9 @@ class G1CardSetInlinePtr : public StackObj {
return (idx * bits_per_card + HeaderSize); return (idx * bits_per_card + HeaderSize);
} }
static CardSetPtr merge(CardSetPtr orig_value, uint card_in_region, uint idx, uint bits_per_card); static ContainerPtr merge(ContainerPtr orig_value, uint card_in_region, uint idx, uint bits_per_card);
static uint card_at(CardSetPtr value, uint const idx, uint const bits_per_card) { static uint card_at(ContainerPtr value, uint const idx, uint const bits_per_card) {
uint8_t card_pos = card_pos_for(idx, bits_per_card); uint8_t card_pos = card_pos_for(idx, bits_per_card);
uint result = ((uintptr_t)value >> card_pos) & (((uintptr_t)1 << bits_per_card) - 1); uint result = ((uintptr_t)value >> card_pos) & (((uintptr_t)1 << bits_per_card) - 1);
return result; return result;
@ -93,14 +93,14 @@ class G1CardSetInlinePtr : public StackObj {
uint find(uint const card_idx, uint const bits_per_card, uint start_at, uint num_cards); uint find(uint const card_idx, uint const bits_per_card, uint start_at, uint num_cards);
public: public:
G1CardSetInlinePtr() : _value_addr(nullptr), _value((CardSetPtr)G1CardSet::CardSetInlinePtr) { } G1CardSetInlinePtr() : _value_addr(nullptr), _value((ContainerPtr)G1CardSet::ContainerInlinePtr) { }
G1CardSetInlinePtr(CardSetPtr value) : _value_addr(nullptr), _value(value) { G1CardSetInlinePtr(ContainerPtr value) : _value_addr(nullptr), _value(value) {
assert(G1CardSet::card_set_type(_value) == G1CardSet::CardSetInlinePtr, "Value " PTR_FORMAT " is not a valid G1CardSetInPtr.", p2i(_value)); assert(G1CardSet::container_type(_value) == G1CardSet::ContainerInlinePtr, "Value " PTR_FORMAT " is not a valid G1CardSetInlinePtr.", p2i(_value));
} }
G1CardSetInlinePtr(CardSetPtr volatile* value_addr, CardSetPtr value) : _value_addr(value_addr), _value(value) { G1CardSetInlinePtr(ContainerPtr volatile* value_addr, ContainerPtr value) : _value_addr(value_addr), _value(value) {
assert(G1CardSet::card_set_type(_value) == G1CardSet::CardSetInlinePtr, "Value " PTR_FORMAT " is not a valid G1CardSetInPtr.", p2i(_value)); assert(G1CardSet::container_type(_value) == G1CardSet::ContainerInlinePtr, "Value " PTR_FORMAT " is not a valid G1CardSetInlinePtr.", p2i(_value));
} }
G1AddCardResult add(uint const card_idx, uint const bits_per_card, uint const max_cards_in_inline_ptr); G1AddCardResult add(uint const card_idx, uint const bits_per_card, uint const max_cards_in_inline_ptr);
@ -110,13 +110,13 @@ public:
template <class CardVisitor> template <class CardVisitor>
void iterate(CardVisitor& found, uint const bits_per_card); void iterate(CardVisitor& found, uint const bits_per_card);
operator CardSetPtr () { return _value; } operator ContainerPtr () { return _value; }
static uint max_cards_in_inline_ptr(uint bits_per_card) { static uint max_cards_in_inline_ptr(uint bits_per_card) {
return (BitsInValue - HeaderSize) / bits_per_card; return (BitsInValue - HeaderSize) / bits_per_card;
} }
static uint num_cards_in(CardSetPtr value) { static uint num_cards_in(ContainerPtr value) {
return ((uintptr_t)value & SizeFieldMask) >> SizeFieldPos; return ((uintptr_t)value & SizeFieldMask) >> SizeFieldPos;
} }
}; };
@ -139,7 +139,7 @@ public:
// which requires that we increment the reference counts by 2 starting at _ref_count = 3. // which requires that we increment the reference counts by 2 starting at _ref_count = 3.
// //
// All but inline pointers are of this kind. For those, card entries are stored // All but inline pointers are of this kind. For those, card entries are stored
// directly in the CardSetPtr of the ConcurrentHashTable node. // directly in the ContainerPtr of the ConcurrentHashTable node.
class G1CardSetContainer { class G1CardSetContainer {
uintptr_t _ref_count; uintptr_t _ref_count;
protected: protected:
@ -163,7 +163,7 @@ class G1CardSetArray : public G1CardSetContainer {
public: public:
typedef uint16_t EntryDataType; typedef uint16_t EntryDataType;
typedef uint EntryCountType; typedef uint EntryCountType;
using CardSetPtr = G1CardSet::CardSetPtr; using ContainerPtr = G1CardSet::ContainerPtr;
private: private:
EntryCountType _size; EntryCountType _size;
EntryCountType volatile _num_entries; EntryCountType volatile _num_entries;
@ -217,7 +217,7 @@ class G1CardSetBitMap : public G1CardSetContainer {
size_t _num_bits_set; size_t _num_bits_set;
BitMap::bm_word_t _bits[1]; BitMap::bm_word_t _bits[1];
using CardSetPtr = G1CardSet::CardSetPtr; using ContainerPtr = G1CardSet::ContainerPtr;
template<typename Derived> template<typename Derived>
static size_t header_size_in_bytes_internal() { static size_t header_size_in_bytes_internal() {
@ -252,10 +252,10 @@ public:
class G1CardSetHowl : public G1CardSetContainer { class G1CardSetHowl : public G1CardSetContainer {
public: public:
typedef uint EntryCountType; typedef uint EntryCountType;
using CardSetPtr = G1CardSet::CardSetPtr; using ContainerPtr = G1CardSet::ContainerPtr;
EntryCountType volatile _num_entries; EntryCountType volatile _num_entries;
private: private:
CardSetPtr _buckets[2]; ContainerPtr _buckets[2];
// Do not add class member variables beyond this point // Do not add class member variables beyond this point
template<typename Derived> template<typename Derived>
@ -263,32 +263,32 @@ private:
return offset_of(Derived, _buckets); return offset_of(Derived, _buckets);
} }
// Iterates over the given CardSetPtr with at index in this Howl card set, // Iterates over the given ContainerPtr with at index in this Howl card set,
// applying a CardOrRangeVisitor on it. // applying a CardOrRangeVisitor on it.
template <class CardOrRangeVisitor> template <class CardOrRangeVisitor>
void iterate_cardset(CardSetPtr const card_set, uint index, CardOrRangeVisitor& found, G1CardSetConfiguration* config); void iterate_cardset(ContainerPtr const container, uint index, CardOrRangeVisitor& found, G1CardSetConfiguration* config);
public: public:
G1CardSetHowl(EntryCountType card_in_region, G1CardSetConfiguration* config); G1CardSetHowl(EntryCountType card_in_region, G1CardSetConfiguration* config);
CardSetPtr* get_card_set_addr(EntryCountType index) { ContainerPtr* get_container_addr(EntryCountType index) {
return &_buckets[index]; return &_buckets[index];
} }
bool contains(uint card_idx, G1CardSetConfiguration* config); bool contains(uint card_idx, G1CardSetConfiguration* config);
// Iterates over all CardSetPtrs in this Howl card set, applying a CardOrRangeVisitor // Iterates over all ContainerPtrs in this Howl card set, applying a CardOrRangeVisitor
// on it. // on it.
template <class CardOrRangeVisitor> template <class CardOrRangeVisitor>
void iterate(CardOrRangeVisitor& found, G1CardSetConfiguration* config); void iterate(CardOrRangeVisitor& found, G1CardSetConfiguration* config);
// Iterates over all CardSetPtrs in this Howl card set. Calls // Iterates over all ContainerPtrs in this Howl card set. Calls
// //
// void operator ()(CardSetPtr* card_set_addr); // void operator ()(ContainerPtr* card_set_addr);
// //
// on all of them. // on all of them.
template <class CardSetPtrVisitor> template <class ContainerPtrVisitor>
void iterate(CardSetPtrVisitor& found, uint num_card_sets); void iterate(ContainerPtrVisitor& found, uint num_card_sets);
static EntryCountType num_buckets(size_t size_in_bits, size_t num_cards_in_array, size_t max_buckets); static EntryCountType num_buckets(size_t size_in_bits, size_t num_cards_in_array, size_t max_buckets);
@ -300,7 +300,7 @@ public:
static size_t header_size_in_bytes() { return header_size_in_bytes_internal<G1CardSetHowl>(); } static size_t header_size_in_bytes() { return header_size_in_bytes_internal<G1CardSetHowl>(); }
static size_t size_in_bytes(size_t num_arrays) { static size_t size_in_bytes(size_t num_arrays) {
return header_size_in_bytes() + sizeof(CardSetPtr) * num_arrays; return header_size_in_bytes() + sizeof(ContainerPtr) * num_arrays;
} }
}; };

56
src/hotspot/share/gc/g1/g1CardSetContainers.inline.hpp
View File

@ -31,7 +31,7 @@
#include "utilities/globalDefinitions.hpp" #include "utilities/globalDefinitions.hpp"
#include "utilities/spinYield.hpp" #include "utilities/spinYield.hpp"
inline G1CardSetInlinePtr::CardSetPtr G1CardSetInlinePtr::merge(CardSetPtr orig_value, uint card_in_region, uint idx, uint bits_per_card) { inline G1CardSetInlinePtr::ContainerPtr G1CardSetInlinePtr::merge(ContainerPtr orig_value, uint card_in_region, uint idx, uint bits_per_card) {
assert((idx & (SizeFieldMask >> SizeFieldPos)) == idx, "Index %u too large to fit into size field", idx); assert((idx & (SizeFieldMask >> SizeFieldPos)) == idx, "Index %u too large to fit into size field", idx);
assert(card_in_region < ((uint)1 << bits_per_card), "Card %u too large to fit into card value field", card_in_region); assert(card_in_region < ((uint)1 << bits_per_card), "Card %u too large to fit into card value field", card_in_region);
@ -44,7 +44,7 @@ inline G1CardSetInlinePtr::CardSetPtr G1CardSetInlinePtr::merge(CardSetPtr orig_
uintptr_t value = ((uintptr_t)(idx + 1) << SizeFieldPos) | ((uintptr_t)card_in_region << card_pos); uintptr_t value = ((uintptr_t)(idx + 1) << SizeFieldPos) | ((uintptr_t)card_in_region << card_pos);
uintptr_t res = (((uintptr_t)orig_value & ~SizeFieldMask) | value); uintptr_t res = (((uintptr_t)orig_value & ~SizeFieldMask) | value);
return (CardSetPtr)res; return (ContainerPtr)res;
} }
inline G1AddCardResult G1CardSetInlinePtr::add(uint card_idx, uint bits_per_card, uint max_cards_in_inline_ptr) { inline G1AddCardResult G1CardSetInlinePtr::add(uint card_idx, uint bits_per_card, uint max_cards_in_inline_ptr) {
@ -64,8 +64,8 @@ inline G1AddCardResult G1CardSetInlinePtr::add(uint card_idx, uint bits_per_card
if (num_cards >= max_cards_in_inline_ptr) { if (num_cards >= max_cards_in_inline_ptr) {
return Overflow; return Overflow;
} }
CardSetPtr new_value = merge(_value, card_idx, num_cards, bits_per_card); ContainerPtr new_value = merge(_value, card_idx, num_cards, bits_per_card);
CardSetPtr old_value = Atomic::cmpxchg(_value_addr, _value, new_value, memory_order_relaxed); ContainerPtr old_value = Atomic::cmpxchg(_value_addr, _value, new_value, memory_order_relaxed);
if (_value == old_value) { if (_value == old_value) {
return Added; return Added;
} }
@ -73,7 +73,7 @@ inline G1AddCardResult G1CardSetInlinePtr::add(uint card_idx, uint bits_per_card
_value = old_value; _value = old_value;
// The value of the pointer may have changed to something different than // The value of the pointer may have changed to something different than
// an inline card set. Exit then instead of overwriting. // an inline card set. Exit then instead of overwriting.
if (G1CardSet::card_set_type(_value) != G1CardSet::CardSetInlinePtr) { if (G1CardSet::container_type(_value) != G1CardSet::ContainerInlinePtr) {
return Overflow; return Overflow;
} }
} }
@ -268,23 +268,23 @@ inline G1CardSetHowl::G1CardSetHowl(EntryCountType card_in_region, G1CardSetConf
inline bool G1CardSetHowl::contains(uint card_idx, G1CardSetConfiguration* config) { inline bool G1CardSetHowl::contains(uint card_idx, G1CardSetConfiguration* config) {
EntryCountType bucket = config->howl_bucket_index(card_idx); EntryCountType bucket = config->howl_bucket_index(card_idx);
CardSetPtr* array_entry = get_card_set_addr(bucket); ContainerPtr* array_entry = get_container_addr(bucket);
CardSetPtr card_set = Atomic::load_acquire(array_entry); ContainerPtr container = Atomic::load_acquire(array_entry);
switch (G1CardSet::card_set_type(card_set)) { switch (G1CardSet::container_type(container)) {
case G1CardSet::CardSetArrayOfCards : { case G1CardSet::ContainerArrayOfCards: {
return G1CardSet::card_set_ptr<G1CardSetArray>(card_set)->contains(card_idx); return G1CardSet::container_ptr<G1CardSetArray>(container)->contains(card_idx);
} }
case G1CardSet::CardSetBitMap: { case G1CardSet::ContainerBitMap: {
uint card_offset = config->howl_bitmap_offset(card_idx); uint card_offset = config->howl_bitmap_offset(card_idx);
return G1CardSet::card_set_ptr<G1CardSetBitMap>(card_set)->contains(card_offset, config->max_cards_in_howl_bitmap()); return G1CardSet::container_ptr<G1CardSetBitMap>(container)->contains(card_offset, config->max_cards_in_howl_bitmap());
} }
case G1CardSet::CardSetInlinePtr: { case G1CardSet::ContainerInlinePtr: {
G1CardSetInlinePtr ptr(card_set); G1CardSetInlinePtr ptr(container);
return ptr.contains(card_idx, config->inline_ptr_bits_per_card()); return ptr.contains(card_idx, config->inline_ptr_bits_per_card());
} }
case G1CardSet::CardSetHowl: {// Fullcard set entry case G1CardSet::ContainerHowl: {// Fullcard set entry
assert(card_set == G1CardSet::FullCardSet, "Must be"); assert(container == G1CardSet::FullCardSet, "Must be");
return true; return true;
} }
} }
@ -298,38 +298,38 @@ inline void G1CardSetHowl::iterate(CardOrRangeVisitor& found, G1CardSetConfigura
} }
} }
template <class CardSetPtrVisitor> template <class ContainerPtrVisitor>
inline void G1CardSetHowl::iterate(CardSetPtrVisitor& found, uint num_card_sets) { inline void G1CardSetHowl::iterate(ContainerPtrVisitor& found, uint num_card_sets) {
for (uint i = 0; i < num_card_sets; ++i) { for (uint i = 0; i < num_card_sets; ++i) {
found(&_buckets[i]); found(&_buckets[i]);
} }
} }
template <class CardOrRangeVisitor> template <class CardOrRangeVisitor>
inline void G1CardSetHowl::iterate_cardset(CardSetPtr const card_set, uint index, CardOrRangeVisitor& found, G1CardSetConfiguration* config) { inline void G1CardSetHowl::iterate_cardset(ContainerPtr const container, uint index, CardOrRangeVisitor& found, G1CardSetConfiguration* config) {
switch (G1CardSet::card_set_type(card_set)) { switch (G1CardSet::container_type(container)) {
case G1CardSet::CardSetInlinePtr: { case G1CardSet::ContainerInlinePtr: {
if (found.start_iterate(G1GCPhaseTimes::MergeRSHowlInline)) { if (found.start_iterate(G1GCPhaseTimes::MergeRSHowlInline)) {
G1CardSetInlinePtr ptr(card_set); G1CardSetInlinePtr ptr(container);
ptr.iterate(found, config->inline_ptr_bits_per_card()); ptr.iterate(found, config->inline_ptr_bits_per_card());
} }
return; return;
} }
case G1CardSet::CardSetArrayOfCards : { case G1CardSet::ContainerArrayOfCards: {
if (found.start_iterate(G1GCPhaseTimes::MergeRSHowlArrayOfCards)) { if (found.start_iterate(G1GCPhaseTimes::MergeRSHowlArrayOfCards)) {
G1CardSet::card_set_ptr<G1CardSetArray>(card_set)->iterate(found); G1CardSet::container_ptr<G1CardSetArray>(container)->iterate(found);
} }
return; return;
} }
case G1CardSet::CardSetBitMap: { case G1CardSet::ContainerBitMap: {
if (found.start_iterate(G1GCPhaseTimes::MergeRSHowlBitmap)) { if (found.start_iterate(G1GCPhaseTimes::MergeRSHowlBitmap)) {
uint offset = index << config->log2_max_cards_in_howl_bitmap(); uint offset = index << config->log2_max_cards_in_howl_bitmap();
G1CardSet::card_set_ptr<G1CardSetBitMap>(card_set)->iterate(found, config->max_cards_in_howl_bitmap(), offset); G1CardSet::container_ptr<G1CardSetBitMap>(container)->iterate(found, config->max_cards_in_howl_bitmap(), offset);
} }
return; return;
} }
case G1CardSet::CardSetHowl: { // actually FullCardSet case G1CardSet::ContainerHowl: { // actually FullCardSet
assert(card_set == G1CardSet::FullCardSet, "Must be"); assert(container == G1CardSet::FullCardSet, "Must be");
if (found.start_iterate(G1GCPhaseTimes::MergeRSHowlFull)) { if (found.start_iterate(G1GCPhaseTimes::MergeRSHowlFull)) {
uint offset = index << config->log2_max_cards_in_howl_bitmap(); uint offset = index << config->log2_max_cards_in_howl_bitmap();
found(offset, config->max_cards_in_howl_bitmap()); found(offset, config->max_cards_in_howl_bitmap());

6
src/hotspot/share/gc/g1/heapRegionRemSet.inline.hpp
View File

@ -78,7 +78,7 @@ public:
}; };
template <typename Closure, template <typename> class CardOrRanges> template <typename Closure, template <typename> class CardOrRanges>
class G1HeapRegionRemSetMergeCardClosure : public G1CardSet::CardSetPtrClosure { class G1HeapRegionRemSetMergeCardClosure : public G1CardSet::ContainerPtrClosure {
G1CardSet* _card_set; G1CardSet* _card_set;
Closure& _cl; Closure& _cl;
uint _log_card_regions_per_region; uint _log_card_regions_per_region;
@ -98,11 +98,11 @@ public:
_log_card_region_size(log_card_region_size) { _log_card_region_size(log_card_region_size) {
} }
void do_cardsetptr(uint card_region_idx, size_t num_occupied, G1CardSet::CardSetPtr card_set) override { void do_containerptr(uint card_region_idx, size_t num_occupied, G1CardSet::ContainerPtr container) override {
CardOrRanges<Closure> cl(_cl, CardOrRanges<Closure> cl(_cl,
card_region_idx >> _log_card_regions_per_region, card_region_idx >> _log_card_regions_per_region,
(card_region_idx & _card_regions_per_region_mask) << _log_card_region_size); (card_region_idx & _card_regions_per_region_mask) << _log_card_region_size);
_card_set->iterate_cards_or_ranges_in_container(card_set, cl); _card_set->iterate_cards_or_ranges_in_container(container, cl);
} }
}; };

4
test/hotspot/gtest/gc/g1/test_g1CardSet.cpp
View File

@ -165,13 +165,13 @@ void G1CardSetTest::translate_cards(uint cards_per_region, uint region_idx, uint
} }
} }
class G1CountCardsOccupied : public G1CardSet::CardSetPtrClosure { class G1CountCardsOccupied : public G1CardSet::ContainerPtrClosure {
size_t _num_occupied; size_t _num_occupied;
public: public:
G1CountCardsOccupied() : _num_occupied(0) { } G1CountCardsOccupied() : _num_occupied(0) { }
void do_cardsetptr(uint region_idx, size_t num_occupied, G1CardSet::CardSetPtr card_set) override { void do_containerptr(uint region_idx, size_t num_occupied, G1CardSet::ContainerPtr container) override {
_num_occupied += num_occupied; _num_occupied += num_occupied;
} }

2
test/hotspot/gtest/gc/g1/test_g1CardSetContainers.cpp
View File

@ -83,7 +83,7 @@ void G1CardSetContainersTest::cardset_inlineptr_test(uint bits_per_card) {
G1AddCardResult res; G1AddCardResult res;
G1CardSet::CardSetPtr value = G1CardSetInlinePtr(); G1CardSet::ContainerPtr value = G1CardSetInlinePtr();
for (uint i = 0; i < CardsPerSet; i++) { for (uint i = 0; i < CardsPerSet; i++) {
{ {